diff --git a/src/lecture8.ipynb b/src/lecture8.ipynb index 48718a4..6eb6f02 100644 --- a/src/lecture8.ipynb +++ b/src/lecture8.ipynb @@ -1,819 +1,774 @@ { "cells": [ { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "# Lecture 8\n", "\n", "## The MNIST data set" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "IRdisplay::display_html('')" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Let us load the MNIST dataset and plot some examples of the training set." ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0gAAANICAMAAADKOT/pAAAByFBMVEVNTU1OTk5PT09RUVFS\nUlJTU1NUVFRWVlZXV1dYWFhZWVlaWlpbW1tdXV1eXl5fX19gYGBhYWFiYmJjY2NkZGRlZWVm\nZmZnZ2doaGhpaWlqampra2tsbGxtbW1ubm5vb29wcHBxcXFycnJzc3N0dHR1dXV2dnZ3d3d4\neHh5eXl6enp7e3t8fHx9fX1+fn5/f3+AgICBgYGCgoKDg4OEhISFhYWGhoaHh4eIiIiJiYmK\nioqLi4uMjIyNjY2Ojo6Pj4+QkJCRkZGSkpKTk5OUlJSVlZWWlpaXl5eYmJiZmZmampqbm5uc\nnJydnZ2enp6fn5+goKChoaGioqKjo6OkpKSlpaWmpqanp6eoqKipqamqqqqrq6usrKytra2u\nrq6vr6+wsLCxsbGysrKzs7O0tLS1tbW2tra3t7e4uLi5ubm6urq7u7u8vLy9vb2+vr6/v7/A\nwMDBwcHCwsLDw8PExMTFxcXGxsbHx8fIyMjJycnKysrLy8vMzMzNzc3Ozs7Pz8/Q0NDR0dHS\n0tLT09PU1NTV1dXW1tbX19fY2NjZ2dna2trb29vc3Nzd3d3e3t7f39/g4ODh4eHi4uLj4+Pk\n5OTl5eXm5ub////2EHiKAAAACXBIWXMAABJ0AAASdAHeZh94AAAgAElEQVR4nO3d+Z8kVZm2\n8e5GR1FQcUNcwXHHBXFfx1FRRMcRfcENcBkVFLdxX0BARBEVFbXn333zvrs6TkXkcyLiRD6R\nlV1c35+qa8l6KiKu9MMxljP/B2BnZ056AOA0ICQgASEBCQgJSEBIQAJCAhIQEpCAkIAEhAQk\nICQgASEBCQgJSEBIQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBIQAJCAhIQEpCAkIAEhAQk\nICQgASEBCQgJSEBIQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSLAJ6fyB0DgnPcNFzBJjlhgh\nVTBLjFlihFTBLDFmiRFSBbPEmCVGSBXMEmOWGCFVMEuMWWKEVMEsMWaJEVIFs8SYJUZIFcwS\nY5YYIVUwS4xZYoRUwSwxZokRUgWzxJglRkgVzBJjlhghVTBLjFlihFTBLDFmiRFSBbPEmCVG\nSBXMEmOWGCFVMEuMWWKEVMEsMWaJncqQHpM75ePyYbl142fyT9nfLBmYJXZYsxDSqrNkYJbY\nYc1CSKvOkoFZYoc1CyGtOksGZokd1iyEtOosGZgldliznK6QviovlLMVt8leZkmz2yx+W3lA\n/lueK5+WE5gl12HNQkhrzZKGkGKHNQshrTVLGkKKHdYshLTWLGkIKXZYsxDSWrOkIaTYYc2y\ne0gPybvER+rrZOEwC2fxSpyX6c6Jx3i2fEO8WHfPxo3iL35f1pmleFRukTOd18vjGy0vtGSW\n+8VvGs+TM33PEM/ygQ1/79/lX5I+y8C7ZTDQfdL6QoRUHYaQsmYhpP0ipOxZCkKKEVIFIcUI\nKUZIFYQUI6QYIW0ri2SXiQ9jf/QTaR9m4Sy/lbIw90H5vfS/TQfw4/8l75N1ZnHU/uXXyrnA\ndzZaXrJ1Fm2Np8uZVj+W1FnEldwl47+99XUJqToMIWXMQkj7R0jZsxDSFkKaREhbCGkLIU0i\npC2EtIWQxvxAysk4/ugGuUJeIFrNaxumfZY/yNXiMW6XJ6T2A/7i3yR9Fi98fUWifApdG9Xy\nuq2z/Hxj/JB9vnjprv/5X0jqLGpobsb9n/SnRlfz9hfSn8W79xHxG7ffkY/NQkhpsxDSNkKa\nQkhbCGkbIU0hpC2EtI2QphDSFkLaRkhTdDTc6hU6nxPkq178hU+KDxgd3j5FZuZLLtkwPgvG\n2/w/xH9s62tkzKK1Ot9lZRDNW+XbHV8SpGuDZt6DZdEsoyF9S/xtPn9Kn3qV3C3/kKxZ7puM\n6K5OP5hyEpE/SphlJh8+PuVN28ZXbb1BrhS/+zxF/M9be++HhJQ0CyFtIaRZCKmHkLYQ0iyE\n1ENIWwhpis9nKEsM5RwCz3GH+Ks6kPzBL2X6dVs3jP7L2P/F/CyJGvLh+leZvD5gx1m+uTFo\n6Avi/0z1d/hYea/oi+WNJ3+W0XMI3iH+r2VvMA3V8u7TMEttgtE4ohMgEmap+Y18Wfw+7OPZ\nv1MnhnhQryl8TbwDvdgQrFURUtIshLSFkKYQ0jZC2kJIUwhpGyFtIaQphLSNkLYQ0hhF43MX\nfKyUlQvX9FHpX1DhD0Y33fFh2mZ524ZD/ZgMvugj+Ivib7lpwydjTC9Mtc7iqzNeuVEaepn4\nVgh+D/mU+N4J3XesGZIWD8th6CUm/+n/Lv7c9D0rdpulds3E+K0ZglU+v9BuswR+JC+Xcu2N\n3+J8Yozv8TF2ckw8CyHtOgshbSGkSYS0hZC2ENIkQtpCSFsIaRIhbSGkLYQ06eJq3IXlOJ8d\n9Fopn+t/5A/yQ/L5/jpbw7/tYSlfdEPXy9ltWl77Zuosf5RzDfQ8iLKalzqL9UPybTH9+Ztl\nLyH17wHpf403ZEF5qSFpCddXuTxNvEznaPymNn1vzFGEtPsshLSFkCYR0hZC2kJIkwhpCyFt\nIaRJhLSFkLYQ0iQdCIObQQ4+8kMOtEh2U/epcg3MxDCzZ/Eai7rw5hh80deRuJoXy3dFE/gB\nFVrqe5vPGsqaxU3oj54bktbwZr526yzWD+nj4rNg/E9vF58Z0/KSjbP0a3BI1ST8heAJFdPv\nvq3bRdH4z3+TtFwRNo2Qdp+FkLYQ0iRC2kJIWwhpEiFtIaQthDSJkLYQ0hZCmrS9LnfsI59h\n4TNmfLKQPuUTMR6f97DHhJC8dXzJjafy4zC7L/opFf789MMyWneSLunxUqKvX3m7eJHM126V\nhnzjzOn1q8Wz+B3iORtnajyGT+1a8YkhQRfNppf5GrbLn6Rbx/WZY7oFzQMti6cTsxDSrrMY\nIR1HSJMIKUZIxxHSJEKKEdJxhDSJkGKEdBwhTdJuGKzV+Y4R5f4mvjKp+6ofUDF7mJ1D8sHh\njeWlu/55Hz8Vf/H9kjXLqF9JCekT0voau63aVb1UtJKVf9fX2vVIM42eGNQ6izmkt2z4Zjln\nO760zuuaOyZFSLvPMoqQCClESG0IiZBChNSGkAgpREhtCImQQnpKi3+7l8DulcF3OJ1uJa9l\naSghJF8b5V/uB6j0f6icOnSPZM0yyluohORTdVpfY6WQOv21zbRZajWVJb13d/pfzJ+l4zcN\nnyH1v+IZvcrr99bBhW0NCGn3WUYREiGFCKkNIRFSiJDaEBIhhQipDSERUlV5jkv0hXIfVl2j\n0zbM7Fk+IkpCdzh9pe9e4WWXa8St+AyQ8gP6juvkRVL9C5bMUuP7efoaKG+N98gTrXfwbJzF\nr6/tPjh4nyl+nq0f8buXkDrlZB8ft4Nzf/qzzFywWzxLxHH5br0+Qha9BiHtPksNIR0hpApC\nmoWQjhBSBSHNQkhHCKmCkGYhpCOE1Gpwz1U9TLZtmNmzOBPdL9O/qJx44wleJYPnfL5hw9/7\nPUmdpcbrQGXBzlcrLXqhhll85Vf/0PQNYMoVYX6szX5Dqrpv+0EuLT8+OYvPCZv5Yr6Sy1fP\nTS/phrMQ0q6z1BDSFEKqIKTjCGkKIVUQ0nGENIWQtvUvnrhwaUXD0x2PhmmbRedWeOs/XXwf\nSDf0b+Kd5Kn84ElN5vsC6uYKf8+fpe+H4mdOepO8RPzQikWv1jDL3dIdlbpu4C3lIZu/kzdK\nOXC1qf4g+bNMG9zaIf1eFn4Qh09jGL1IwoeET3vxkdPyVNJjsxBSyix9hDQLIW0jpOMIaRZC\n2kZIxxHSLIS0jZCOI6RZCKnHv9jldBdPnIuur5gxTNssD274jotnp12+8ZCsNEvnESlnd7ih\nll+82yy+akXnbuhyl1vKWp1vHeENcabvMxsrzTJtMEvrj0/O4vW3l4lPMLlqw++0+uAqP5bE\nH/kdz98287Ep4SyElDJLh5DmI6QeQjqOkOYjpB5COo6Q5iOkHkI6jpDmI6QeP3mi3DTST72c\n+fyJ4TDtsyimB7s7OAx5IK9TNSxMLZ7FvHhYTgyavhVl/iw6Oapcr+FzgrwmNThuPyv64pqz\n1AzODmq5emLJLI+KztFyKj4n6kPiW37cJtNPQ52YhZDSZjFCmoWQthHScYQ0CyFtI6TjCGkW\nQtpGSMcR0iyEdJHX6tyQt4SPGn/0W1k4zMJZfKz8QL4v13fPOPSF+HubxRNoefBybw0/7WH5\nLQd3m8W8j3z7wzOBz8veZhmI7h55UrNkIKTEWQhpPkK6iJC2ENJ8hHQRIW0hpPkI6SJC2kJI\n8xHSRT4naPAQzNfJonsgHg2Ts2E8gRvy/SIXPdRwySw3S1mw+7m0vsaOs/heh9/YeLv48pqo\nITeub/vGirPUBLc8OXNf67VISbOkIaTEWQhpFkLqIaQthDQLIfUQ0hZCmoWQeghpCyHNQkg9\nvhCorNW5q52HOaQN0zaLFux83HqT+K4bPqj3PEtwp9XC5wn5hq+LzuBqnaVmsGDX8rDL9FnS\nEFIFIe08Sw0h9RBSDyHNRkg9hNRDSLMRUg8h9RDSbITUc6uUu6ouPzHo+DCHtGHaZuluNOun\nqEw/WnPNWdaUNUu5yeqiS5FSZ8lASBWEFCOkGCFVEFKMkGKEVEFIMUKKEVIFIcUIKZb7DNkd\nHdaGYZYIs8QIqYJZYswSI6QKZokxS4yQKpglxiwxQqpglhizxAipgllizBIjpApmiTFLjJAq\nmCXGLDFCqmCWGLPEHBKAXRESkICQgASEBCQgJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQg\nJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQgJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQg\nJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQgJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQg\nJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQgJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQg\nJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQgJCABIQEJCAlIQEhAAkICEhASkGAT0vkDoXFO\neoaLmCXGLDFCqmCWGLPECKmCWWLMEiOkCmaJMUuMkCqYJcYsMUKqYJYYs8QIqYJZYswSI6QK\nZokxS4yQKpglxiwxQqpglljWLHfJM+W3cqKzZCCkCmaJEVKMkCqYJUZIMUKqYJYYIcUIqYJZ\nYoQUI6QKZollzXKNnJMfyInOkoGQKpglRkgxQqpglhghxQipgllihBQjpApmiRFSjJAqmCVG\nSDFCqmCWGCHFCKmCWWKEFCOkCmaJEVKMkCqYJUZIsdMR0g1yVn4pJzrLCrJmua/jf94tn904\n1/GnfiYrzvJzeZo8Qx6QRS+02yy/l1vlXeLD53Vyhzy24W/7qviL35L6LISUO8sKCGmNWQhp\nGyHNQkjHEdI2QpqFkI4jpG2ENAshHUdI2xySjwRvmBOdxf4u3iE/6dy67Rfi9h/qpM9iXhp7\nbudV4g12WY8/daV8R1acxb/OS3fLX2jJLM7Hx8sVUrbBu8V7xTVdveEfmHlsEVL2LEZIU7MQ\n0ooIiZBaEFIFIRFSC0KqICRCanEJhvTItn9I+Y6/iL/wI3mDaG891i1/TFuyYXzI+k/3ysro\nH/u4rDhL4Ya8f67olH3W+2D40Q2d3WfxPvq63Cn9aC7rh/Qi0eFztf/5Gglecrft4hudvFD8\ny2vvHDO1zqJD0b/ch4on+KDUjovHuh84I4R0ESEREiFVENIIQuohpBGENIKQeghpBCGNIKSe\nEwvpYXmPBAs8N4qXOvwd10mwkz4lM//QJTvJ+6L8yuofqxM9XiFetllnlrJWV/aUbybqkMrn\neh9c+Mh7avDRTrP4Pc0bP4jmAu+8ey76nXiPu6bnyf2y+3YpPiBlguUvtGiW7t3NW+MmGf3+\nj0r5geWnCBFSG0KaQkiENAMhTSEkQpqBkKYQEiHNQEhTnnwhecHCZ5NU98PY58sXvBg08w9d\nspO+KeWorD4jRNvE33avrDPLHf095Y9+3/F+CM4TKqelDH508SxdDdGO8tFxu9R+3OcP+Xs/\nLv0vLg7p1/ICqb10s9ZZ9J5bDhVv8ur36i2kvM95FXV00ZeQEmchpBGEREhzEdIIQiKkuQhp\nxJM0pLeL/+KXiR9U+ONtXmd4r3xI/LnPSLfj3iwz/9AlO8n/x7MH9UfRiRQ+jLsDKfgP6IxZ\ntJFfK/7LXy4tJ1JkzeI1hn5DJaTPyRMy+holpH7PrbN0vCG8euEX9Os/KuVbfiW6vOI28V8x\n/bqts+gvf70M3ucG3+bPaV3qXMNBQ0hJsxBSHSER0myEVEdIhDQbIdUREiHNRkh1T+aQyqLP\nb6Rl3G9LN8NXZOZPtm4Y3QmhrKxUV6K620f621Y6s0Gn45cN/1Np+fGEWXzNRDmXoc/vbtMN\nWQnJ5S2Zpc93hCyzDE52+b5cLt13eKl3uqYls3jRthw0fu8r2+X+3mKdP/ihTL8uISXNQkh1\nhERIsxFSHSER0myEVEdIhDQbIdU9mUN6UJzEX6VlVK+L+GKA52y0/HjDhvFC0MyVle4Wf/62\n/Fl8m8duw/s8oZk/mTXL4JqJvuaTcUpIvr6icZYBPfuiXBniNVzvO19yUm6++HT50sazxD9Q\nu0Xl8lnEv3xwQpZb8U08X9E7pLwrZ777ENLusxDSCEIiJEIiJEI6hpAIqUNIfYR0HCHNtu+Q\nlvFdIn16jIdp3YMNG8Z/Y3fw+reVBzr4yPblPz5jqLt9pG9SkD+LLyjShi8nKXl33du5vW/m\n686fRUfgl85t81pdy2/yoy/9o74WLeGeDd1kPm59JpDfW/24SX/BNflqJX2/H1Dhz79JRl97\nSUjmldvgRhoXPvIBPO9EsuOzENKusxDSxGiE1IqQCCkYjZBaERIhBaMRUitCIqRgNEJq4ENn\ncLcUPdOxbZg5s3h5pds3ZSdFH5WrlfSp0ccJLJrF+qeUnJv+6JaNma89b5bBFUTis2yan/bw\nWfHPJzyN4s+iq5D8R79PfJSUR1/6Riz+tu6H9hKSlSXDctB4xXnmSWTDWQhph1mMkGKEtAQh\nEdIAIS1BSIQ0QEhLEBIhDRDSEl6wK/vRY+n0kLZh5szSXV7UdPDqoRAtW6hhJ82Munyk5aKs\nWb4rV0o/JF8DNPP1rX8Z2WXvkMZZBpyOXsv3V/VanaPxDvSzUYMfKiHdMvl+szgkL+sOHoLp\nX7f8fjWEtNMsRkgxQlqCkAhpgJCWICRCGiCkJQiJkAYIaYlBSL6v5h822oaZM8vXxH+7nl2o\nlcFfliej+HmG5Wkp5cDQzsufxXwWkG5+Uk4C8l7xoOU8IafjL2jnVZ8/0zhL+Qv7BncyHeXD\nx4tkfjUf49Eta6dmGfArapb/FH/Ky3R+fR8vgx/Q6URPFZ+9M72FloSkR6B+NXp3W3Ty1vFZ\nCGmHWYyQYoS0BCER0gAhLUFIhDRASEsQEiENENISfvDLuY7PNWkfJvHaqP5NM8/Ne3LsSrNY\nd7+WV9SeP7NklnJjEZ+f5Rvb6l9zQ/rjRnnUyRtldOlq3nbx25tfUUn4U16wu1YGd1o135q2\nO3q8gjj9W1r3Uf9xw/7IC4rlcy1vb8NZCGn9WYyQCGkGQppCSIQ0AyFNISRCmoGQphASIc3Q\nLdFcMHNTDIdJPHg/2H/QSvvNLCZm8a1CZ+bgBbvyhFlNljVL+Qt9BZEjUEP/lPGX9Yky3W7z\nWlrW3U0dkl/2ixv+lE8Rul6+IP6c4/q6vET0/Z+W6V/QMIt5tbS7OM1ru46mu3FN/RHEc2ch\npGWzENIIQlqMkEYQEiHNRUgjCImQZviZPEVKSM13C7gwTE5I3+z/V2XLfSHnz+I7D0y/tP+f\ndP8/9R7I/13feq5+Q0jTL/Yn8cHb/ejLxH/PbrMU3v3Plus2ftnxEexN8gHx2kM5aHTlRv7T\nSzxLdzQMHhZSzn/xbSFP9jIKQhpBSIQ0FyGNICRCmouQRhASIc1FSCMIiZDm8nMfyuZ4mywc\nZudZvD7Wvy2k71PZdKPKWbOUXRB80Ut6vmag/J/ovlXBoj01NsvHxH+rr1ypPf9Tt8+47zbR\nkX1d2Thjl0y0zjLQLeSW2Af8BT+XwifGzHvIZOss3g3+dcGTLMuq3Q2tF9oMZyGkZbMQ0hRC\nakRIWwjpPCE1I6QthHSekJoR0hZCOk9Izd4jHsbLP/2nC7QNs/MsPselXFqwzsFrZRd0n/Iv\n8rX/vn9EOVbulF1WhOqzuJCrxL/JXd29rWyS4seSOcuA/+APbUQh3Si+i+UTrf00zlL+9OBS\nGldWDpVFYxzNQkjLZiGkKYTUiJCOEFIPITUipCOE1ENIjQjpCCH1EFIjnydT9s8jsnCYnWcZ\nXIW0zsFr5fIin1yjFaGyy8panc+HWTRBwyzlYZI1ZZP4CP6erDPLHs2bpbyp6X4Zn5Tulp7H\nDhXfznOXWQhp2SyEdOIIqToMIS2ahZBGENIShERIA4S0BCER0sClE1JZtbOTCqlcd+IDetFV\nSA2zlF3QfVQ+5ZOUFp2atGgWv1+4Jl+U1O2Ky8Wf+ox4oOXvLfNm2aN5s/gP9ppc8D7nj7y3\nfiK7zEJIy2YhpBNHSNVhCGnRLIQ0gpAaERIhRQipESERUuSSCMnPPLhSTj4kX1VztXjDLH+w\nwLxZfBlN6VYf+P4ZO67+LJql+Jfcc9HMG5qsNMvqWk9X0kldPkC823wfGp/oteOhcjQLIS2b\nhZBOHCFVhyGkRbMUhBQjpPkI6Twh1RDSfIR0npBqDj2kcvCefEi5JmfxIuGtnYQFseWz7BGz\nxAipgpBizBIjpApCijFLjJAqCCnGLDFCqiCkGLPEdgzJ0VzW9yQJaY+YJXZYsxBSiFlizBIj\npApmiTFLjJAqmCXGLDFCqmCWGLPEMq5HSnNYG4ZZIswSI6QKZokxS4yQKpglxiwxQqpglhiz\nxBwSgF0REpCAkIAEhAQkICQgASEBCQgJSEBIQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBI\nQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBIQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBI\nQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBIQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBI\nQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBIQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBI\nQAJCAhIQEpCAkIAEhAQkICQgASEBCQgJSEBIQAJCAhIQEpBgE9L5A6FxTnqGi5glxiwxQqpg\nlhizxAipgllizBIjpApmiTFLjJAqmCXGLDFCqmCWGLPECKmCWWLMEiOkCmaJMUuMkCqYJcYs\nMUKqYJYYs8QIqYJZYswSI6QKZokxS4yQKpglxiwxQqpglhizxC7BkB6WL130TjkbOCP/Tx6X\nJzbyZ9kPZokd1iyEtMMs+8EsscOahZB2mGU/mCV2WLMQ0g6z7AezxA5rFkLaYZb9YJbYYc1y\nKYX0R3mJnJOrNi6Tc9PesXGHPDEvqcPaSZf2LA/IvfI68Vvc4G3vh7KPWSIezWP8Vha9BiEt\nnWWPLvVZCGnPCCl2qc9CSHtGSLFLfRZC2jNCil3qsxDSnk1umL/Kx+RGuX/DS3fvFW+OG/v+\nTfpF/VISZtmjvczyY/Ex/hNJmMV7S1v7xTL+Pvcc+fVGy8gJ28UrujeIxyAkQtoVIRESISUg\nJEIipASEREiElICQnkQhNfuN3CXdLvuw/E3WnuXP8nv5vHxUPi6/2mh5ocWzPCY3yYvktfJz\n6X/b3+Va8dLV9yRhlg/KeD99Srnlz0vYR98U//JnivfWohcipPVmISRCOhmElDwLIU0hpHkI\niZBGEdI8hERIo/YY0r/kM+Ilnedu+EQPL57d0efD5EzFW+UfUh9mlaj/KTdvlF32WRn9od1m\n+ZlcI9GWeNXGD2Tmqy2Z5dvyevEf7EL80cvF51p13/sL8RefLl7J2m2W26VfiQ/Zf/Q59BMN\n6VbxaWZeulv+QoS0yiyEREhDhLQAIRHSECEtQEiENERICxASIQ15s0Z3F2nl03W8QFQfZr0V\nxM9tlF3md4HR72+dRbE6Dh+o/iXeP7eIV+2U8s3eEorp3eI3qenXbp3l6xtOwmPoX1/XQuGv\n/D7nz/nEKn/voxvPFn/ex1XCLL7pTJfmneK3s8G3eV2z7BW907b8nVkh+ZcT0jyEREgBQmpF\nSIQUIKRWhERIAUJqRUiEFNhjSP8rQRfPEu+Cp4r/6VaeJ0+R/g98WSaGWS8kHTBrhvSFjbIu\n9xoZnIzj95BXi77DZ6RoqEenX3veLF4H+450u+fX/St8NOMXvC8+Jb8Trev5U64sa5YSkhqq\nftsgJK0IT7926yyj9rhqR0izENJxhLSNkGYhpOMIaRshzUJIxxFSj++leL30k/DR4P9z3Fvi\np+L/k9r/F7l/9PXdDhL/l/X0nRnXDEmXBpRd5v/fP2sWX3agtxA35PtXlss0vF28CvEm6VJ7\nj8z8BfNm8T0W/depoajRL4q/w9eSvEX0L2c/es5J6yze1747xuh1CYOQ3r8x/dqts4zaz2ID\nIc1+aULqIaQeQpr90oTUQ0g9hDT7pQmph5B6CGn2SxNSDyH1eJ3Jyz/Owatzb94YX2zyTfb0\noIirupBKYBPDrBLSn0S3K/AGu0K8YJUwi09N8NM3FYdT8W0RveVcq+98OLiMQm8rD8rMP2De\nLP7t/hNrlyOUkMy75hUbj0jqLDMNQjq1l1EQ0hRC2gkhEdIFhLQTQiKkCwhpJ4RESBcQ0k6e\nLCGZHvlwv+4y8Ko7R0/36Pj0k64h3xbxZB836bm7vTW9YNcwy33SBeLlO58Y9DQ5E/DNKicr\nXjRLCen5Gx+Qu8U3yfQh+9ru3cS8vth6R8TUfeQTmk40pHeJfzkhTSIkQqohpAaEREg1hNSA\nkAiphpAaEBIh1ewxpBZerPLu6kLyKTKzh1klpE9It7dSH335Dekq8RKQ33Oihuxhaf0D5s3i\nZ1e6H51q45TP1rihmVdDLZllmnfD5eK94oeXznu7TZ2lrNp5NfdQHutCSIQ0EyGNISRCmomQ\nxhASIc1ESGMIiZBmIqQx35X+LjvZkHy7j64hP0Vj9BaVrbP4GqBqNYG3S3CrxIRZ+nxfz3vE\nq3neF94IvuvKQ9L6kotnifgSrbJgN/PWK7vN8thF5VPleiTf2HP0QRwTsxDSDrMQ0lKENIaQ\nCGkmQhpDSIQ0EyGNISRCmomQKnwZTv+WJ2cvrsTMXYpJDckN+Yav3k66JaxPlkmdxX+1z4LS\ntVqlF1219WY/DdQHTHlyqOlmMS1/ym7bxdGUkMwPp1j0arvN4vOnfF7VlVIGanm/XTzLExeV\nT71Q9nM90kyEREjTCGkSIRHSNEKaREiENI2QJhESIU0jpEneNf0Fu7OteyshJG+n/lrdBXqO\nS/T0xaxZdOrPg53Bb/qLlJD0LMqW1168Xdz5C2QQki8EWvSSrbNojdSnA/l2rr6Tju8HU2b5\nT5m5lrrbLIGyXQiph5COI6QphFRBSMcR0hRCqoOeq88AABCVSURBVCCk4whpCiFVENJxhDTl\n4EPa2/UlfoCK13x8h9dr5VyFL9bxmtH0qSCpK4gnFdIbxftDlyi930929T99g5pFL9k6y+0b\n3vi+QMoPGY52jW8C2z0LZ6VZ+nztkX/5Ia/aEdJxhERIcxHSCEIipLkIaQQhEdJchDSCkAhp\nLt9hpIR0s7QPMzGL1+Qe7uhaEj/itlZOlY+r0dNSUkPy6pnuf/p8h6THurT8+JJZfGvVq+V5\n4ie3lMe67CUk/85XbvhX+ja03u7VvfKMDb/P5c8SuFcOb9WOkEYQEiHNRUgjCImQ5iKkEYRE\nSDP4xgW+v14JaeaTK4bDhLP4NAEfBG+TlmB8Ib7/WzL44svE+3f+LDvwbQkcki65+LXM/Mkl\ns9wi/jN/3D3k4UOyv5B8DOgOEb51hBdcfLR6Ap9voXeUd18n3U5x9q8Wv0VmzRLwY1b8Ow9l\nsYGQZiEkQhpHSLMQEiGNI6RZCImQxhHSLIRESOPeJ6Whl8qfpH2YcJbPysxy3iG6J8JPfasC\nL8w9dJH37DVSfqD2/IHUkHwihd9qHNKLN9a8pEPKn+lzTryIV65f8J0kWl+yYRb/dU8XPZrE\n//JpJf7lPr3hZ+Lv1YNAf/dl0RuMv8O32rhKEmYZ5f3h3/kWWf5ChERITQipNgshEVIDQqrN\nQkiE1ICQarMQEiE1IKTaLEtD8gkgfs6jzux4xo4LdkfDhLOU0zgC3uafFz/Jcfra/2/JrZ3a\nD6y5aqfnTrb85JJZXiTecg7pLvE//WRSd9X6kg2zeKVQXTxVY/id1ntLt+l8dvXho1pAvU3u\nlqxZRt0kJ7tqR0iNCImQIoTUiJAIKUJIjQiJkCKE1IiQCCkSPObyrM/IWfRggaNhwlm8baOI\nPrfhM08W/bZFsyzj66i8LLS3kMrpQA7YzwnxP32uVeurLZrFyfb3mG/lsfyXL5+lpjz60nvG\n9/1YOAshtcyyDCERUhUhzUdIhFRFSPMREiFVEdJ8hERIVR+RQUjLF+yOhsleKVsudZbBDSJ1\nZK89ix/zWe7H6N3jw9irra2vtmiWP4iuLPIEvihp0ZljGbPU+F3fS3d+COYu/yNASCFC2nkW\nQpqFkOYjJEKqIqT5CImQqghpPkIipIgvr3mDpC3YHQ3z5Ahpb0+j+Kj4MPZ1WrVrr/YyywoO\naxZCChFS8iwrOKxZCClESMmzrOCwZiGkECElz7KCw5qFkEKElDzLCg5rlgUh+XYVgwU7P7Bj\n52EOacMkzuKLnrw2tN+QVsAsMUKqIKQYs8QIqYKQYswSI6QKQooxS4yQKggpxiyxRSF5Eeo1\nUhbsfOuRnYc5pA3DLBFmiRFSBbPEmCVGSBXMEmOWGCFVMEuMWWKEVMEsMWaJZTzWJc1hbRhm\niTBLjJAqmCXGLDFCqmCWGLPECKmCWWLMEnNIAHZFSEACQgISEBKQgJCABIQEJCAkIAEhAQkI\nCUhASEACQgISEBKQgJCABIQEJCAkIAEhAQkICUhASEACQgISEBKQgJCABIQEJCAkIAEhAQkI\nCUhASEACQgISEBKQgJCABIQEJCAkIAEhAQkICUhASEACQgISEBKQgJCABIQEJCAkIAEhAQkI\nCUhASEACQgISEBKQgJCABIQEJCAkIAEhAQkICUhASEACQgISEBKQgJCABIQEJCAkIAEhAQkI\nCUhASEACQgISEBKQgJCABIQEJCAkIAEhAQkICUhASEACQgISEBKQgJCABIQEJCAkIAEhAQkI\nCUhASECCTUjnD4TGOekZLmKWGLPECKmCWWLMEiOkCmaJMUuMkCqYJcYsMUKqYJYYs8QIqYJZ\nYswSI6QKZokxS4yQKpglxiwxQqpglhizxE5bSA/J++Ws3CcnNUsaZokd1iyEtNYsaZgldliz\nENJas6RhlthhzUJIa82ShllihzULIa01SxpmiR3WLKcrpO/Juc7dclKzpGGW2GHNQkhrzZKG\nWWKHNQshrTVLGmaJHdYshLTWLGmYJXZYsxDSWrOkYZbYYc1yukL6hrih78iJzpKGWWKHNQsh\nrTVLGmaJHdYshLTWLGmYJXZYsxDSWrOkYZbYYc1CSGvNkoZZYoc1y6kJ6YtypTikv8sJzOJ1\nw7fKHbJ8goRZVnCpz/JPeVBeKD5Unin/I3+VhbMQUu4shLQ3hFRBSMmzrOBSn4WQJhHSPlzq\nsxDSJELah0t9loMP6V/yD/Fy2Z3ygo0zndvlCakPs3AW//2vknJ20ImGdFnnC3JSs6zgUp/l\n63JZn48Xf/RS+a20z0JIubMQ0t4QUnUYQkqdZQWX+iyENIaQ9uZSn4WQxhDS3lzqsxxySI/J\nJ+TcJH+vowuHWTjLPVJ+yZfEyzPL/6jFs/gkpcFO8nwnMIv5BjD/IWc7fmfzLmvdSrvN8gP5\nvSx/jeWz/HTjJVKi0ad+6uXVd0j5wsMbbbMQUoiQ1piFkMYQ0nGENIKQxhDScYQ0gpDGENJx\nhDSCkLZ5JerbcrOUw/hp8nzRfN+S54m/+B6pHeNLdtLj8lrx618ny0/02G0Wc8UHEtJHxPvj\nbM39G2vOoqNEB+XDN8lT5HK5Xn4sf6nwjWv8HT7QfQ/d/vJn6yzaFdXFOa8nf0b8LTrfrOW1\nCamCkHJmIaQphBQjpB5CmkJIMULqIaQphBQjpB5CGvF9GZyR83TxyscvpP8Drxd/262SuWqn\nA+H+MsanpfU1IosP3j/Jm6SE5PWx/c2iVbEbxb/cvdwij8if5fPiL+h97rvyI0mdxTl0G6Lq\nXHd8T/M75pJZ/I7f/SKv1UXf5nPcfLQSEiER0jZCmkJI2whpCyFNIaRthLTloELyGoH+s7k0\n5Hy+LD+X4If8n4cfknK0+1gLh9k5JG+w1teI7Pb/4Ps/lMth4qN6L7N4N1yx4Up8M4KfyGB9\nx5/ztzx7wx+8TlJn8RNIp+OYDult4reCX8mSWXwE6nQOv9ePf+8bhZAIiZC2EdIUQuohpBgh\nTSGkHkKKEdIUQuohpNihhOQcPibdIXu1TC/1+MT5c321pbuEkB6V1teIpIbkMzv+KCvO4uW4\nl4u6uEqCBdQLvPF9wkB3isNHJWuWX8u/ibbBW8QX0HhGX9fhezTe0afVxsc7vxR/VLvRx7xZ\nfiN+S9Gu8PE4/gNectXK3SelusI3nIWQQoS0yyyEVEFIbQiJkEKE1IaQCClESG0IiZCGvN7j\n2+p1R+vL5C6prb9d8MCGV1zKga57Rr7Af0xCSJrMS0M+Ej4u/rxP9Ph7X8vrLpmlbxCSP/KJ\nVYtebd4s7xZvCS2lVu9y+DfpN+TnMvxOsmb5sPSX3rzU+1n5g0y/RtYsPto8graLEx//ge78\nLu87/2vOLIQUIqRdZiGkbYS0ACER0hAhLUBIhDRESAsQEiEN+ekKpQSt1XkBpvoD3lM/FD3/\n8Vyf2nqgPkzbwatjpby0r6rxAo/vhujP+UaIOmPmits2vFzkyqZfOz+kN8uiV5ucxWuVzxKH\nUSvWJwZ5BfFsT8sZTA2rdu7zsm0e9H3iR3XM/MWLZ/EZbP7Fz9kYX9b14ekVPkIiJELqIaRJ\nhHSEkEYQ0iRCOkJIIwhpEiEdIaQRBxCSF+z+XcrhqqWxBzpe6vHTBXyykK/6eOt2Py8S3+tv\n9DBuPXi1t8ov8W0I/UueKv6cj5P+LP8to0/gXDRLnzeOL9YqIb1aFr3a5CxeoithfHDjFx3/\nwb7GxrO8QryqtmJI3r4+J0pvqH7oqY+NIKvLfE/PGYtpS2fxo1j9m/Q/AuN7vtz8pAvpXpkz\nCyGlzNJHSIR0DCEtRUiEdAwhLUVIhHQMIS1FSIR0jEN6p5yr0K0znu3TfmrfYV4umzNM28Hb\nuz7qQkj+/M/E5ya5YF8XoyWz8tAKP3VgdJsmrNr5N3nd0B/5HJlFrzY5i3eUb+txdox3pZdc\nfdpM9/n/kaxZpgf1A0q0inrbK6VsIV2AVLsL7/JZyhVxWo2rnjpl/qpXG7v3/9mzEFLKLH2E\nNDIoIRHSXIQ0MighEdJchDQyKCER0lyENDLoky8k89/u61d8Y43RYAau2fC9I6YXyY6GyQmp\n8KUvXcY+eekGqf7A4ln6/NJ+xKN/09qrdua/zmtMukTr+s5XxE90KUumDr0Laeb73LxZvMl/\n3ak85fQYH1beQpp25hlc82Yx33zWz//UrviaDK5Q8y1ZfBPXl0i3nujNN3sWQkqZpY+QCGmI\nkBYgJEIaIqQFCImQhghpAUIipDqf8vHgRV4aUyrXRA29R1rvepofUkDnz3xw7ZDM193sL6QW\n3giO6NqNlrvDTM7ie7CUE4C0POpDpfoDPoGo+/6Z77nzZim+Id2u8JupL5nz/WW1Da4NTl7y\nD82ehZBSZgkQEiGFCKkNIRFSiJDaEBIhhQipDSER0lzflEFDL5bRm5zUh1klJO8bXS71CfFZ\nTU/ikLw1/MBYh6R7kKTO4gexeG3MJwDpL/f1YX62i2/P6+8os/ixv95COoWn5SHArdtF19IF\nvVy47uj58lIhJEKagZAIaYiQFiAkQhoipAUI6cBC8ikL5ag0rz0sauhomLZZFIKfjOFf/gHx\ncw79RZ8977tb9h/ZaT6JP/X+EQFf+39wIf1VyqUV6SEVvhej/mN+cNw+V/ymVs4y8Rdu2vjn\n9KkQS2YxvbRP8fCDSz4tXxSf++Dt4jN4CImQZiAkQhoipAUIiZCGCGkBQiKkIUJagJAOJyT/\nP9VlTcp8F4TxJ2HOGKZ9ljuljOEnCbhnbwofK90XfY9Cb53pk/UTDl4vXR16SLrkYs1ZdIcI\nv21dVuMt5KXe0YecJMwyzQ9S7Y6XtlkIKW2WAUI6T0ijCGkWQjpPSKMIaRZCOk9IowhpFkI6\nT0hVOuP+7VIO3pnPd5gzTPuG8bUdN8u5gI+VK0X5rLkiFDjMkPyEihKSlqnWnsXb3ScGfVV8\n685ndbxS5uXV1tfND8krv13ibbMQUtosA4R0hJAChDQbIR0hpAAhzUZIRwgpQEizEdIRQurz\npf36TeVA9WkWLff0mxgme8MslzCLzzw5uJDukRJS6wF8yvbRACGlI6Q9zLIjQqoOc6p2EiGt\njJCqw5yqnURIKyOk6jCnaicR0sou1ZB80/7+qpgvSlowcG2Y07eT3ioHHFJ3I5KTmGVHhFQd\n5vTtJEJaDyFVhzl9O4mQ1kNI1WFO304ipPUQUnWY07eTCGk9l2BIPgHondI1dKu0nHQzZ5jT\nt5O81Oma/ktOdBYbhKSrk05slh0RUnWY07eTCGk9hFQd5vTtJEJaDyFVhzl9O4mQ1kNI1WFO\n304ipPVcgiGZb23SnRiU29DRMKd5Jy23Zki64emJzbKjw5qFkEKndhZCWgUhVZzaWQhpFYRU\ncWpnIaRVEFLFqZ2FkFax4zNkcx3Whjmls/xefPual8sPN05slh0d1iyEFDq1sxDSKgip4tTO\nQkirIKSKUzsLIa2CkCpO7SyEtApCqmCWGLPECKmCWWLMEiOkCmaJMUuMkCqYJcYsMYcEYFeE\nBCQgJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQgJCABIQEJCAlIQEhAAkICEhASkICQgASE\nBCQgJCABIQEJCAlIQEhAAkICEhASkICQgASEBCQgJCABIQEJCAlIQEhAAkICEhASkICQgASE\nBCQgJCABIQEJCAlI8P8BeE2obAWwvz8AAAAASUVORK5CYII=", - "text/plain": [ - "plot without title" - ] - }, - "metadata": { - "image/png": { - "height": 420, - "width": 420 - } - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "library(keras)\n", "mnist <- dataset_mnist() # loading the data set\n", "par(mfrow = c(4, 6), oma = c(0, 0, 0, 0),\n", " mar = c(.1, .1, .1, .1), pty=\"s\") # setting some parameters for plotting\n", "for (i in sample(1:60000, 24)) {\n", " image(t(mnist$train$x[i,28:1,]), col = gray.colors(256, rev = T), axes = FALSE)\n", "}" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "The inputs are loaded as 60'000 arrays of 28x28 pixels with values between 0 and 255." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1] 60000 28 28\n", - "[1] 0\n", - "[1] 255\n" - ] - } - ], + "outputs": [], "source": [ "print(dim(mnist$train$x))\n", "print(min(mnist$train$x))\n", "print(max(mnist$train$x))" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "To prepare for fitting with Linear Discriminant Analysis we transform the 28x28-\n", "dimensional arrays into 784-dimensional vectors, i.e. the training data is\n", "reshaped from 60'000x28x28 to 60'000x784. We also search for all pixels\n", "that have zero variance accross the whole dataset and exclude them. These are\n", "usually pixels at the borders of the images that are 0 for every input." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X <- array_reshape(mnist$train$x, c(nrow(mnist$train$x), 28^2))\n", "idx.zero.var <- apply(X, 2, var) == 0 # identify pixels with 0 variance\n", "data.train <- data.frame(X[, !idx.zero.var]) # exclude those pixels\n", "data.train$Y <- mnist$train$y" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Now we load the MASS library and perform LDA. This will take quite of time,\n", "because the data is rather high-dimensional. If you have installed R on a\n", "sufficiently powerful personal computer, I recommend you to run it there." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Warning message in lda.default(x, grouping, ...):\n", - "“variables are collinear”\n" - ] - } - ], + "outputs": [], "source": [ "library(MASS)\n", "lda.fit <- lda(Y ~ ., data.train)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Let us prepare the test data in the similar way as the training data and plot\n", "the mis-classification table and the accuracy." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X.test <- array_reshape(mnist$test$x, c(nrow(mnist$test$x), 28^2))\n", "data.test <- data.frame(X.test[, !idx.zero.var])\n", "data.test$Y <- mnist$test$y\n", "p <- predict(lda.fit, data.test)\n", "table(p$class, data.test$Y)\n", "mean(data.test$Y == p$class) # accuracy" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Next, we repeat the same with multinomial regression.\n", "We set the argument `MaxNWts = 8000` to tell the function that it is allowed to\n", "build a model with at most 8000 weights. This cell will take even longer to\n", "evaluate, than the one before." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "library(nnet)\n", "mn.fit <- multinom(Y ~ ., data.train, MaxNWts = 8000)\n", "p2 <- predict(mn.fit, data.test)\n", "mean(data.test$Y == p2)\n", "table(p2, data.test$Y)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Now we prepare our first feature representation. To have a similar situation as\n", "in the XOR-Problem, where all input data was around 0, we scale the data first\n", "to lie between 0 and 1 and then we subtract for each pixel the mean over the\n", "entire training set, such that each transformed pixel has a value between -0.5\n", "and +0.5. When we do feature engineering there are always many choices.\n", "The exact scaling, for example, does not matter here. What matters is, that the\n", "784 dimensional vectors point in different directions, such that the pair-wise\n", "inner products are sometimes positive and sometimes negative and the relu\n", "non-linearity has an effect. If all inner products would be positive, the relu\n", "non-linearity would not have any effect and the feature representation would be\n", "a linear transformation of the original data. Linear feature transformation can\n", "be sometimes useful (we will discuss this towards the end of this course), but,\n", "for example, the XOR-problem could not be solved with a purely linear\n", "transformation." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X.p <- X/256\n", "m.X.p <- colMeans(X.p)\n", "X.p <- sweep(X.p, 2, m.X.p) # subtract the means from all input points\n", "feature_vectors <- t(X.p[1:200,]) # we just pick the first 200 images as feature vectors\n", "relu <- function(x) ifelse(x > 0, x, 0)\n", "newdata <- data.frame(relu((X.p %*% feature_vectors))) # create the feature representation\n", "newdata$Y <- data.train$Y" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Let us run now multinomial regression on the feature representation.\n", "Note that we have less parameters now, because we went from 784 input dimensions\n", "to a feature representation with 200 features." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "mn.features.fit <- multinom(Y ~ ., newdata, MaxNWts = 3000)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "To test the fit, we also need to transform the test data in the exact same way\n", "as the training data. Note that we subtract the pixel-wise means of the\n", "training data, because it would not be the same transformation, if we subtracted\n", "instead the means of the test data. Next we compute the feature representation\n", "of the test data and the predictions." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X.p.test <- sweep(X.test/256, 2, m.X.p)\n", "p3 <- predict(mn.features.fit, data.frame(relu(X.p.test %*% feature_vectors)))\n", "mean(data.test$Y == p3)\n", "table(p3, data.test$Y)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Let us turn now to learning the features with a neural network.\n", "We use `keras` here, as it is much faster than `ADtools`." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "lines_to_next_cell": 0 }, "outputs": [], "source": [ "nn <- keras_model_sequential()\n", "nn %>%\n", " layer_dense(units = 200, activation = 'relu', input_shape = 784) %>%\n", " layer_dense(units = 10, activation = 'softmax') # with the softmax the outputs of the neurons indicate the probabilities of the digit classes (see slides)\n", "nn %>% compile(\n", " loss = 'categorical_crossentropy', # this is equivalent to choosing the negative log-likelihood loss (see slides)\n", " optimizer = 'adam', # this is a fancy version of stochastic gradient descent (usually you don't need to tune the learning rate with ADAM)\n", " metrics = c('accuracy') # we also track the accuracy during learning.\n", ")\n", "history <- nn %>% fit(\n", " X.p, to_categorical(data.train$Y), # the target is encoded as a categorical variable (this needs to be done explicitly for keras)\n", " epochs = 10, batch_size = 128,\n", " validation_split = 0\n", ")\n", "plot(history)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "In the plot you see how the training loss decreases (this is the negative\n", "log-likelihood loss which we optimize) and at the same time the classification\n", "accuracy on the training set increases.\n", "\n", "Next we compute test loss and accuracy:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "lines_to_next_cell": 0 }, "outputs": [], "source": [ "nn %>% evaluate(X.p.test, to_categorical(data.test$Y))" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Compare this result again to the ones obtained with LDA, multinomial regression\n", "and multinomial regression with hand-crafted features to see how impressively\n", "well the learned features work on this data set.\n", "\n", "In the following cell we extract the learned feature weigths and plot some of\n", "them." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "weights <- nn$layers[[1]]$get_weights()[[1]]\n", "dim(weights)\n", "par(mfrow = c(4, 6), oma = c(0, 0, 0, 0),\n", " mar = c(.1, .1, .1, .1), pty=\"s\")\n", "for (i in sample(1:200, 24)) {\n", " image(t(array_reshape(weights[,i], c(28, 28))[28:1,]), col = gray.colors(256, rev = T), axes = FALSE)\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Review again the slide on nomenclature and loss versus accuracy in the lecture\n", "and once you are ready, please head over to the\n", "[quiz](https://moodle.epfl.ch/mod/quiz/view.php?id=1109835) and answer the\n", "questions on the first page.\n", "\n", "## Convolutional Neural Networks" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "lines_to_next_cell": 0 }, "outputs": [], "source": [ "IRdisplay::display_html('')" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "We do not construct the random Gabor features here. If you are interested you\n", "can read more about [Gabor filters on\n", "Wikipedia](https://en.wikipedia.org/wiki/Gabor_filter). Here we move directly to\n", "convolutional neural networks.\n", "\n", "First we reshape the data again to be images of 28x28x1 pixels. The 1 at the end\n", "is used to indicate that we are dealing with gray-scale images; it would be 3,\n", "if we had colour images. Remember that in the video we used volumes to describe\n", "convolutional neural networks. Here the input is 28x28x1 dimensional volume." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X.p.c <- array_reshape(X.p, c(nrow(X.p), 28, 28, 1))\n", "X.p.c.test <- array_reshape(X.p.test, c(nrow(X.p.test), 28, 28, 1))" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Now we create our first convolutional neural network.\n", "We will use one convolutional layer with filters of size 10x10, padding 0 (keras\n", "option 'valid'), and stride 1. The output volume of this convolutional layer can\n", "be computed with the formula from the slides to get 19x19x200. Before we feed\n", "this volume to the classifier with 10 softmax neurons, we need to flatten this\n", "volume to a vector of length $19\\cdot19\\cdot200 = 72200$. This is done with the\n", "special layer `layer_flatten()` below." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "lines_to_next_cell": 0 }, "outputs": [], "source": [ "conv.nn <- keras_model_sequential()\n", "conv.nn %>%\n", " layer_conv_2d(200, # number of filters\n", " kernel_size = c(10, 10), # size of the filter\n", " strides = c(1, 1), # stride in x and y dimension\n", " activation = 'relu',\n", " input_shape = c(28, 28, 1)) %>%\n", " layer_flatten() %>% # (see text above)\n", " layer_dense(units = 10, activation = 'softmax')" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "With the `summary` function we can confirm the volume computation we did above:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "summary(conv.nn)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Let us now compile and fit this convolutional network." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "conv.nn %>% compile(\n", " loss = 'categorical_crossentropy',\n", " optimizer = 'adam',\n", " metrics = c('accuracy')\n", ")\n", "history <- conv.nn %>% fit(\n", " X.p.c, to_categorical(data.train$Y),\n", " epochs = 5, batch_size = 128,\n", " validation_split = 0\n", ")\n", "plot(history)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Let us compute the test loss and accuracy." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "conv.nn %>% evaluate(X.p.c.test, to_categorical(data.test$Y))" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "And plot some of the learned filters." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "filters <- conv.nn$layers[[1]]$get_weights()[[1]]\n", "dim(filters)\n", "par(mfrow = c(4, 6), oma = c(0, 0, 0, 0),\n", " mar = c(.1, .1, .1, .1), pty=\"s\")\n", "for (i in sample(1:64, 24)) {\n", " image(t(array_reshape(filters[,,,i], c(10, 10))[10:1, 1:10]), col = gray.colors(256),\n", " axes = FALSE, xaxs = 'r', yaxs = 'i')\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Once you are ready, please head over to the\n", "[quiz](https://moodle.epfl.ch/mod/quiz/view.php?id=1109835) and answer the\n", "questions on the second page.\n", "\n", "## Transfer Learning (optional)\n", "\n", "The following video is short and optional, but demonstrates a nice idea with\n", "cool applications: the learned features of a neural network trained on a really\n", "large dataset can be used as fixed features to obtain good performance on a\n", "smaller dataset." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "lines_to_next_cell": 0 }, "outputs": [], "source": [ "IRdisplay::display_html('')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "There is no code for this section.\n", "\n", "If you are interested you can look at the code for applying transfer learning to\n", "the flowers data set\n", "[here](https://cran.r-project.org/web/packages/tfhub/vignettes/hub-with-keras.html)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Regularization and Early Stopping" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "lines_to_next_cell": 0 }, "outputs": [], "source": [ "IRdisplay::display_html('')" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Let us create a simple dataset. We will use the first 25 points as training data\n", "and the rest as validation set." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "set.seed(3)\n", "X <- 4*runif(50) - 2\n", "Y <- X^2 + rnorm(50)\n", "plot(X, Y, col = c(rep('red', 25), rep('blue', 25)))" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Because we will fit different neural networks below, I wrote the following\n", "helper function to compile and fit the neural network and make predictions on a\n", "grid. We include also the argument `callbacks` which will be useful for early\n", "stopping. We use the `mse` loss (Mean Squared Error), which is just the residual\n", "sum of squares (RSS) divided by the number of training samples. This is the\n", "standard negative log-likelihood loss for a regression problem." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "myfit <- function(nn, X, Y, grid = seq(-2, 2, length = 100), callbacks = NULL) {\n", " nn %>% compile(\n", " loss = 'mse',\n", " optimizer = 'adam'\n", " )\n", " history <- nn %>% fit(\n", " X, Y,\n", " validation_split = 0.5,\n", " callbacks = callbacks,\n", " epochs = 1500,\n", " )\n", " list(nn = nn, history = history, grid = grid, pred = predict(nn, grid))\n", "}" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Our first neural network has 2 hidden layers with 50 neurons each." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "nn1 <- keras_model_sequential() %>%\n", " layer_dense(units = 50, activation = 'relu', input_shape = c(1)) %>%\n", " layer_dense(units = 50, activation = 'relu') %>%\n", " layer_dense(units = 1, activation = 'linear')\n", "res1 <- myfit(nn1, X, Y)\n", "plot(X, Y, col = c(rep('red', 25), rep('blue', 25)))\n", "lines(res1$grid, res1$pred)\n", "plot(res1$history)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Now we repeat the same with regularization in all layers." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "nn2 <- keras_model_sequential() %>%\n", " layer_dense(units = 50, activation = 'relu', kernel_regularizer = regularizer_l2(l = .1), input_shape = c(1)) %>%\n", " layer_dense(units = 50, kernel_regularizer = regularizer_l2(l = .1), activation = 'relu') %>%\n", " layer_dense(units = 1, kernel_regularizer = regularizer_l2(l = .02), activation = 'linear')\n", "res2 <- myfit(nn2, X, Y)\n", "plot(X, Y, col = c(rep('red', 25), rep('blue', 25)))\n", "lines(res2$grid, res2$pred)\n", "plot(res2$history)" ] }, { "cell_type": "markdown", "metadata": { "lines_to_next_cell": 0 }, "source": [ "Finally, we do not use regularization, but use the callback\n", "`callback_early_stopping` that stops gradient descent when the validation loss\n", "`val_loss` starts to increase." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "nn3 <- keras_model_sequential() %>%\n", " layer_dense(units = 50, activation = 'relu', input_shape = c(1)) %>%\n", " layer_dense(units = 50, activation = 'relu') %>%\n", " layer_dense(units = 1, activation = 'linear')\n", "callback <- callback_early_stopping(monitor = \"val_loss\")\n", "res3 <- myfit(nn3, X, Y, callbacks = callback)\n", "plot(X, Y, col = c(rep('red', 25), rep('blue', 25)))\n", "lines(res3$grid, res3$pred)\n", "res3$history$params$epochs <- length(res3$history$metrics$loss) # this is a hack to fix a problem of keras\n", "plot(res3$history)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Once you are ready, please head over to the\n", "[quiz](https://moodle.epfl.ch/mod/quiz/view.php?id=1109835) and answer the\n", "questions on the third page.\n", "\n", "## Exercises\n", "\n", "### Conceptual\n", "\n", "**Q1**\n", "Here below there is an image we would like to process with a convolutional network with one convolution layer with two 3 $\\times$ 3 filters (depicted below the image) and relu non-linearity.\n", "\n", "![](img/Conv.PNG)\n", "\n", "(a) Determine the width, height and depth of the volume after the convolutional layer.\n", "\n", "(b) Compute the output of the convolutional layer assuming the two biases to be zero.\n", "\n", "**Q2**\n", "Given a volume of width $n$, height $n$ and depth $c$:\n", "\n", "(a) Convince yourself that the convolution with $k$ filters of size $f\\times f\\times c$ with stride $s$ and padding $p$ leads to a new volume of size\n", "\\begin{equation*}\n", " \\left( \\frac{n + 2p - f}{s} + 1\\right)\\times \\left( \\frac{n + 2p - f}{s} + 1\\right)\\times k\\, .\n", "\\end{equation*}\n", "\n", "(b) Keras knows the padding options `valid` and `same`. The first one, `valid`,\n", "means no padding at all. The second one, `same`, means that the output volume\n", "should have the same x and y dimension as the input volume. In this exercise we\n", "compute the padding needed for option `same`.\n", "What padding $p$ do you have to choose such that the input and output volumes have the same width and depth for stride $s=1$. Check you result for the special case of $n=4$ and $f=3$.\n", "\n", "## Applied\n", "\n", "**Q3**\n", "In this exercise you will try to identify metastatic tissue in histopathologic scans of lymph node sections. The input data consists of greyscale images of 32 $\\times$ 32 pixels that were derived from the Histopathalogic Cancer Detection competition (https://www.kaggle.com/c/histopathologic-cancer-detection).\n", "The original dataset contains colour image patches of 96 $\\times$ 96 pixels taken from larger digital pathology scans ($220\\,025$ in the training set, and $57\\,458$ in the test set).\n", "A positive label indicates that the central 32 $\\times$ 32 pixels region contains at least one pixel of tumor tissue.\n", "For this exercise we look at $40\\,000$ images from the training set, where the central patch of the original image was extracted and transformed to greyscale to reduce the computational burden.\n", "\n", "(a) Load the data with the command\n", "`load(url(\"https://lcnwww.epfl.ch/bio322/PCaml.rda\"))`; two new variables will be added to your workspace: `PCaml_x` and `PCaml_y`.\n", "\n", "(b) In general it is recommendable to look at the raw data first. Here, you may want to plot 24 randomly chosen example images that were labelled to contain at least one pixel of tumor tissue and 24 randomly chosen images that were labelled as tumor-free samples. Can you see by eye any difference between the two classes?\n", "\n", "(c) Split the data into a training set of size $n=30\\,000$ and a test set of size $n=10\\,000$.\n", "\n", "(d) Perform logistic regression to have a benchmark.\n", "Plot the ROC, compute the AUC and compute the accuracy on the training and test set.\n", "\n", "(e) Fit a convolutional neural network to the data.\n", "Compare ROC, AUC and accuracy of the neural network fit to the logistic regression fit.\n", "\n", "(f) (Optional) Play with the network architecture. You may change the number of filters per convolutional layer, the number of layers, add regularization etc. and see how the test error depends on your architectural choices." ] } ], "metadata": { "kernelspec": { "display_name": "R", "language": "R", "name": "ir" }, "language_info": { "codemirror_mode": "r", "file_extension": ".r", "mimetype": "text/x-r-source", "name": "R", "pygments_lexer": "r", "version": "4.0.2" } }, "nbformat": 4, "nbformat_minor": 4 }