Standard GAN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## As usual in python we import stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as se \n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers.core import Reshape, Dense, Dropout, Flatten\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.layers.convolutional import Convolution2D, UpSampling2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.datasets import mnist\n",
    "from keras.optimizers import Adam\n",
    "from keras import backend as K\n",
    "from keras import initializers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "K.set_image_dim_ordering('th')\n",
    "\n",
    "# Deterministic output. For reproduceability, if you hate it just remove it.\n",
    "np.random.seed(1000)\n",
    "\n",
    "# The results are a little better when the dimensionality of the random vector is only 10.\n",
    "# The dimensionality has been left at 100 for consistency with other GAN implementations.\n",
    "randomDim = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Beat the dead horse *aka* import mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load MNIST data from keras\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train = (X_train.astype(np.float32) - 127.5)/127.5\n",
    "X_train = X_train.reshape(60000, 784)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The optimizer we gonna use here is called adam smith..... JK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "adam = Adam(lr=0.0002, beta_1=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's build a casual Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator = Sequential()\n",
    "generator.add(Dense(256, input_dim=randomDim, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(512))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(1024))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(784, activation='tanh'))\n",
    "generator.compile(loss='binary_crossentropy', optimizer=adam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/scipy/stats/stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.\n",
      "  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1099988d0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xl81NW5x/HPQ9gERbHEhUWjBReQRY0IUqp1BbW4oVJQscWmoF712tbWltpeu1jb61LrikvFHcSNKigqWsXKEhQQRAUtyiZQUVCqsuS5f5zJNYaETJKZObN836/XvDKTOc58JcmTk/M7i7k7IiKSX5rEDiAiIqmn4i4ikodU3EVE8pCKu4hIHlJxFxHJQyruIiJ5SMVdRCQPqbiLiOQhFXcRkTzUNNYbt2vXzktKSmK9vYhITpo9e/a/3b24rnbRintJSQnl5eWx3l5EJCeZ2fvJtNOwjIhIHlJxFxHJQyruIiJ5SMVdRCQPqbiLiOQhFXcRkTyk4i4ikodU3EVE8pCKu4hIHoq2QlUkm40Zk1y7srL05hBpKPXcRUTykIq7iEgeUnEXEclDKu4iInko6eJuZkVm9rqZPVnDcy3MbJyZLTazGWZWksqQIiJSP/XpuV8MLKzluRHAx+7eGbgOuLqxwUREpOGSKu5m1hE4AbijliYnAWMT9ycAR5mZNT6eiIg0RLI99+uBy4CKWp7vACwFcPfNwDrgG41OJyIiDVLnIiYzOxFY7e6zzeyI2prV8Dmv4bXKgDKAPfbYox4xRWqnBUciW0um594PGGRmS4CHgCPN7L5qbZYBnQDMrCmwI7C2+gu5+xh3L3X30uLiOs93FRGRBqqzuLv75e7e0d1LgCHAVHc/q1qzicDwxP3BiTZb9dxFRCQzGry3jJldCZS7+0TgTuBeM1tM6LEPSVE+ERFpgHoVd3d/EXgxcf+KKp//Ajg9lcFERKThtEJVRCQPqbiLiOQhFXcRkTyk4i4ikodU3EVE8pCO2ZOC5Q5vvQVPPQVz58L778PatfDNb8LGjVBaCp06xU4p0jAq7lJwVq2Cu+4Kt8WLw+f22ANKSqBzZ1i0KBT9Z56Bww6Dk06CHXeMGlmk3lTcpWAsWQK//S3cey9s2gRHHAGXXgonnBCKe1XXXw+TJsHUqTBnDvzkJ9C+fYzUIg2jMXfJe198AQ89BPvsA/ffDyNHwsKF8MILMGrU1oUdoFUrGDwYfvUraNo0FPs1azKfXaShVNwlr73zTuitv/gijBgB774LN9wA++2X3H+/++5wySWweTNcdx2sW5fWuCIpo+IueWvqVLj2WjALwyq33AIdOtT/ddq3h4svhvXr4YEHUp9TJB1U3CXvuIfx8nHjoGdPGD06XChtjD33hBNPDOPvr7+empwi6aTiLnnnySfhiSegT59wQEfLlql53WOOgY4dw/j955+n5jVF0kXFXfLKnDmhuPftC8OHQ1FR6l67qAjOPjuMuz/xROpeVyQdVNwlb6xZA3ffHWa/DBsGTdLw3V1SAv37w0svhQVPItlKxV3ywubN4SxVM/jRj6BZs/S914ABYVx/ypT0vYdIY9VZ3M2spZnNNLO5ZrbAzP6nhjbnmtkaM5uTuJ2XnrgiNXv+efjggzAU065det/rG98Iwz7TpsGHH6b3vUQaKpme+5fAke7eE+gFDDCzPjW0G+fuvRK3O1KaUmQb1q0Ls2O6d4devTLzngMGhL8WrrkmM+8nUl/JHJDt7v5Z4mGzxE2HX0vWePzxsJ3A6Rk86HGXXeCQQ+Dmm+GjjzL3viLJSmrM3cyKzGwOsBp41t1n1NDsNDObZ2YTzEx76UlGzJoF//wnHHUU7LprZt/7uOPgP/8JF3FFsk1Sxd3dt7h7L6Aj0NvMDqjW5O9Aibv3AJ4Dxtb0OmZWZmblZla+Rht1SApccQXssAMcf3zm37tjR+jXD269FSoqMv/+IttSr9ky7v4J8CIwoNrnP3L3LxMPbwcOruW/H+Pupe5eWlxc3IC4Il+ZMweefjr02rfbLk6GUaPCtsHPPRfn/UVqU+eWv2ZWDGxy90/MbDvgaODqam12d/eViYeDgIUpTypSzR//GHrthx+eXPsxY1KfYfDgsLHYLbfAscem/vVFGiqZnvvuwAtmNg+YRRhzf9LMrjSzQYk2FyWmSc4FLgLOTU9ckWDxYnj44dBzbtUqXo4WLcJukxMnwrJl8XKIVJfMbJl57n6gu/dw9wPc/crE569w94mJ+5e7ezd37+nu33H3t9IdXArb//5vWKh0ySWxk4RFU+5w++2xk4h8RStUJeesXRtmqAwfHvZbj22vvcK4/z336MKqZA8Vd8k5Y8fCl1/CBRfETvKV4cPDMX7TpsVOIhKouEtOcQ9TDw87DHr0iJ3mK6ecAttvH3rvItlAxV1yyosvhqPzRo6MneTrWrcOM2fGjw8Lm0RiU3GXnHLrrdC2bSik2eacc+DTT7XXu2QHFXfJGatWwaOPwrnnxlu0tC2HHx6O4xtb4/pskcyqcxGTSCzVFx09+2zYiXGnndKzIKkhqufo1g0mT4Y//SnkrFRWltlcIuq5S86YMSOchLTbbrGT1K5Pn3DRd0ZNW+uJZJCKu+SE5cth6VI49NDYSbZt111h771h+vRQ5EViUXGXnDBzZjgTtbQ0dpK69ekDK1aEX0Yisai4S9arqAjFvWtXaNMmdpq6lZZC06bw6quxk0ghU3GXrLd4cdhyINuHZCq1bh0WWM2aBVu2xE4jhUrFXbLejBlh98WePWMnSV7fvmHO+/z5sZNIoVJxl6y2aRPMng0HHhgKfK7o1i3sNT99euwkUqhU3CWrvfEGfP557gzJVCoqgt69Yd482LAhdhopRCruktVmzAgXUffdN3aS+uvTJyy6Ki+PnUQKUZ3F3cxamtlMM5ubOG3pf2po08LMxpnZYjObYWYl6QgrhWXDhjBmfcghoSecazp1gg4dNGtG4kim5/4lcKS79wR6AQPMrE+1NiOAj929M3Ad1c5YFWmI2bNDz7dP9e+2HGEWsv/rX2EnS5FMSuaYPXf3zxIPmyVu1dfenQRUbpc0ATjKzCxlKaUgzZgRTlrq1Cl2koY79NBQ5LXPu2RaUmPuZlZkZnOA1YQDsqvvnNEBWArg7puBdcA3anidMjMrN7PyNWvWNC655LUlS8L89t69Q3HMVTvuGGbOjB2rOe+SWUkVd3ff4u69gI5AbzM7oFqTmn78ttpZw93HuHupu5cWFxfXP60UjAceCB97946bIxUOOwyWLYMpU2InkUJSr9ky7v4J8CIwoNpTy4BOAGbWFNgRWJuCfFKA3OG++6BzZ2jXLnaaxuvZM/x/3Hln7CRSSJKZLVNsZjsl7m8HHA28Va3ZRGB44v5gYKq79sSThpkzBxYuzL257bVp2jSc0jRxIqxeHTuNFIpkeu67Ay+Y2TxgFmHM/Ukzu9LMBiXa3Al8w8wWA5cCP09PXCkE990HzZrBwQfHTpI6I0aE1bb33hs7iRQKi9XBLi0t9XKt7pBqtmyBjh3DFMKBA2OnSZ2ysrDfzLp1sGBBbl8klrjMbLa717n5tVaoSlaZOhU+/BDOOit2ktQ777ww3DRtWuwkUghU3CWr3HdfmD54wgmxk6Te974XzlW96abYSaQQqLhL1tiwAR59FE4/HVq2jJ0m9Vq1gu9/Hx55BFaujJ1G8p2Ku2SNiRPhs8/yc0im0qhRYUuF22+PnUTynYq7ZI377gtbDfTvHztJ+nTpAgMGwG23hdkzIumi4i5ZYfVqeOYZGDYsHISdzy64IByg/fjjsZNIPsvzHyPJFePGhWmQ+TwkU2ngQNh7b7j22rAaVyQdVNwlK9x3H/TqFTbZyndFRXDppeEIvldeiZ1G8lXT2AGk8IwZ8/XHq1bBzJlw2mlbP5evvv99+PWv4c9/hm99K3YayUfquUt0M2eGFZuHHBI7Sea0agUXXhhmCL1VfacmkRRQcZeo3ENx32cfaNs2dprMuuCCMJ//mmtiJ5F8pOIuUb3/fpgpkw/7ttdXcTH84AfhII+lS2OnkXyj4i5RzZwZtsQ96KDYSeK47LLw8aqr4uaQ/KPiLtFUVMCsWXDAAWEMuhDtuWfovd9xh3rvkloq7hLNW2/B+vX5cyhHQ11+efio3rukUjInMXUysxfMbKGZLTCzi2toc4SZrTOzOYnbFemJK/lk5sxwQbF799hJ4qrae//gg9hpJF8k03PfDPzY3fcH+gAXmFnXGtq97O69ErcrU5pS8s7GjfD662GsvVmz2Gni+8UvwnTQX/86dhLJF3UuYnL3lcDKxP1PzWwh0AF4M83ZJI/NmwdffFE4s2SSWZx1+OFh5sx//zf06JH+TJLf6jXmbmYlwIHAjBqe7mtmc81sspkVwCJyaYyZM8OhHPvuGztJ9hg4MBzmUTmDRqQxki7uZrY98Ahwibuvr/b0a8Ce7t4T+CtQ4353ZlZmZuVmVr5mzZqGZpYct2EDzJ8fVqTm+w6Q9dG6NYweHXbHfPbZ2Gkk1yX1o2VmzQiF/X53f7T68+6+3t0/S9yfBDQzs3Y1tBvj7qXuXlpcXNzI6JKrZs8OO0AW+iyZmlxwAZSUwE9/GqaKijRUMrNlDLgTWOju19bSZrdEO8ysd+J1P0plUMkfM2fCrruGgznk61q0gD/8AebODTtlijRUMj33fsDZwJFVpjoeb2YjzWxkos1gYL6ZzQVuAIa4a6dq2doHH8CiRaHXHroDUt2ZZ0JpaRii+fzz2GkkVyUzW2YasM0fQ3e/EbgxVaEkfz34YPhYKLNk6qtyVs23vx0O8xg2LBzLV11ZWWZzSe7R5SzJqPvvh732CptmSe323Tcs7po8ORwaLlJfKu6SMW+8EW66kJqcU08Ni73+/vfYSSQXqbhLxjzwQDhi7uCDYyfJDe3bh1OaXnoJPvwwdhrJNSrukhEVFaG4H3MMtGkTO03u+O53oXlzeHSrCcgi26biLhkxfXqYKTN0aOwkuaVNm3BBde5cePvt2Gkkl6i4S0Y8/HDogQ4aFDtJ7jnqKNh5Z5gwQQubJHkq7pJ2FRWhuB93XNhPRuqneXM4+eTwl8+MmnZ1EqmBiruk3fTpsHw5nHFG7CS565BDwr7vjz8eZtCI1EXFXdKuckjmu9+NnSR3NWkCp58On3yiTcUkOSruklYVFWGsWEMyjdelC/TqFXaN/Pe/Y6eRbFfn9gMiyarpQIp334Vly8IUyGQOrJBtO/nkMHPm6qvhz3+OnUaymXruklazZ0PTptCzZ+wk+WH33cMK3xtvhJUrY6eRbKbiLmlTUQGvvQZdu8J228VOkz9OPBE2b4bf/z52EslmKu6SNv/6F3z8sbYbSLXiYvjBD8Iw1/vvx04j2UrFXdJGQzLpM3p0+Khxd6mNirukReWQzP77a0gmHTp1guHD4Y47tKmY1CyZY/Y6mdkLZrbQzBaY2cU1tDEzu8HMFpvZPDM7KD1xJVdUDsmUlsZOkr8uuww2bYLrroudRLJRMj33zcCP3X1/oA9wgZl1rdZmINAlcSsDbklpSsk5GpJJvy5dwqrfm28Ov0hFqqqzuLv7Snd/LXH/U2Ah0KFas5OAezyYDuxkZrunPK3kBA3JZM7ll4eTmm7UIZdSTb0WMZlZCXAgUH37og7A0iqPlyU+p5m4BWjJktCTPOmk2EnyV9UFYd27hwurO+8MzZp9vZ3OWi1cSV9QNbPtgUeAS9x9ffWna/hPvIbXKDOzcjMrX7NmTf2SSs4oL9eQTCYddRR8+mn4dxeplFRxN7NmhMJ+v7vXdCbMMqBTlccdgRXVG7n7GHcvdffSYp2QnJeqDsm0ahU7TWHYb79wJN/zz4Nv1aWSQpXMbBkD7gQWuvu1tTSbCJyTmDXTB1jn7hqSKUCVQzJauJQ5ZnDkkbB0KSxaFDuNZItkeu79gLOBI81sTuJ2vJmNNLORiTaTgPeAxcDtwPnpiSvZbvbscAi2hmQy69BDoXVrmDo1dhLJFnVeUHX3adQ8pl61jQMXpCqU5Cb3UNy7dtWQTKY1bw79+4ftgNeuDRdXpbBphaqkjPaSiat///DxlVfi5pDsoOIuKaNZMnG1axf+apo2DbZsiZ1GYlNxl5SoqNCQTDb49rfDUXzz58dOIrGpuEtKvPpqKCraSyau7t3DcYYvvxw7icSm4i4pMW6chmSyQVERHHZY6LmvXRs7jcSk4i6NtmULPPwwHHAAtGwZO41861vh4z//GTeHxKXiLo02bVrYU1xDMtmhXTvYZx+YPl0rVguZirs02rhxYffH7t1jJ5FKffvCmjWaFlnIVNylUTZvhkceCYc2a0gmexx4ILRoAWPHxk4isai4S6P84x+wejWceWbsJFJVy5ahwI8fD59/HjuNxKDiLo0yblzY02TgwNhJpLq+fWH9enjiidhJJAYVd2mwTZvg0Udh0CAtXMpG++wDe+yhoZlCpeIuDTZ1Knz0UTjHU7JPkyZw1lkwZQqsWhU7jWSairs02Lhx0KYNDBgQO4nUZtiwsDXE+PGxk0imqbhLg2zcCI89Fs5J1SyZ7NW1a1g1fP/9sZNIpqm4S4NMmhT2khkyJHYSqcuwYTBjBixeHDuJZFIyx+zdZWarzazGfebM7AgzW1fllKYrUh9Tss0998Auu8Cxx8ZOInUZMiQcxffgg7GTSCYl03O/G6hrVPVld++VuF3Z+FiSzdauhSefhKFDw2Zhkt06dQpbAd9/v7YjKCR1Fnd3fwnQ/nLy/8aNC9MgzzkndhJJ1tCh8Pbb8PrrsZNIpqRqzL2vmc01s8lm1i1FrylZ6t57ww6QvXrFTiLJGjwYmjXThdVCkori/hqwp7v3BP4KPF5bQzMrM7NyMytfs2ZNCt5aMm3RonAwx9lnh3FcyQ077xxWET/0kI7gKxSNLu7uvt7dP0vcnwQ0M7N2tbQd4+6l7l5aXFzc2LeWCO65JxT1YcNiJ5H6GjYMVqwI+wFJ/mt0cTez3cxCH87Meide86PGvq5kn82b4W9/C4uWOnSInUbq68QTYfvt4YEHYieRTEhmKuSDwKvAvma2zMxGmNlIMxuZaDIYmG9mc4EbgCHuuiafj555BpYvh/POi51EGqJVKzj1VJgwAb74InYaSbc6J7K5+/fqeP5G4MaUJZKsdccdYW77iSfGTiINNWxYGFqbPBlOOSV2GkknzVKWpHz4Ifz97/DjH0Pz5rHTSLLGjPn64y1bYIcd4He/Cyc1VVVWlrlckn7afkCSMnZsKAwjRsROIo1RVASHHALz5ukQj3yn4i51qqgIQzL9+4c9wiW39e4dLo6/9lrsJJJOKu5Sp2efDZtOjRxZd1vJfiUlUFwMs2bFTiLppOIudbrppnAh9bTTYieRVDALvfe33oJ162KnkXRRcZdtWrIkbBL2wx9Cixax00iq9O4dNhFT7z1/qbjLNt16a+jp/ehHsZNIKu22WzhfdebM2EkkXTQVUmr1xRfhQmqPHmFetOSX3r3DgqZVq2DXXWOnkVRTz11q9eCD4QDsI46InUTSobQ0/FWm3nt+UnGXGrnDdddB9+6w336x00g6tG0bprbOnKlDPPKRirvUaOpUeOMNuOQSbe2bz3r3htWr4f33YyeRVFNxlxpdd12Y/jh0aOwkkk4HHRSOStTQTP5RcZetvP02PPUUjBoFLVvGTiPp1KpVOFVr1iwd4pFvVNxlK9dfH+a0jxoVO4lkwqGHwvr1YShO8oemQhao6rsFVlq/Hu68M/zAP/FEZjNJHN27hx783XfDMcfETiOpop67fM3UqWFTKf2QF45mzcIv8wkTwtRXyQ/JnMR0l5mtNrP5tTxvZnaDmS02s3lmdlDqY0omfPFFOF+zZ8+wglEKR//+sHFj2NpZ8kMyPfe7gQHbeH4g0CVxKwNuaXwsiWHaNPjPf8IZqVJYOnSAvn3DcJ3mvOeHOou7u78ErN1Gk5OAezyYDuxkZrunKqBkxubN8NxzYVHLXnvFTiMxlJWFmVIvvxw7iaRCKsbcOwBLqzxelvjcVsyszMzKzax8TfUzviSqWbPg44/huONiJ5FYzjgDdtwRbrstdhJJhVQU95rWL9b4h527j3H3UncvLS4uTsFbSypUVMCUKeFP827dYqeRWFq1gnPOgYcfhpUrY6eRxkpFcV8GdKryuCOwIgWvKxkyfz6sWAHHHqutBgrdRReFIbqbb46dRBorFcV9InBOYtZMH2Cdu+v3fg555hnYeedwcLIUts6dYdAguOWWcHFdclcyUyEfBF4F9jWzZWY2wsxGmlnliZqTgPeAxcDtwPlpSysp9+674XzUY46BoqLYaSQbXHppmO9+772xk0hj1LlC1d2/V8fzDlyQskSSUZMnQ+vW0K9f7CSSLfr3h4MPDpvH/fCH0ERLHXOSvmwF7IMPwra+Rx+t81HlK2ah9/722zBxYuw00lAq7gVs8mTYbjv4zndiJ5Fsc8YZYfz9N78Js6kk96i4F6gVK+C11+DII0OBF6mqaVO44gqYOxceeyx2GmkIFfcCNXlyGIo58sjYSSRbfe97sO++8Otfq/eei1TcC9CiRWFF6uGHw/bbx04j2app01DYFyyA8eNjp5H6UnEvQH/8Y/jB1ba+Upczzgirln/5y7BrqOQOFfcCs2QJ3HNPmO7Wpk3sNJLtiorClMj33gsfJXeouBeYP/0pzFs+9tjYSSRXHHMMnHwy/P73sHx57DSSLBX3ArJ8eThC7/vfh7ZtY6eRXHLNNWHPmZ/9LHYSSZbOUC0gv/99mPXws5/Bs8/GTiPZprZzdSsddRTcfz8MH67rNblAPfcC8a9/we23w3nn6TAOaZjjjw/HL44YEQ5Sl+ym4l4grrwyzJAZPTp2EslVzZqFXvvy5fCTn8ROI3VRcS8Ab70VZsicf344kEOkofbeG3784/BX4NNPx04j26Ix9wIwenTYYkAXwyQV9twT2rcPc+BHj4addqq5XVlZZnPJ16nnnudefhkeeQR++lPYZZfYaSQfNGsWtgL+8ssw+0pbE2SnpIq7mQ0ws7fNbLGZ/byG5881szVmNidxOy/1UaW+KirC1q0dOmiMVFKrfXsYOhTeeQeefDJ2GqlJncMyZlYE3AQcQzgvdZaZTXT3N6s1HefuF6YhozTQAw9AeXkYb2/dOnYayTd9+4Y93596CvbYA3r1ip1Iqkqm594bWOzu77n7RuAh4KT0xpLG+uwzuPzycKLOsGGx00i+GjoUSkrgrru0ejXbJFPcOwBLqzxelvhcdaeZ2Twzm2BmnVKSThrsV7+CZcvgL3/RMWmSPs2bw8iRYfvom28OnQrJDsn82FsNn/Nqj/8OlLh7D+A5YGyNL2RWZmblZla+Zs2a+iWVpM2aBTfcEH7odDaqpFvbtjBqFKxbBzfeCBs3xk4kkNxUyGVA1Z54R2BF1Qbu/lGVh7cDV9f0Qu4+BhgDUFpaWv0XhKTALbfAH/4Qdnzcb7+6l5SLpMLee4eVq7fdFr7nRo2KnUiS6bnPArqY2V5m1hwYAnzt2Fwz273Kw0HAwtRFlPqYNCkMxwwZouPzJLMOPDCc3vTGG+Fivqv7FlWdxd3dNwMXAs8QivZ4d19gZlea2aBEs4vMbIGZzQUuAs5NV2Cp3YsvhpkLvXuHHzSRTDv8cBg4EKZNC1teSDzmkX69lpaWenl5eZT3zkerV4epaBUV8ItfQMuWsRNJoXIP02//+c8wRPPDH8ZOlF/MbLa7l9bVTtsP5IGNG8OUtI8/DitRVdglJjM46yzYccdwUX+HHcIwoWSWinuOq6iAc8+F55+Hu+8OS8JFYisqggkTwjbBZ50VHp9+euxUhUUzoHOYe9he4MEHwwyZ4cNjJxL5SqtWYWuCPn3CX5YPPxw7UWFRcc9RW7bAf/1XWKR08cXw8612/BGJb/vtwwyu3r3D0Mxdd8VOVDhU3HPQhg1wyilw001hQ7Brrw3jnCLZqE0bmDIFjj46zIW/5hpNk8wEFfccM3t26AU99VQo7n/+s7YXkOzXujVMnAinnRY6JOefD5s2xU6V31QWcsRnn4X9Yg49NCzzfvrp8AMikitatIDx4+Gyy+DWW8PF1n//O3aq/KXinuU+/zzsE/PNb8LvfhcuTM2fr9PnJTc1aQJXXw1/+xu89FJYm/GPf8ROlZ9U3LPUO++EOesdO4YLpt26wauvhsUhtR1rJpIrzj0Xpk8PM2qOPDJMCPjPf2Knyi+a554l3OG11+Cxx+Dxx2HBgjA3+JRT4MILw7JukVySzKZ1F14I8+aF3vz48WFXyYEDNUEgFdRzj2jzZpg6FS66KBw6XFoKV10FxcVw/fXw/vthbrAKu+Srli3hjjvghRfC3vAnnADf+Q688krsZLlPPfc0qa3XsnEjvPkmvP562D1vw4bwDX7ccWGjpRNPhHbtkn89kXxwxBEwd274Pv/d7+Bb3wqdmosvhkGDwl+xUj8q7hnw2WehkL/+eijsmzaFscYePcLMgWOP1RmnIi1ahIV5P/hBmE3z17/CqaeGA97PPDMsgiot1ZBNsrQrZJpcdVXoicyZA4sWhT1g2rYNswN69YIuXdQbEdmWLVvCz9D06WGG2JYtYdbYkCFw8slw0EGFucYj2V0hVdxTZPPm8E04aVK4zZ0bPt++/VcFfY891OsQaYgNG0JHadYseOutMAGhdWvo2hX23z98bNs2tC0ri5s13bTlb5q5h2+yl14KF0WnTIFPPoGmTcN44amnhoK+666xk4rkvtatw3nA/frB+vWwcGEY4nzzzVDwIVyr+uY3w1/J/fqFgl/Ifx0n1XM3swHAX4Ai4A53/2O151sA9wAHAx8BZ7r7km29Zq713NetC73xG28MwyyLF3910vuOO8IBB4Tb/vvreDuRTHGHFStCkV+8GN57LxR/CPvI9+wZOlmVH7t1y/2fz5T13M2sCLgJOIZwWPYsM5vo7m9WaTYC+NjdO5vZEMIB2Wc2LHo87rB2Lbz7bvhGWbQozMGdMyd801Rq1w66dw/j5l26hKmLGm4RyTyzcMG1Q4ewats9fHzllTBMOnduOOegsiNmFhYGVv7sdukCJSXhv2/fHnbbLUzJzAd19tzNrC/wG3fSCHpjAAAGIUlEQVQ/LvH4cgB3v6pKm2cSbV41s6bAh0Cxb+PFU9Vzdw8XWioqwseqty+/DF/UDRvCx6q3Tz6BDz/8+u3998Pnq+rcOZxH2qtX+Dh/fuipi0huqKgIe9gsWxZ6+atXf3XbsGHr9rvsEgp9+/ZhWHWnncLPfOXHyvtt2oQZPpW35s2/fr+oKD2dvlSOuXcAllZ5vAw4tLY27r7ZzNYB3wBSvi3Qo4+G/VUqC3hjrgc3bx5+U++2W1hE1K9fKOadO4exu7322vpPuKVLa34tEclOTZqEgr3LLmGGTVWnnQYffBCKfvXb8uWh579u3Vc9/4YwCxkqPzZpEnbG/O1vG/f/Vef7JtFzPx04zt3PSzw+G+jt7v9Vpc2CRJtlicfvJtp8VO21yoDKa9n7Am83MHc70vCLo5GyMRNkZy5lSl425lKm5KUj157uXlxXo2R67suATlUedwRW1NJmWWJYZkdgbfUXcvcxQKPXWppZeTJ/lmRSNmaC7MylTMnLxlzKlLyYuZJZAjAL6GJme5lZc2AIMLFam4lA5Qmeg4Gp2xpvFxGR9Kqz554YQ78QeIYwFfIud19gZlcC5e4+EbgTuNfMFhN67EPSGVpERLYtqUVM7j4JmFTtc1dUuf8FcHpqo21TNm6jlY2ZIDtzKVPysjGXMiUvWq5o2w+IiEj6FOC2OyIi+S8niruZ7Wxmz5rZosTHttto28bMlpvZjbEzmdmeZjbbzOaY2QIzG5nOTPXI1cvMXk1kmmdmaV1NnOzXz8yeNrNPzOzJNGYZYGZvm9liM/t5Dc+3MLNxiednmFlJurLUI9O3zew1M9tsZoPTnaceuS41szcT30PPm9meWZBppJm9kfiZm2ZmXWNnqtJusJm5mWVm9oy7Z/0N+BPw88T9nwNXb6PtX4AHgBtjZwKaAy0S97cHlgDtsyDXPkCXxP32wEpgp9hfP+Ao4LvAk2nKUQS8C+yd+NrMBbpWa3M+cGvi/hBgXJq/XslkKgF6EPZvGpzOPPXM9R2gVeL+qCz5t2pT5f4g4OnYmRLtdgBeAqYDpZn4GuZEzx04CRibuD8WOLmmRmZ2MLArMCUbMrn7Rnf/MvGwBZn5SymZXO+4+6LE/RXAaqDORRHpzJTI8jzwaRpz9AYWu/t77r4ReCiRraqqWScAR5mldeegOjO5+xJ3nwdUpDFHQ3K94O6Vx1pPJ6yBiZ1pfZWHrYF0X1RM5nsK4LeETs4Xac7z/3KluO/q7isBEh93qd7AzJoA1wA/zZZMiVydzGweYXuGqxPFNHquKvl6E3oc72ZLpjSqaSuNDrW1cffNQOVWGjEzxVDfXCOAyWlNlGQmM7sgsUr+T8BFsTOZ2YFAJ3dP23BjTbJmP3czew7YrYanfpnkS5wPTHL3panqaKUgE+6+FOhhZu2Bx81sgruvip0r8Tq7A/cCw929Ub3CVGVKs5q+Mar37JJpk0qZfr9kJZ3LzM4CSoF0H+WeVCZ3vwm4ycyGAqP5aoFlxjMlOp3XAeemMUONsqa4u/vRtT1nZqvMbHd3X5koSKtraNYX6G9m5xPGt5ub2WfuXusFjgxkqvpaKxJ78PQn/LnfYKnIZWZtgKeA0e4+vTF5UpUpA1K2lUaGM8WQVC4zO5rwC/zwKkOQUTNV8RBwS1oT1Z1pB+AA4MVEp3M3YKKZDXL3tB5okSvDMlW3NxgOPFG9gbsPc/c93L0E+AlwT2MKeyoymVlHM9sucb8t0I+Gb5aWylzNgccI/0YPpzlPUpkyJBu30kgmUwx15koMN9wGDHL3TPzCTiZTlyoPTwAWxczk7uvcvZ27lyRq03TCv1f6TyrKxFXbxt4IY57PE75QzwM7Jz5fSjgZqnr7c0n/bJk6MxEOOJlHuII+DyjLhn8r4CxgEzCnyq1X7K8f8DKwBvic0CM6Lg1ZjgfeIVxj+GXic1cSfuAAWgIPA4uBmcDeGfia1ZXpkMS/xwbCSWcL0p0pyVzPAauqfA9NzIJMfwEWJPK8AHSLnala2xfJ0GwZrVAVEclDuTIsIyIi9aDiLiKSh1TcRUTykIq7iEgeUnEXEclDKu4iInlIxV1EJA+puIuI5KH/A18tu8Y/4SgwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mu, sigma = 0, 0.1 # mean and standard deviation\n",
    "s = np.random.normal(mu, sigma, 1000)\n",
    "se.distplot(s, color=\"b\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Geneartor is generating images but how?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"model.png\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Activation Functions](https://cdn-images-1.medium.com/max/1200/1*ZafDv3VUm60Eh10OeJu1vw.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 256)               25856     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               131584    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1024)              525312    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)    (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 784)               803600    \n",
      "=================================================================\n",
      "Total params: 1,486,352\n",
      "Trainable params: 1,486,352\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "generator.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now we build the second player (the discriminator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_5 (Dense)              (None, 1024)              803840    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)    (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 512)               524800    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_5 (LeakyReLU)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_6 (LeakyReLU)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 1)                 257       \n",
      "=================================================================\n",
      "Total params: 1,460,225\n",
      "Trainable params: 1,460,225\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "discriminator = Sequential()\n",
    "discriminator.add(Dense(1024, input_dim=784, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(512))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(256))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(1, activation='sigmoid'))\n",
    "discriminator.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "discriminator.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Did you ask yourself how do actually connect them?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combined network\n",
    "discriminator.trainable = False\n",
    "ganInput = Input(shape=(randomDim,))\n",
    "x = generator(ganInput) \n",
    "ganOutput = discriminator(x)\n",
    "gan = Model(inputs=ganInput, outputs=ganOutput)\n",
    "gan.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "dLosses = []\n",
    "gLosses = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**So far we have two fully connected neural networks each plays a certain role. Both are making up our generative model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "sequential_1 (Sequential)    (None, 784)               1486352   \n",
      "_________________________________________________________________\n",
      "sequential_2 (Sequential)    (None, 1)                 1460225   \n",
      "=================================================================\n",
      "Total params: 2,946,577\n",
      "Trainable params: 1,486,352\n",
      "Non-trainable params: 1,460,225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "gan.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now training this GAN is a bit different"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/468 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epochs: 200\n",
      "Batch size: 128\n",
      "Batches per epoch: 468.75\n",
      "--------------- Epoch 1 ---------------\n"
     ]
    }
   ],
   "source": [
    "# Plot the loss from each batch\n",
    "def plotLoss(epoch):\n",
    "    plt.figure(figsize=(10, 8))\n",
    "    plt.plot(dLosses, label='Discriminitive loss')\n",
    "    plt.plot(gLosses, label='Generative loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.legend()\n",
    "    plt.savefig('images/gan_loss_epoch_%d.png' % epoch)\n",
    "\n",
    "# Create a wall of generated MNIST images\n",
    "def plotGeneratedImages(epoch, examples=100, dim=(10, 10), figsize=(10, 10)):\n",
    "    noise = np.random.normal(0, 1, size=[examples, randomDim])\n",
    "    generatedImages = generator.predict(noise)\n",
    "    generatedImages = generatedImages.reshape(examples, 28, 28)\n",
    "\n",
    "    plt.figure(figsize=figsize)\n",
    "    for i in range(generatedImages.shape[0]):\n",
    "        plt.subplot(dim[0], dim[1], i+1)\n",
    "        plt.imshow(generatedImages[i], interpolation='nearest', cmap='gray_r')\n",
    "        plt.axis('off')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('images/gan_generated_image_epoch_%d.png' % epoch)\n",
    "\n",
    "# Save the generator and discriminator networks (and weights) for later use\n",
    "def saveModels(epoch):\n",
    "    generator.save('models/gan_generator_epoch_%d.h5' % epoch)\n",
    "    discriminator.save('models/gan_discriminator_epoch_%d.h5' % epoch)\n",
    "\n",
    "\n",
    "def train(epochs=1, batchSize=128):\n",
    "    batchCount = X_train.shape[0] / batchSize\n",
    "    print ('Epochs:', epochs)\n",
    "    print ('Batch size:', batchSize)\n",
    "    print ('Batches per epoch:', batchCount)\n",
    "\n",
    "    for e in range(1, epochs+1):\n",
    "        print ('-'*15, 'Epoch %d' % e, '-'*15)\n",
    "        for _ in tqdm(range(int(batchCount))):\n",
    "            # Get a random set of input noise and images\n",
    "            noise = np.random.normal(0, 1, size=[batchSize, randomDim])\n",
    "            imageBatch = X_train[np.random.randint(0, X_train.shape[0], size=batchSize)]\n",
    "\n",
    "            # Generate fake MNIST images\n",
    "            generatedImages = generator.predict(noise)\n",
    "            # print np.shape(imageBatch), np.shape(generatedImages)\n",
    "            X = np.concatenate([imageBatch, generatedImages])\n",
    "\n",
    "            # Labels for generated and real data\n",
    "            yDis = np.zeros(2*batchSize)\n",
    "            # One-sided label smoothing\n",
    "            yDis[:batchSize] = 0.9\n",
    "\n",
    "            # Train discriminator\n",
    "            discriminator.trainable = True\n",
    "            dloss = discriminator.train_on_batch(X, yDis)\n",
    "\n",
    "            # Train generator\n",
    "            noise = np.random.normal(0, 1, size=[batchSize, randomDim])\n",
    "            yGen = np.ones(batchSize)\n",
    "            discriminator.trainable = False\n",
    "            gloss = gan.train_on_batch(noise, yGen)\n",
    "\n",
    "        # Store loss of most recent batch from this epoch\n",
    "        dLosses.append(dloss)\n",
    "        gLosses.append(gloss)\n",
    "        \n",
    "        if e == 1 or e % 20 == 0:\n",
    "            plotGeneratedImages(e)\n",
    "            saveModels(e)\n",
    "\n",
    "    # Plot losses from every epoch\n",
    "    plotLoss(e)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    train(200, 128)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## But this is just the tip of the ice-berg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Activation Functions](https://raw.githubusercontent.com/farisology/tensorflow-generative-model-collections/master/assets/etc/GAN_structure.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}