{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gluon example with DALI\n",
    "\n",
    "## Overview\n",
    "\n",
    "This is a modified [DCGAN example](https://gluon.mxnet.io/chapter14_generative-adversarial-networks/dcgan.html), which uses DALI for reading and augmenting images.\n",
    "\n",
    "## Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/mxnet/python/mxnet/optimizer.py:136: UserWarning: WARNING: New optimizer mxnet.optimizer.NAG is overriding existing optimizer mxnet.optimizer.NAG\n",
      "  Optimizer.opt_registry[name].__name__))\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "import matplotlib as mpl\n",
    "import tarfile\n",
    "import matplotlib.image as mpimg\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import mxnet as mx\n",
    "from mxnet import gluon\n",
    "from mxnet import ndarray as nd\n",
    "from mxnet.gluon import nn, utils\n",
    "from mxnet import autograd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 10 # Set low by default for tests, set higher when you actually run this code.\n",
    "batch_size = 64\n",
    "latent_z_size = 100\n",
    "\n",
    "use_gpu = True\n",
    "ctx = mx.gpu() if use_gpu else mx.cpu()\n",
    "\n",
    "lr = 0.0002\n",
    "beta1 = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lfw_url = 'http://vis-www.cs.umass.edu/lfw/lfw-deepfunneled.tgz'\n",
    "data_path = 'lfw_dataset'\n",
    "if not os.path.exists(data_path):\n",
    "    os.makedirs(data_path)\n",
    "    data_file = utils.download(lfw_url)\n",
    "    with tarfile.open(data_file) as tar:\n",
    "        tar.extractall(path=data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_wd = 64\n",
    "target_ht = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nvidia.dali.pipeline import Pipeline\n",
    "import nvidia.dali.ops as ops\n",
    "import nvidia.dali.types as types\n",
    "import numpy as np\n",
    "\n",
    "class HybridPipe(Pipeline):\n",
    "    def __init__(self, batch_size, num_threads, device_id):\n",
    "        super(HybridPipe, self).__init__(batch_size,\n",
    "                                         num_threads,\n",
    "                                         device_id,\n",
    "                                         seed = 12)\n",
    "        self.input = ops.FileReader(file_root=data_path + \"/lfw-deepfunneled/\", random_shuffle = True)\n",
    "        self.decode = ops.nvJPEGDecoder(device = \"mixed\", output_type = types.RGB)\n",
    "        self.resize = ops.Resize(device = \"gpu\",\n",
    "                                 resize_x = target_wd, resize_y = target_ht,\n",
    "                                 image_type = types.RGB,\n",
    "                                 interp_type = types.INTERP_LINEAR)\n",
    "        self.rotate = ops.Rotate(device = \"gpu\", interp_type = types.INTERP_LINEAR)\n",
    "        self.cmnp = ops.CropMirrorNormalize(device = \"gpu\",\n",
    "                                            output_dtype = types.FLOAT,\n",
    "                                            crop = (target_wd, target_ht),\n",
    "                                            image_type = types.RGB,\n",
    "                                            mean = [127.5, 127.5, 127.5],\n",
    "                                            std = [127.5, 127.5, 127.5])\n",
    "        self.uniform = ops.Uniform(range = (-10., 10.))\n",
    "        self.iter = 0\n",
    "\n",
    "    def define_graph(self):\n",
    "        inputs, labels = self.input(name = \"Reader\")\n",
    "        images = self.decode(inputs)\n",
    "        angle = self.uniform()\n",
    "        images = self.resize(images)\n",
    "        images = self.rotate(images, angle = angle)\n",
    "        output = self.cmnp(images)\n",
    "        return (output)\n",
    "\n",
    "    def iter_setup(self):\n",
    "        pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = HybridPipe(batch_size=batch_size, num_threads=4, device_id = 0)\n",
    "pipe.build()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe_out = pipe.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f0ad05d9cd0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztfWuMJNd13jlV1e95784+uLsiKZISTUkUJdOyFBmOHpYhO4b1xxD8gCEHApgfdiAjDiwpCQI7SAAZASw7SGCAiB0LiGNJfkWyfthWaMm2DEkWKVHiY/nmkvuenZmdR8/0q6pufnRvn++cme7t3Z3tWarPByz2VtftqttVdafOud853+EQAjkcjslCtN8DcDgc44dPfIdjAuET3+GYQPjEdzgmED7xHY4JhE98h2MC4RPf4ZhA3NDEZ+YPMvOzzPwCM39irwblcDhuLvh6A3iYOSai54joA0R0hoi+RUQ/F0J4eu+G53A4bgaSG/juO4johRDCS0REzPxZIvoQEQ2c+MzsYYIOx01GCIGv1udGTP1jRHQats/0PnM4HLc4buSNPxKY+SEieuhmn8fhcIyOG5n4Z4noBGwf732mEEJ4mIgeJnJT3/F9jmEGNg9oExFlA77CuuNeJtTdiKn/LSK6h5nvZOYiEf0sEX1xb4blcDhuJq77jR9CSJn5V4jor4koJqI/CCE8tWcjczgcNw3XTedd18nc1Hd8P+MWMfVHWdW/6Yt7DsctiaFT46rzZnfEcAQefIwdExi/h/3sezK9vmHtBg/ZdTgmED7xHY4JxASZ+qOab74MMRDGfEVzNtjrhn2VaWvuA2PT7tv9XthzabNa71Nmdb7r4XYf16B9e7EmNuQYV/fO9wb+xnc4JhA+8R2OCYRPfIdjAjFBPv73qe9u/nQPpJF2uNbyRWZ7EGmqqzbEjc9z7UAPvNpmfFGE47Bjl2MqX33IrdzhPu/1bR9C0w13z2+t58/f+A7HBMInvsMxgZggU38ArH2G20Ppn9GOOcQy3LEzwDZa3/YQ+Neao9HMdDuOHMJEg/mdg0JDd5jiarx6HIpgG0Jf4TF3mvox7Btg9hvscDl4EJ035MbscBfC7u0h3TAa75owJo/A3/gOxwTCJ77DMYG4dUx9tLwGW6/GZDWRZGgeG7MxH9GE0kFg5vg6zAzag6PFriX7Ec3SYavpNCRSDQeGl3HnaveAc10DIutmqMPLUYeb88OAI4sGfK7N+zg2Lgc8FFmWw3fMrx52wUe9QGFA+3pxE81+f+M7HBMIn/gOxwTCJ77DMYEYq4/PzJQUe6eMjF9Mo/nFI3uIpuNgsmkIdgxjwLj2KKNq0GF2MI5wfaId6xwDfGEzdPTPM3MMtS8bIA9Dw+9TCEi/yfEt3aYi93Zk3cE6QTRgfYX02s7Ogaieg3bssn2DuN7DD10DuTXENh0Ox2sUPvEdjgnEWE39QKFPWYVhYXE7EkqGmWij4jrs8es04UelrEYVU4xtYgt0i4ydWyqW+u1CsSDtpKD6cQyhZbF+DNJUzPtWu43fUv1azUa/nZjjlyvFfrtarfTbly6sqH7tdrPfznItKodeC0YaRrEOixuqbzdkayD2wnUbFvU5LNFniN/CCbhM6Y0J8Pkb3+GYQPjEdzgmED7xHY4JxHgLakQc4lLvb81QP/61hestfLAjsw4oMLwesfn7nIAfWCho33pmdq7fnp2d7beLxaLqVymX++3qVFXt29pu9dtLy8v9dqut/cpWU/qlhvZrNrf77YUFGcfM1Kzqd+68FFxO044Zx2a/zdHg5SikHO21V5vXqWTJmMk48nwZJthhaEtoR0Bb2pDoWknu9eWtJg3CnpTJZuY/YOYlZn4SPltg5i8z8/O9/+evdhyHw3HrYBRT/w+J6IPms08Q0SMhhHuI6JHetsPheI1gJFOfme8goi+FEN7c236WiN4TQjjPzEeJ6KshhDde9TgRh7i8+9+aW8XUv7bssS52GFZDhBuGZ6pBRB6YtgVDtxXA1F+YX1D75mC7DOa8jZhb39jot5tbdbVvG8zIegPoNvNbytVpGbl5jDod+V4nFUpwbram+h06dBDGqN2F8xek6nqrnUE/fbImjNGKiozM4A1hjFEgZcf9hM0auF3zNe1azdeEZp2d0vdzpip9axXph6Y9EdFcTbY//ZdP0CDsiak/AIdDCOd77QtEdPg6j+NwOPYBNxzAE0IIw6rgMvNDRPRQd+NGz+ZwOPYCt4ypP+pfhev+2zEwWcMcf4ipn+HftxGDCTnolfAEjh/boqlwL+KymMSVUkX1w9X6xcVFta8FUXcbm2LOr62tq37r62v9dq6i84gyMKU5ETPUmvpMsi820XRZKiv+eUATXl+PmRn5nUeOHjHjkO+tb8gK//ZWQ/XLQWAj7ejjZ21hCtTtM9c+wUBGq0+I+8zje7Ai5vcbj8ga9/13HlD9XndEfufCvHYD5qbkGNNg9s8ad6EEkXv3/qsv0SDcTFP/i0T0kV77I0T0hes8jsPh2AeMQuf9MRF9nYjeyMxnmPmjRPQpIvoAMz9PRD/W23Y4HK8RXNXHDyH83IBd79/jsTgcjjFh/GKbPfdjpxcymveu5RevQRt9RKAbu8PfH1jGaVh22GDhfptZhxlulURonaOHte87f0Aou0ZTR3BdXr/cb6+srPbbW9vbqp8S2DBObaY2MS3OGojiW3eM/18oyKM1VRT/FiP6iIi2tmR9YXVVr0NMz87IMaZlXSMYbZD2tlyDQmLrfMG6CawZRGa86Lsnkb4e+Jxpgo3oWFG+eP9BoU9/+E4dDXnnCVmnqQJlR6TXFwrqXDqSkdO9Wx33WH2HYwLhE9/hmECM19RnphB17ZqwIzxqxMSWoftkrz0+mu2jlnSyiLGkE5xrWNSh1bMLYN7nJiIvlMQEPHBQaLoji4dUv1YmlNVmfVPt21gXcxkj5pJEjwN/ZtrR4W4xXLtiApp4xgTGaEBLjyE1WQMXpljRxvImJAQ1t7SpXyjI+aYhuWe6MqX6NVLQy0/1bymWhBKL4HpEttQWhvyZH5PDPewY2nI5E3P82XMSabg4qynHI9PH+u2ZRB+jiH4GcLxpaupwZW7qOxyOG4BPfIdjAuET3+GYQIydzovoitjmYF39HRlQQ46mOw7OfLv++m04DszSGi3LLrGK/kpEQ4dkTlUk2w3pKzI+Yacl/mO7qX3JGMpJl4FqSjuaA8OozlJFj6MK6wFRJOe2WXHoFseG6iuAL3xgAWk57Z+/cuZMv91I9RijHKk4aU+XNVUWAzXZbphjACUW1BDNb0kH7TGh2kZwpN2WfUsQOvzMGU1b3nVM1jLmZ/XzElfke+1IvsdF3a+SztFewd/4DscEwie+wzGBGG8JrRCI+/ahKaU0tC70wCOOfu5Rte6hHZvIugj+TmJEXs7a/CtGsl01FFgCdE1e0PsWZsQknl8QCi+O9LVKO2K+ppnOrKuW5JYGCPzKjfs0XRO34tDstNq3OCcRaMWitK3uHWoGJoaaRBotLojZbyMID9Ykou3C5TW175WLove3vSFU3xETyVhIZPznWzqSMUaWDh4yzOjrfoDFCvQu3E7Y0IDwjKzBNX52WT8Tr1+V6zh97C61bzoW96ddluPX6EXVb7Ozd9PV3/gOxwTCJ77DMYEY/6p+z9QPplouRoXZMkI6Mm5wP/w7tmNVX620S7tgVqNLsIJbMiZ8wrD0C9FzsRF6O7IoptvBg3olNoPV9OeXtNbdApjc1bK0K2W9qt+Elfz5VJvOAQQwChAVt3j0NtXvgTf+QL/97h9+h9o3PQNRjvDTOpkWuWiDgEfLmNgolb22JslCzz/znOp3ti7jn6vqx3F9WtyA05ck+ajVnFH9DsNvW1nXkYztLbkGMSZIGRcMX4E7q1+Bi2CeOeRULrdA/jroRJyvXRL3pNz8QbXvYPuojBdcpiPF86rfQtQXuqav/+GvqH3v+qX/vmPUw+BvfIdjAuET3+GYQPjEdzgmEOOl85gpjru0RmT85xg8KytoWAR/rAiiEYnJokogSqtkEpuQ5lqYkcivQwcMlTUjtMvBGe2nLc6L7z4/J7778fmy6hfFQmWtt9Qu+tvvvNRvX25dUPtqVTlOgDLIM9MHVb8IOKpaVV+s9bWlfvsARMm96233q37/7O1v67cX5nRZKxVQCGslmYlaw0i+dqZ/aBvKbR1eFOHJxPjI0+fP9duXNrbUvjQWn3mzIcfPTKktLCO2sKCLOq3CmkeUyzoEsz4GZufl6WA6uZ3rtYFN6NtM8TnVAqlPXpJxvF4zsESJjDlP5Z5xqq/p3JRk+JUjfa2uFf7GdzgmED7xHY4JxFhN/QIHOlHumlgFozc/BSWGDs6ZMkuQ5HEETPMDszpZ4wBEnNljzE7J8TH6Koq0yReBOWsvDtKAXJDxVzqadumkYno2Uz3GF06d6rdnZo+pfVtgHpdqcvaopE3sYwtCDSVB02hZXbT033T36/rtt9//JtXv0CHR7SuW9BgjpFpRsCM1dB5EEJZNFGKSyDUuQV2Ae+/9AdWvUpNzh5dPqX1N4NXWUbBDD4PabdmHVYCJiKYhGrK9Dea80bPL08F6fJSJ35ga6jak8JQEOXfHuAv1urg0m9kral/t0D399lRFntt0Vbual0GoJItW6Ebgb3yHYwLhE9/hmED4xHc4JhBj9fFvOzhD/+EX30VERIdmtE9YLguHVDR/jgqQnRYDDRiZOmwMgusx6VDWmCU8NoIw3TSYjDPMvjJRnRmj2GYHPtf8TJvFNzu9osNym5n4/51I+3AN8AtngX5s5XqMSUt+p2HY6PZjsm7wpvvEnz58SNfYS4ACi4wgSAL+OoY6s8nAIxDpCMGEWUN4cwT3r1LVNNfr77hDvlPQQpyNk8/22/MQvntxTYfl1jclnLc6rcuG56msCa2Bpn+U6AvXwXBsE7TLQX5nZrTtQ4bXBK6HETDlhmyvLuksxHpTxlKZluMVzX2pwn2ZiiwneG0YpYTWCWb+CjM/zcxPMfPHep8vMPOXmfn53v/zVzuWw+G4NTCKqZ8S0a+FEO4joncS0S8z831E9AkieiSEcA8RPdLbdjgcrwGMUjvvPBGd77U3mfkkER0jog8R0Xt63T5DRF8loo8PPVmU0+Fql3o5WtWRR5gsFbEtVLR7dp61xXVZLqMBj/2ArmEyGm2MlI8Gii6gLmAwlzEDU//U2WW1j4sSmZUZYQssZbXdFFruUKSNKdSEX5zRtOVdt4mpe+cJMfurxsQug4Z/UrSRh1iDAF0fw6OpqD69K8A7JQdhEituUgL34diho2rfxqaY9KfPCWX66jn97DRzyZErVXXmXgWiIesgCBIi/YylLXDjTKltJWJixo9akZgQGgxd3azLGE8+eVrtKy4KvXffW8Q1mWssqX7HwGWqFbUre624psU9Zr6DiN5GRN8kosO9PwpERBeI6PANjcThcIwNI098Zp4ioj8jol8NIWzgvtB9he4a4MzMDzHzo8z86Pp2Z7cuDodjzBhp4jNzgbqT/o9CCH/e+/giMx/t7T9KREu7fTeE8HAI4cEQwoOzVWvCOxyO/cBVfXzuStn8PhGdDCH8Nuz6IhF9hIg+1fv/C1c7VppmdHG1S70cmtdZcZyhz2z8KHTXh5Wxvi5YfX+oB7fj8OK3MfZL9B+0DCi79YY+fgOouBaZcFugg7K23Jr2lqZ/ZiEMdWZK++d3HRc/+SBc41LBUILg48dlTSvi+kgAf5dtKCv47iHX6ZCosIRJlJEtBwf3PSlp+mp+TtY2ZqoS2otlsYmIVusSslubPqD2Hb/9RL+9tSFhriEztQqA6uOOCecFai4y95ohQ5HBB7elvDuglHTunPbxw5Mv9Nu33Slh1odZ37NKIsc41NRCnNeKUXj8dxPRLxLRE8z8eO+zf0fdCf95Zv4oEb1CRB++oZE4HI6xYZRV/a/RYB3r9+/tcBwOxzgw1si9dprT6eWumfaW12tTP+yJ2T4a0GBluyapxD2GldoWsy4zpaW2tqHEVab3NVtiRjZSHYHWhK5TU0K/ZZk2bcuxUHj3nHid2ncUymsXwK6OYxuhCKauMeHx56hkNBOdp7cHu2eKFjXipgwUW94y0ZwVcWOOHhHSaG5KPzunl2SteXl5Ve27+41v6LexfFdjW0dUJkUx4fO2icTsQLlx46skQAejdW8pUnx1RqSPf+miZO41QEi1HWvaMpShZkKu3aJrhcfqOxwTCJ/4DscEYrymfhbozGrXzImNrjmjJr6tysrY78ahpfoGV9y1kWpYsTXKZSWcjem2BdruqxvapMwjMQEbZnW6DefugEtQMSvJaETOT+nqs7WKuAgRRgZG2jREUz+xEYRYHDbBarnGnFfRixoZJlPBJW0bMzqD5W97jEJRxnznseP99v1gvhMRPXteoiNX1rSpn7blGpdB5GLblPKqQAXeqKPH2GzKdm7iOQNEUeJvKVU0U4JuYjHRx1hbklX+jQsy/vSQjvpMgAXKSbsSl/7be4mI6Mf+66M0CvyN73BMIHziOxwTCJ/4DscEYsx0XqDTK90oK0ujDSuMzUAbYR29YL+laCkrDAH+uVozMH/7VCRWwewCcQkQ8Ihz7ROmUM74zEXt48cLUuctrZ/RY0zAB62LP4e18oiIqhXJwFs4oGvzIV2Wg1/Jsfbxo0S2bYlrvAZxAvtY+5XpNujqGyHO5U3xoet1uQbtLf1bAvj8FSNMkkJJ8WmINLzr9uOq3+ycRDKevqAjxzfWheqbmhI/PjfrSBWIDMyamkZLIFvRVtcOA7JFW20d/VcuybVLMyPwmkpk5rnnJCIvndUrWhnJb+HERGLm3es/Kivub3yHYwLhE9/hmECM1dTP04zqy12zL8kNLQI0SSBt8sXA56HJbnNG0A2IY216YjIIoZleUhnGFDB6LGhRBwKdtiyX44eOKWMNyTYX1zRtdPSEmOaFjUtqXwPOjdZgw1BPc4tg3ifGtgP6Dc1BNskxmP8RTM2yVOkOyhe3trQJfOpVoaFeOa0TT06dPtVvt8BVKdjy1KDgcWRGC47Mg6gIllgrF/VjOz8jlOb5S5rOW7ks21gaO0n0BakBndeu62eiBG5G24j6xzHS0PJ5MMoknY5sp6bceF4Qt66+LNex2NIJRzhncvPw989nJ8UA+Bvf4ZhA+MR3OCYQPvEdjgnEWH38NM1pZaXrr3Ku/ZxCEKd2h2J4ASkqWAsImm4LJNvtSItX5ODUcjjUb2fROdUvTsSvTNtayDIBSgxDMPNY+3OnIASzOq1Dajtt8ZMrFV2zbnMd6qGVxFfLM+3rHYFS0O22psfOXBD/tAwhr9PT+rfMQJnvmh6Gok8vr8o6xJmzmn489aqIRC6v6FpuCRxj8aiIYRwx+v63HZWsu8xkzK2AcEY7l+tWMOsa83CNSyX9TOC6xOyUXINiQa8jRZB1Vyhp2jIuyLgSE86bgPhGlIqvnplnM48GU82Y19euy7OTb+h7SzmsE5gaflfqOo7m4fsb3+GYSPjEdzgmEGM19Ts50cV618RfzUzGGWq5R7akE+jPQUQU86w+fkdMtKisze8UNOHaTTFzi0Wt5d4OcszNuraBF8Gkn00kAy81AhXnlkUDLq5oUz/riOnJmTXXxIwMqZh5jQ3ttnz2T/60354zdvp0Tc5321GJEjxhot2Ow6kjIy5RA1O3ALp6MzV9z06AmR6bvMmz58VFOH1KXIRzZ7Vr9cLLQpnedZtWaJ+fF9O8CTSVLYV9eE6uwYEp/VuKUEa8ADRanJuaDKCrlyQmkzGR+2s1AxMlsAGZeoayo1Se29hEUTL0jZpyrXjTRBA25d7yjrLkXdeFebR3ub/xHY4JhE98h2MCMVZTPxDRFSP4mYtaN+2uE2LmtdtmxR+sms1tWN0tHlH9UG6tdkCvYmcgyNDaEFNxrqhN4K22/C3caOi/i1NVMdcOgo3XNNVyQyxmetGU/uVETNbMJGskWI0XTMO33neP6vf2B+7vt1999ZQe/wYkckDyTWZM2waIS7Rbevw1qFqLi+SVgrZzG0U0/fV6cgYiJtPzYopXy9o1WT53sd8+axJgDs1Ltd9CRa5jzbAhR+bFPTu6YJOWoPosROttmsQn1CS0rg9WV7aS7uChUlFFn7ZUvxzcuijTpn4R3KSoASv3dT0PQhMSq2q6JFpaOtjtY0VVBsDf+A7HBMInvsMxgfCJ73BMIMabnUdErZ4/8w/PmfLUB2/vt4ttTV/VQMRwqwP+Vqb/bjVy8NMamnqaBqHFTiSRWGlLHwMZtkLR+HoMVCJkX3FmRRfA8Uv1vgpQRa1I+4EplG5KwWVeu6jFJZ787uP99rPP61JKRw/JWskdr3t9v12r6TUVFPPc2tTZf6VYrh2KbaQNQz+CUMn8AR1dSFCWqwClsfA+EBHdf++9/fZsVe8rwuLOdl2eiWpZ+8hzIKKxUNPHYPgtGL24ftlEdua49qJ9a6TIIlMXoFwE0RKIzmt29PPdhjUQjPYj0msDcUvanbqZnpncp82OPsZTG10fv57ukY/PzGVm/idm/i4zP8XMv9n7/E5m/iYzv8DMn2PmG1P4dzgcY8Mopn6LiN4XQngrET1ARB9k5ncS0W8R0adDCHcT0WUi+ujNG6bD4dhLjFI7LxDRFdu40PsXiOh9RPTzvc8/Q0S/QUS/d7XjZT0hjfXLWlOeQOCgaPTsSx0xcYqJmKitXAtZlKeK0E/TS1Uw0ys1iLpL9TiqRaGGeEZH3U0VkfaSti3+fWhaotHith5jCXTrSgVtDrag9NbUtJjK9boutcXCgNFMSZu2B+fF5K6UxAQum6SUAObs9pY29adRfw7cGKtxOA1JL9VZTaMdhYSSCKiyUqzdJxTm4FTTih1wfRLoVzK0YhWEOUpWlwSoyQREXGxyU1yAfbm+LzkIwVRKmkYLW0Kf4iNXNPe2o/QP9RhjOF8GVPbmur4vGdCuRUPbcY+CHLXuxEiLe8wc9yrlLhHRl4noRSJaC6GvPnmGiI6NeE6Hw7HPGGnihxCyEMIDRHSciN5BRPde5St9MPNDzPwoMz86asqgw+G4ubgmOi+EsEZEXyGidxHRHHM/yf04EZ0d8J2HQwgPhhAeHF89XIfDMQxX9fGZeZGIOiGENWauENEHqLuw9xUi+hki+iwRfYSIvnDVkyURLc51fdy7F/X7/7ay1AmrxtqnrSXi62AoZGp8sQjopYIJuyyAgEc8BcfQhfQoh2wuDvr4sVJThPOSPteBqqwNRKypuAKEm5Zz4+xBBloRRBzfcPddqtt997253166qNcQpsDvXpiT9Qr0x4mIcqAZmw3tS2K9uU5brpW+GnqMtWlNF5KiwOS+xKb2HEEJ8JwNBdYCuhOEJiOjic8g4lIwVNwc+P8Zyb6so9d2IhAmze0bCvYVzXPVhFcnamME81wlUJNBFYMkohKKdACPu17XVPDGtlyfmt5FxeTadPVHIf2OEtFnuFtlMSKiz4cQvsTMTxPRZ5n5PxPRd4jo90c7pcPh2G+Msqr/PSJ62y6fv0Rdf9/hcLzGMNbIvTiKaK5X1vndb9IZVieKJ2WDNUEWo5Y+au4ZjfYApiJb/Xaww4IST9DLHDHQdJExSxmOgUePI23WHZgVyq4Y62NMTQkdxIkWlCAwnVPQdjt2RGchHgCz+tiRQ2ofasAPahMR1TfW++2srSMIUdsNy0elRs8+Qk15fQSKwSROwOwPVqACLp3VikfnIqSgKd8xuvRAgRWMrVutyvXugO5ibEx2zM6LrYsHfdma8EAXKpEY0ojhh7IpuYZuUQ5c30ZTP5vLG6BjaGpPJFGzd17jSg2Ax+o7HBMIn/gOxwRirKZ+lue00auculjVZYraIOpQNtF0jLFxWAnLmF2s1p3N37QBq51DF0Ht8bEzmKVJrPsdOiCr+tbUR2s5njUr4WB9bsNKe9mUe6qCeVmtapepBNcxxig5I2ceOqALyHqMGK3Xhmq2qV2Qx5V2vYvKsBIeKVNWP3IZsAtWmCRGtwDM7ywfnCy0eEC7PsduE6GV00si153ERv4ahTJMIk4Brnfe0m4RK5EOGO6O8m4C4xkqFwdN/a2O7rgKlYajpnYXKtWt3nnc1Hc4HAPgE9/hmED4xHc4JhDj1dVPczq32vVdo6D9ubAjxw33CdBXMi74yOWDhkL5aYbOAycu0g6/6peAFv1th3TW2nYTMgPN1a9Vhd6LG+LD1etaNCKC6MJyMD4+UKER6MFnNsoRhSFsuBf4u6j9326YtRfYN1XUkYFRQWi0SInP63N1YFz2eiOdl0E7NwIpCejsz8zqUtvFAmRDgghK2WTZbQF9itmEREQVLJNtogbboJEfQ2RgMFGIqvSCCYEc9CS1THTeRl2uf9sIcaSN3ppZsPGVu8Pf+A7HBMInvsMxgRirqU8kVuT5NU2L3DEtJmswpqc16ccGQ3PhlqYSDb1Uln2LC1rM41RDfve2CeAqQKmw1qYc89LyZdWvlUKySWbtRrl2BaDUCpF2pQpg9qZsdAGLci9KiZjErZamBFtNMT23t3W5pxKY0rHSm7ciFxhRaRKm4PqnaDrH+vmIKnKui1sX1b7nn36u327CdWsGE/1HqKGodlERNANDQV9HVQMX6NOQZgP7WQ6Zgc7Dfg1D561tyrNjLH069fzzRKTvyTD4G9/hmED4xHc4JhA+8R2OCcTYffwrOLuiffzbj6GvZzPrdveBbD/lEQ2LmYSN61cFguPbRQigfKZrOosqqovv1zH129JMvtcCQYZzS8uqX31b/LjZaZvtBllgcH2SRN/qUln84saWvhdbEA6qwkltRhvQapnRFMlRO14pVOhrFUCMJGR6jDm8l9Cl3XHPgH67YEpLP/XiS/32wUMH++31LbMmgYKgpcHUctGU6O7AuoEqFW6p5sG7BtJ5poQkrW3AmkpDr8vcdqB7PwvJaO9yf+M7HBMIn/gOxwRi30z9l1b1qd+VCmUSGU181HNXQgjD7PQd+3Y3723ZYx6WYjU4hlB3g2i0sjEb0bxPW9o+ToECgjID9MpZrdu3tiElwA4dXFT7sPwTluSy7kgCZnpU00WQtgKY/kCdFed1lGChKN8rxVaIBimeAAAdY0lEQVTnHYQzIMIvmGuawzUNRosuQLhbDBFyiakRkEOG3LqJLjy3IZGSU2DqtzMjkAKUXckcX5nziRGJwSxEiJrb8fgNoaT1UwVluEy05QoktG6ZDNbXHbqbiIgKycuDTwTwN77DMYHwie9wTCD2zdS/uGFEF6AkEJMOacPVe50oo4+pIutGHIc1PZWpf71L/qhZZwbZ3BYzPQs6UQSj2HBB9/yKFi05f1m2T3Q0MzDdQKUPqHSbmxVziPirGnekCJF7czMi0W2jBJug1ReM6YwuTRjmFaFIhymvFSDhqFCQJKCkYBNg5HuX1nSUYwQnvLy0KjvaerylOdBJLGrXp90UUZTUaAYmIHzS6mC5MQ0U3xjmQEbwDHRMQtAmyGuvrWlT/87D3e/xiE++v/EdjgmET3yHYwLhE9/hmEDsm4+/tKYjp5TAoRETCIw+15AYrj0pzgfU4dDoP3TajDAEbBdiXZ6qmYqPT0YMIgafHIUblta1j7+0Cpr4hgLrgA8agd+dGS33FLIEI3OMCKIGk0T6FTr6vmxvCVW2uaXLnm22gLaE3zJnhElqZdm2AhgRRgqCwKbR2qRGU8Z46cKq2lcGUc0OCIdWjR9fqgiFZzPrsBxbZuhf9PHzDmQ8tvX1zu2gBwAjHnPzQG/Bstjyul1v6Z5vZ22C3THyG79XKvs7zPyl3vadzPxNZn6BmT/HzMWrHcPhcNwauBZT/2NEBOVu6LeI6NMhhLuJ6DIRfXQvB+ZwOG4eRjL1mfk4Ef0LIvovRPRvuMt5vY+Ifr7X5TNE9BtE9Hujnvj8ijb1kYaw2uCBdyfqImPWKL1yNn/TYBPNtcyInGOZLCOXr5JG1KlMP8yT6BhhhPqGmO21o0fNELE8mBzkkkk8OfnymX77J977o2ofinQUMCrOiOIvXxDBinxbU4IFuHYbiZjAVg8edeo22/p3diDib+awRBdWy9rUL4IpnrVtPQVBG35Ly9CKJ597sd+2dBtG/zXA/C6a5wMj8FJj6utoTn0dsa5BDiJ5cayPP8zUH2Sd26q9qINyeUP/zkZP7z/ke2vq/w4R/TqJg32AiNZC6MuYnCGiYyMey+Fw7DOuOvGZ+aeIaCmE8Nj1nICZH2LmR5n50ev5vsPh2HuMYuq/m4h+mpl/kojKRDRDRL9LRHPMnPTe+seJ6OxuXw4hPExEDxMRsS016nA49gVXnfghhE8S0SeJiJj5PUT0b0MIv8DMf0JEP0NEnyWijxDRF67lxGeWtV9JBfGPotQIIRTEx+WOhG7uEGfE4nNsxRS0x9g/l6FMghJ/MDRX2J1uy0yYZFSQnc2mFrlQGvap9tOK4NfHII6ZF7RhdvLlU/32+Us6RLV0WMQ9GWio2NSsi6BGW7NtNPeBomrDOGyp7RKWqy7rzL2oKn1nFiUrLok0+ZNCeHOroymwDOr9bUPW3dklTdk9+/Jp6Wdq201DSfEYwqdjI5rZghLXRVOrkMHZtvX9UGCzgiW5W3q9IoV7vYMlHkBDB+vjwwfLm3oc9V5YcbajNsHuuJEAno9Td6HvBer6/L9/A8dyOBxjxDUF8IQQvkpEX+21XyKid+z9kBwOx83GvkXuXd7WpsrapphetaIeVgzRTBma4kYAHfXbo6BNvkiVIhaTKWWTEQaXJDOUTwJa7MjOWOYwhXFsb+vIPZSEbzd0tFsZqLNChOWjtW146sKFfvvJl59X+xbn3tRvT8P3qkYrbv7wETlv0WQJAl2GNFdsot3iIOMtF41ARQWEVeB77Y6+7x0w7y0Vl0IE5/Km0KDf+t6Tqt/L5y7121lkBEGAVhtW9gxN8elpXQshxuxQo5NYT+X+4rWq1HRJsTZQlZZaRdouG6LN14FneHNDj+OKKzRqhKDH6jscEwif+A7HBGLfTH2LM6tirh0+pqO7MliNxaq6IWjzEoUsErPiH0DaQkX8GQGJgOcaIvTBOZqQ+lxZJqZtnhrzGPyCNNVuAIp2FKDNptwTrsL/3Td0eMWhuRk5Xk1W2mdNObCkBKvRB3SF2QKYx1jWy9qeCayMs1kJT+FetHPUATSRddDOzPVeAlGNJ05KtPj3nn5W9VuDyMbilGYXUrg3ZfgtU9OGhQAfLDLiKVVIpuqYqMEIVu8rU+IiVMqaAVlfA4n0HWIkON6B3agN13TVCNmsrHVdpjTb4yQdh8Px/QOf+A7HBMInvsMxgbh1fPwl8XffclxnrW2Q+K3tCHTkjY/fgbS4EGmfdjqIeEUtl/WEQkdTaih2GEw5ZpX5xLjuoM+VQnRXyVCTmN2V20wq8K0TiLorsKYmMTrru08/o/Ytzsv6yMKBAzJcM8YpKC1dLhtBENCYR+38otGURyc0Nb5vGCDEGQzd1ITMwNV1HZH35FPi1z93UkphZYYOK0H0XK2qaTQKEBEK/dhwsBiV2DERlVEV9f319xjXZdCvT/V9x4hQe61aeB1hqcTo0VAOz+NLl/UYnzjXvY6NttN5DodjAHziOxwTiFvG1D+/IkNZi4+ofU+nb+m3G/HxfrtmRt8uiqnVzDXdMRUk8uuOSJI6jubfVv2qvNJvB6NFH8HlCqAPZ81GNP2napo2isGEb7R0UkpUkL4FrN5qzEuklDbqOhnkm4890W8fgKSRD/5zHV1dPiLXOCSmJBUcH4Ussh0ah6jpp01PjMhrgRhJ05a4euV8v/2tx76h9i0tCwV27z339ttb5r688Mq5fnu2ok39TltcyCJQmDZKsAKuzw7xlC05RtMk33TAdWmn8szVSjpSMm3LdWyZpKhOApGkKPbSMtQhJD6tdPQz8dSpLqXppr7D4RgIn/gOxwTCJ77DMYG4ZXz8J5ZEMOHEhTerfd/evq3fnrpDfL3jZe0DoQ+3nenspbWG+O5FFv/uQHJQ9ZsGqi8LpoYf+riMfrEeR5qLH9hp63EkILbRbms/s1wCoQgU0TA+Pm4ZSX+6CGGuf/3Vr/Xbx03G2fSDcusLJrOuFIufjMKbxYIOP261JVQ2NesVbfDlV8FXf/GFF1W/V0/JesvigQW17743vKHfXqnL8RstfU1RwCMxlOPWpuyrTQnVmXU0RYoUntWmX1tf67e3TXZegHLjTbgGkYk/npqWGoSNjhZPwecqbUGGqaXz2jAuk4VYr3fXF3ZQxAPgb3yHYwLhE9/hmEDcMqb+k1D6aH5DZ+edK4pyd7ktZtERw+cdm4USRqn+m9Zsyvc2L77Sb7d3MFRgzrKmBDNwERLIwGOjI4cqHe1LS2pXDaizxIqAMIqFyOeRjRYbYs1hVN/mtpjiTz7ztOr3lnvv7LfL6zpyLwAtFTI5N9e1vn8bsgubW3rf5VUxZy9flnbBlMK+4y4Zx1TB+C1wHbNUTPOOyVacXpDIzrX6utqHJa8xgrBshEnwGndamhLbBtdi29B5UxUx4UtA4QVj6kclLF+u92GptkKEbdWN8PJwbMp19zycUfVs/Y3vcEwgfOI7HBOIW8bUv7whK7+NqhGvgNX7V8+Ixlzz+OtUv6wD4hKRXsVOgyTjlMBkp0RfgiyT7YiMAAZITbfbsKpfnFX9qCxRgm3SpmFgic7rpHofaut1IPqPjf5yDKZi2KHVrNb8+63TF86rbivrwnIcOqKZDTSJV5elX8NIhXeAvdhRHRai/7DMFMpdE4ksNBFRwUTTtSCBJ4dITOMh0eyCXP/Lq3W1L4HkmzK0s9SwECgQYtw/lAC3WtiYcxRjOTAjOILMTKWsn7mO0SG8gp0C8XgNtDsSx1a2Yzj8je9wTCB84jscEwif+A7HBOKW8fEba0LnTZV0RttyUyiaaciy29zSPucTZ2XfgYOH1b72ZYkYuy+ByKlE+4QpilIatc12ImsPr2yKQOU26YizubJ8b6t0h9rXYvFpg8loa26D9jqDGIYp9xRFIHIxpKRzAGqvYcpTnb8oNOM9QKkRERUTEI0Ev3hmxqybYFlv45/jONCH3TK0XyvHaDcjWrqNoiXyO2eMoGayDPfT0Fn6egBdGgbr6hfNuk8CvruVrWd4d05NCa2YJkYMc0XWSuJEv29zEBZBam/HWxnHbJ+d9s4uwzDSxGfmU0S0Sd3Y1DSE8CAzLxDR54joDiI6RUQfDiFcHnQMh8Nx6+BaTP33hhAeCCE82Nv+BBE9EkK4h4ge6W07HI7XAG7E1P8QEb2n1/4MdWvqffwGx0NERHlD683Pl0U7rgZJEbO5NvVfrgtl16hqc/BYIjbQdCzmZgFcByKiSIlLmMg6iMZabYspvpzPqH45Cw3VKmrN+jr8NpvIkTXFfItjiAw0lW6LoA9vdd7RvEeN9izSv2VlTX73yqrWuiuXxLQtA5VaZC1yUUCTONYUbIZluCBaL4r0uyYBwREr5pHDdgTU5JwpT4UmfMuIaJTBPQtDEljaUGW3lBjxFNTq65iELKDt2lBxt1Yx7ggcI7HCKii+oardmirMEM5pPDxqNLr79rqEViCiv2Hmx5j5od5nh0MIV8jhC0R0ePevOhyOWw2jvvF/JIRwlpkPEdGXmVlJu4YQAg8IEu79oXhot30Oh2N/MNIbP4Rwtvf/EhH9BXXLY19k5qNERL3/lwZ89+EQwoOwNuBwOPYZV33jM3ONiKIQwmav/eNE9J+I6ItE9BEi+lTv/y/s1aAuL2uf8w3venu/XQFW6mBN+5UXZiR0cyu1lAkKGoofWDChjhH0i3Pji3XEf5opyzpBK9PrBJ2WCDdQ0Lr99ab0bbU05cMFoc4YfN/IiHmivn3TZBCm4MdizcHLW1pA4okXTvXbM9N6jaJYFh/66CIIgJr6eDEDzWhCSAf5mjb8uAC/Jcv0mg367jHE0SYmOjUDKtGW4cZQ3FTp5WsDFUVRQsWE5YJfb4U4K0C1rkAWYu2EXoeIC3K+mnndNgNSjrhTX0NcEkrNNdjsHcO4/gMxiql/mIj+onfDEiL6PyGEv2LmbxHR55n5o0T0ChF9eMRzOhyOfcZVJ34I4SUieusun68Q0ftvxqAcDsfNxS0TuYd45qnvqO3KG36o3/6hN4sO28KM1oOvNsRFWLmkI/IC6Mqh5ZybUls5pH6ZJDCKIVvvyLTQcrWOXt6oXxLzPtvQkWrrdXEDNjN9+WtFEcQoQCZgM9XmK5Zxjo2wBZbvykGQb9PorZ989YKMt6FptFfOSabkO3/wgX77bQ/oZZoK6vEZLRKk7XCMdrxYNtyybVj2HHUB07a+Hluge9825bVacNAUdOrKphR2AejISsFEjrbk+rS2NNV8cFFKul1YFzeubOjkClCQVePSrK7rY/ZhoxCjQRtEaf9pdV19h8MxAD7xHY4JhE98h2MCcUv6+I2NS2r74iWhSZ45BzXrUhO6CZRPnutstCwHnzwRvzshfa4UwlBz1iKUDahbvAm++uNPP6v6Pfb42X77yZf0WsMyhqhaZR2gwGqwLzdlm1FzH1VliIjaSh9ejtHKjE8IVFnnoqZPl5bkej/13HP99pFH/k71u//Nb+q373vT3Wrf4XkJs56bk7Dl3IQfo8hlbkQ0CbI026BWtG7UfrYboNRj0tOasB6wCZmBUws6oxJr6bEJqUUFHltCewp891JDxmWz5PDZLJR1tmUOApv4PfN4UBGT8wwd2X80R8zO8ze+wzGB8InvcEwgbklTv7OpzePWltg8Fy6JWXc01rRO3hQ756AJj6qBEMUMiBaWMx2J1W6Jebm0oamRZy6ImMJ3nhPz+B+fXVb9zqyIOdg07sgWmHVl1rROFWy7KoqomyhE1OPPjAAmqzJf0gxGVAQjvJrWPIyxnLS0z7+qf+fTZ/++3576yt+rfffec0+/fdfr75Cxm1dNa1tcsNV1fXxk7bbB3D61pN0zzHwrlTSviKb/Np7LmPOzUGLMipZgBmShNDgL8cCCuDTBRC5GMMZtI/SZQDbkNpTQDmzi8GCzbejIuBfBGVxX3+FwDIJPfIdjAnFrmvpbZqW9IGbNpQasipM2u2ZKYkLdNqdXTmcviG4flslabR5V/R57/GK/ffKMTr759iun+u1TS2JiX9rSJlkLKumWcqOZDuIhrbY2y+qRjP9gWbTuY1OJdmZWTMqVyzoJKAKTHo3BzJiNDMkgO4xDSDwpgKBEME9LCklGWyZg7NGTUv/gW08JM1CpaqZkC6rqNttGVARD+SAisVrTTAZW8S0YM73ZQSZGXMhGXV83Dkfk+IYp6YC7UKyYcmNgwk9BNd7ICJMUS+JKhIY+d04yriboDhpvhCr4SMemHkTfu9S/fxD8je9wTCB84jscEwif+A7HBOKW9PG3159X24XwVL/dXL+r397Y1LXzbp8VQYkpOqf21UpCvz31/Jl++y//9h9Vv698D4QyYl1KuUHor6PWunZwMXEqNfRKDNScLX+9BVFhGVB21aquNzc9LYIjhYL2JdNUC270z2VELnHbimO0QHiyCplp5bLW1W/C49MMei2jA/RbswkRbS3jx8PFSor6d1ZRmGRb7su0SQUMifjdjaZWeN9qQx0D+J1p0OPADEibDUkRRt2VzD4ZfxsiClNTZ6AKEX71y3raBXj/dsDHT01+KEZilsyzKRqxJvpxAPyN73BMIHziOxwTiFvS1LcJNu85LObrC6df6bebZ7UZ/dyrQtnVL35T7dt4SRJMLr74ar994bKOEqyDBdjaUS8J9OEDmK+WEANTMTfmPJZIsi4Caswh9XR4RpfhxsQcW3YazXQ8XmTMeWveD9qHlFq9rkVFOvlg0zmH7xVAYKRY0nRYAbYjI9KBYhl5Jib7bEWb+guLQm+mqaFItyH5BhKwOOhnDMVNrLZgtQL0rClnNlUTunO7M/hazcB9Wi0b3X51THnWranfRjfRuDvz091jrtZ3d/Us/I3vcEwgfOI7HBMIn/gOxwTilvTxLRbPP9Zv523xCV987uuq36mLEm57/lVNCW6vCJ2XgvZ6akQLI6i1xlbsAKkSDCfdEfMKwgpGQVLVRjNlodGlw3p2lr6ahjUE6+PXMSx1W/y9HXXjoEZgZOrqIUUYQbtiyperK5cZDXhYvwhIHUb2kYP6AWY9JAqyXoE0Za2o/ewHfuB2GK/e9xJkVKLeaNFkt3VaEjrMRrByCsKMi8b/r0C2Hmib0pahLY8evU3G0dJ++IVLkpVYgFBilWlJRAGuVccI67d7fUfU4fA3vsMxifCJ73BMIF4Tpv7TX/+Hfvs7Z4WyuxS0WbcJmVjNli25LD81gGY9sT6Gtr5NBJeKwgNqz4w3gBkZGdOWYV9mMveQftuGkldLl7QmXiEC098wjhFk3RXAbclNdh7Sb5xrsxGpuFZbzO24oKPWkCK0blGM2oVwvHZbC4egzl5ITVly2MdwYzITFXfPncekX0FHtH35G4/32w0QFeFUj6MCEXnBlB7Hfao0OBE1QC0kKYtL0FzXEYRZLq7E9KwpWYb07JS4U1tNm60o97bV2b10mtX6G4SR3vjMPMfMf8rMzzDzSWZ+FzMvMPOXmfn53v/zVz+Sw+G4FTCqqf+7RPRXIYR7qVtO6yQRfYKIHgkh3ENEj/S2HQ7HawCjVMudJaIfJaJfIiIKIbSJqM3MHyKi9/S6fYaIvkpEH78Zg/zGeWmvRSCLXNZmXbkt5lQnXVP7MjDbI7SPM30MNJVCpKO7WJm2IGSxw7zCFW1tlgbVNtFzKqpPPl7f1FFgIZfyVxWz4o/y2i1IGimaaDesEpwbdqEDYhZYbXbdVNwtwgp6rawj8mLFFMi5mkYoI4Ol8NToAjahOvEcuEz1hjbTp6qSPHT7cX0/p0DKOs/EpN5a12IYm3U55usOG/csl2tQKGpmownXZ25GrnG7pce4uSHn297W97MC1w4j/JodLQSDXl3IbNQnXRNGeePfSUSXiOh/MfN3mPl/9splHw4hXJmSF6hbVdfhcLwGMMrET4jo7UT0eyGEtxHRFhmzPnRXpXb9m8PMDzHzo8z86I0O1uFw7A1GmfhniOhMCOFK1sufUvcPwUVmPkpE1Pt/abcvhxAeDiE8GEJ4cLf9Dodj/Liqjx9CuMDMp5n5jSGEZ4no/UT0dO/fR4joU73/v3CzBtkoiIgBCjcUI00vBSiRXC3pjDZuSURbBhlcwQgXMOjes4loUxltg5PbqAN+H+fDBDD0Pm0yyW9JTRbfGkbnFUwmGZRnZqDs2qmmhhKgJu04EEWMkjNUGV6C7abx/xP5Hq5lWLOQQSy+ZKLppmflvicgTLq6qvX3GcqNJybDb2FWBDDbdbkvDdMPIw2jRFO8MQw6jobdM+hnjt8G+u1/f/5LA751/WjU16/eCTAqj/+vieiPmLlIRC8R0b+krrXweWb+KBG9QkQfvqYzOxyOfcNIEz+E8DgR7Waqv39vh+NwOMaB10TkXgA9t4WqRD2ZSkSUwZIFJ6aKLNArDLZb0VYuBa24tDM4sk6LXBjzz7gICOxp6Twsc5XBPpvLk0JkWchNdBe05+clpmpzU9NXKVw8ax4nEJ2GkWpFQ58WgUq0wh4duHZt0N8PwVKHsq9hquDWt0R8owhRfDMmXPHiikTJZZGmLQ9Apd7lhiTsxEaX/vyymMrn/+Ex2nN87dt7f8wbgMfqOxwTCJ/4DscEwie+wzGBYOtz3dST8Yg1fA2OHZPsq9kZ8Fs3tlU/LFncybR/HkN95hh88rbxK1EAs9HQFBXSXnguy+lwnO3e7yrIoW9mhT4B6p4ZWhEZsSpklVVMSC1m3dnxY7ZYAtRWuapLfuNagNXtR9HPSF23fGC/lglzzeHHbazrDEXHYARbE30X+Bvf4ZhA+MR3OCYQ4zb1L1E32OcgES1fpfvNxq0wBiIfh4WPQ+Nax3F7CGHxap3GOvH7J2V+dL9j92+FMfg4fBz7NQ439R2OCYRPfIdjArFfE//hfTov4lYYA5GPw8LHoXFTxrEvPr7D4dhfuKnvcEwgxjrxmfmDzPwsM7/AzGNT5WXmP2DmJWZ+Ej4buzw4M59g5q8w89PM/BQzf2w/xsLMZWb+J2b+bm8cv9n7/E5m/mbv/nyup79w08HMcU/P8Uv7NQ5mPsXMTzDz41dk4vbpGRmLlP3YJj4zx0T0P4joJ4joPiL6OWa+b0yn/0Mi+qD5bD/kwVMi+rUQwn1E9E4i+uXeNRj3WFpE9L4QwluJ6AEi+iAzv5OIfouIPh1CuJuILhPRR2/yOK7gY9SVbL+C/RrHe0MIDwB9th/PyHik7EMIY/lHRO8ior+G7U8S0SfHeP47iOhJ2H6WiI722keJ6NlxjQXG8AUi+sB+joWIqkT0bSL6YeoGiiS73a+beP7jvYf5fUT0JepmH+zHOE4R0UHz2VjvCxHNEtHL1Ft7u5njGKepf4yITsP2md5n+4V9lQdn5juI6G1E9M39GEvPvH6cuiKpXyaiF4loLUjdqnHdn98hol8nERk8sE/jCET0N8z8GDM/1Pts3PdlbFL2vrhHw+XBbwaYeYqI/oyIfjWEoKomjGssIYQshPAAdd+47yCie2/2OS2Y+aeIaCmEcBMkb64ZPxJCeDt1XdFfZuYfxZ1jui83JGV/LRjnxD9LRCdg+3jvs/3CSPLgew1mLlB30v9RCOHP93MsREQhhDUi+gp1Teo5Zr6SazuO+/NuIvppZj5FRJ+lrrn/u/swDgohnO39v0REf0HdP4bjvi83JGV/LRjnxP8WEd3TW7EtEtHPEtEXx3h+iy9SVxac6CbLg18Bd4Xpfp+IToYQfnu/xsLMi8w812tXqLvOcJK6fwB+ZlzjCCF8MoRwPIRwB3Wfh78NIfzCuMfBzDVmnr7SJqIfJ6Inacz3JYRwgYhOM/Mbex9dkbLf+3Hc7EUTs0jxk0T0HHX9yX8/xvP+MRGdp27d6zPUXSU+QN1FpeeJ6P8R0cIYxvEj1DXTvkdEj/f+/eS4x0JE9xPRd3rjeJKI/mPv89cT0T8R0QtE9CdEVBrjPXoPEX1pP8bRO993e/+euvJs7tMz8gARPdq7N/+XiOZvxjg8cs/hmED44p7DMYHwie9wTCB84jscEwif+A7HBMInvsMxgfCJ73BMIHziOxwTCJ/4DscE4v8D2dnUTtvHlmkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0adec3f910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipe_out_cpu = pipe_out[0].asCPU()\n",
    "img_chw = pipe_out_cpu.at(20)\n",
    "%matplotlib inline\n",
    "plt.imshow((np.transpose(img_chw, (1,2,0))+1.0)/2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nvidia.dali.plugin.mxnet import DALIGenericIterator\n",
    "dali_iter = DALIGenericIterator(pipe, [\"data\"], pipe.epoch_size(\"Reader\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build the generator\n",
    "nc = 3\n",
    "ngf = 64\n",
    "netG = nn.Sequential()\n",
    "with netG.name_scope():\n",
    "    # input is Z, going into a convolution\n",
    "    netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, use_bias=False))\n",
    "    netG.add(nn.BatchNorm())\n",
    "    netG.add(nn.Activation('relu'))\n",
    "    # state size. (ngf*8) x 4 x 4\n",
    "    netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, use_bias=False))\n",
    "    netG.add(nn.BatchNorm())\n",
    "    netG.add(nn.Activation('relu'))\n",
    "    # state size. (ngf*8) x 8 x 8\n",
    "    netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, use_bias=False))\n",
    "    netG.add(nn.BatchNorm())\n",
    "    netG.add(nn.Activation('relu'))\n",
    "    # state size. (ngf*8) x 16 x 16\n",
    "    netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, use_bias=False))\n",
    "    netG.add(nn.BatchNorm())\n",
    "    netG.add(nn.Activation('relu'))\n",
    "    # state size. (ngf*8) x 32 x 32\n",
    "    netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, use_bias=False))\n",
    "    netG.add(nn.Activation('tanh'))\n",
    "    # state size. (nc) x 64 x 64\n",
    "\n",
    "# build the discriminator\n",
    "ndf = 64\n",
    "netD = nn.Sequential()\n",
    "with netD.name_scope():\n",
    "    # input is (nc) x 64 x 64\n",
    "    netD.add(nn.Conv2D(ndf, 4, 2, 1, use_bias=False))\n",
    "    netD.add(nn.LeakyReLU(0.2))\n",
    "    # state size. (ndf) x 32 x 32\n",
    "    netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, use_bias=False))\n",
    "    netD.add(nn.BatchNorm())\n",
    "    netD.add(nn.LeakyReLU(0.2))\n",
    "    # state size. (ndf) x 16 x 16\n",
    "    netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, use_bias=False))\n",
    "    netD.add(nn.BatchNorm())\n",
    "    netD.add(nn.LeakyReLU(0.2))\n",
    "    # state size. (ndf) x 8 x 8\n",
    "    netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, use_bias=False))\n",
    "    netD.add(nn.BatchNorm())\n",
    "    netD.add(nn.LeakyReLU(0.2))\n",
    "    # state size. (ndf) x 4 x 4\n",
    "    netD.add(nn.Conv2D(1, 4, 1, 0, use_bias=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loss\n",
    "loss = gluon.loss.SigmoidBinaryCrossEntropyLoss()\n",
    "\n",
    "# initialize the generator and the discriminator\n",
    "netG.initialize(mx.init.Normal(0.02), ctx=ctx)\n",
    "netD.initialize(mx.init.Normal(0.02), ctx=ctx)\n",
    "\n",
    "# trainer for the generator and the discriminator\n",
    "trainerG = gluon.Trainer(netG.collect_params(), 'adam', {'learning_rate': lr, 'beta1': beta1})\n",
    "trainerD = gluon.Trainer(netD.collect_params(), 'adam', {'learning_rate': lr, 'beta1': beta1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:speed: 222.241457791 samples/s\n",
      "INFO:root:discriminator loss = 1.505121, generator loss = 6.288426, binary training acc = 0.585938 at iter 0 epoch 0\n",
      "INFO:root:speed: 649.422288887 samples/s\n",
      "INFO:root:discriminator loss = 1.596359, generator loss = 12.483607, binary training acc = 0.856126 at iter 100 epoch 0\n",
      "INFO:root:speed: 655.019377131 samples/s\n",
      "INFO:root:discriminator loss = 0.521827, generator loss = 6.631699, binary training acc = 0.844450 at iter 200 epoch 0\n",
      "INFO:root:speed: 659.310065677 samples/s\n",
      "INFO:root:discriminator loss = 0.300034, generator loss = 4.885581, binary training acc = 0.937500 at iter 0 epoch 1\n",
      "INFO:root:speed: 616.350547846 samples/s\n",
      "INFO:root:discriminator loss = 0.200199, generator loss = 3.784022, binary training acc = 0.860226 at iter 100 epoch 1\n",
      "INFO:root:speed: 584.805042079 samples/s\n",
      "INFO:root:discriminator loss = 0.717995, generator loss = 6.369466, binary training acc = 0.873834 at iter 200 epoch 1\n",
      "INFO:root:speed: 664.692635119 samples/s\n",
      "INFO:root:discriminator loss = 0.708390, generator loss = 5.441530, binary training acc = 0.820312 at iter 0 epoch 2\n",
      "INFO:root:speed: 663.790266025 samples/s\n",
      "INFO:root:discriminator loss = 0.276476, generator loss = 4.742358, binary training acc = 0.886835 at iter 100 epoch 2\n",
      "INFO:root:speed: 630.603871453 samples/s\n",
      "INFO:root:discriminator loss = 0.249326, generator loss = 3.799110, binary training acc = 0.882618 at iter 200 epoch 2\n",
      "INFO:root:speed: 670.599102655 samples/s\n",
      "INFO:root:discriminator loss = 0.528475, generator loss = 3.571980, binary training acc = 0.851562 at iter 0 epoch 3\n",
      "INFO:root:speed: 668.351739626 samples/s\n",
      "INFO:root:discriminator loss = 0.464067, generator loss = 6.675239, binary training acc = 0.897973 at iter 100 epoch 3\n",
      "INFO:root:speed: 667.862862375 samples/s\n",
      "INFO:root:discriminator loss = 0.440649, generator loss = 3.453180, binary training acc = 0.896105 at iter 200 epoch 3\n",
      "INFO:root:speed: 595.226963502 samples/s\n",
      "INFO:root:discriminator loss = 0.241294, generator loss = 4.953326, binary training acc = 0.960938 at iter 0 epoch 4\n",
      "INFO:root:speed: 555.039577531 samples/s\n",
      "INFO:root:discriminator loss = 0.623782, generator loss = 5.216733, binary training acc = 0.894338 at iter 100 epoch 4\n",
      "INFO:root:speed: 510.986385649 samples/s\n",
      "INFO:root:discriminator loss = 0.468707, generator loss = 3.081922, binary training acc = 0.885650 at iter 200 epoch 4\n",
      "INFO:root:speed: 616.036535046 samples/s\n",
      "INFO:root:discriminator loss = 0.473579, generator loss = 3.601946, binary training acc = 0.882812 at iter 0 epoch 5\n",
      "INFO:root:speed: 631.699045987 samples/s\n",
      "INFO:root:discriminator loss = 0.969454, generator loss = 5.355912, binary training acc = 0.877475 at iter 100 epoch 5\n",
      "INFO:root:speed: 650.657979445 samples/s\n",
      "INFO:root:discriminator loss = 0.558474, generator loss = 4.593861, binary training acc = 0.877293 at iter 200 epoch 5\n",
      "INFO:root:speed: 624.354169526 samples/s\n",
      "INFO:root:discriminator loss = 0.469535, generator loss = 5.138221, binary training acc = 0.945312 at iter 0 epoch 6\n",
      "INFO:root:speed: 630.995204694 samples/s\n",
      "INFO:root:discriminator loss = 0.438681, generator loss = 3.152040, binary training acc = 0.875851 at iter 100 epoch 6\n",
      "INFO:root:speed: 653.660419076 samples/s\n",
      "INFO:root:discriminator loss = 0.327069, generator loss = 4.408347, binary training acc = 0.881685 at iter 200 epoch 6\n",
      "INFO:root:speed: 530.838463699 samples/s\n",
      "INFO:root:discriminator loss = 0.588785, generator loss = 2.339013, binary training acc = 0.835938 at iter 0 epoch 7\n",
      "INFO:root:speed: 661.320929866 samples/s\n",
      "INFO:root:discriminator loss = 0.190490, generator loss = 4.891218, binary training acc = 0.873144 at iter 100 epoch 7\n",
      "INFO:root:speed: 588.864076797 samples/s\n",
      "INFO:root:discriminator loss = 0.345964, generator loss = 5.012253, binary training acc = 0.873290 at iter 200 epoch 7\n",
      "INFO:root:speed: 631.823941176 samples/s\n",
      "INFO:root:discriminator loss = 0.327764, generator loss = 2.914495, binary training acc = 0.890625 at iter 0 epoch 8\n",
      "INFO:root:speed: 639.584315576 samples/s\n",
      "INFO:root:discriminator loss = 0.354618, generator loss = 4.216711, binary training acc = 0.883354 at iter 100 epoch 8\n",
      "INFO:root:speed: 643.682258633 samples/s\n",
      "INFO:root:discriminator loss = 0.321935, generator loss = 3.864093, binary training acc = 0.877604 at iter 200 epoch 8\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "import time\n",
    "import logging\n",
    "\n",
    "real_label = nd.ones((batch_size,), ctx=ctx)\n",
    "fake_label = nd.zeros((batch_size,),ctx=ctx)\n",
    "\n",
    "def facc(label, pred):\n",
    "    pred = pred.ravel()\n",
    "    label = label.ravel()\n",
    "    return ((pred > 0.5) == label).mean()\n",
    "metric = mx.metric.CustomMetric(facc)\n",
    "\n",
    "stamp =  datetime.now().strftime('%Y_%m_%d-%H_%M')\n",
    "logging.basicConfig(level=logging.DEBUG)\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    tic = time.time()\n",
    "    btic = time.time()\n",
    "    iter = 0\n",
    "    for batches in dali_iter:  # Using DALI iterator\n",
    "        ############################\n",
    "        # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))\n",
    "        ###########################\n",
    "        data = batches[0].data[0]  # extracting the batch for device 0\n",
    "        latent_z = mx.nd.random_normal(0, 1, shape=(batch_size, latent_z_size, 1, 1), ctx=ctx)\n",
    "\n",
    "        with autograd.record():\n",
    "            # train with real image\n",
    "            output = netD(data).reshape((-1, 1))\n",
    "            errD_real = loss(output, real_label)\n",
    "            metric.update([real_label,], [output,])\n",
    "\n",
    "            # train with fake image\n",
    "            fake = netG(latent_z)\n",
    "            output = netD(fake.detach()).reshape((-1, 1))\n",
    "            errD_fake = loss(output, fake_label)\n",
    "            errD = errD_real + errD_fake\n",
    "            errD.backward()\n",
    "            metric.update([fake_label,], [output,])\n",
    "\n",
    "        trainerD.step(data.shape[0])\n",
    "\n",
    "        ############################\n",
    "        # (2) Update G network: maximize log(D(G(z)))\n",
    "        ###########################\n",
    "        with autograd.record():\n",
    "            fake = netG(latent_z)\n",
    "            output = netD(fake).reshape((-1, 1))\n",
    "            errG = loss(output, real_label)\n",
    "            errG.backward()\n",
    "\n",
    "        trainerG.step(data.shape[0])\n",
    "\n",
    "        # Print log infomation every ten batches\n",
    "        if iter % 100 == 0:\n",
    "            name, acc = metric.get()\n",
    "            logging.info('speed: {} samples/s'.format(batch_size / (time.time() - btic)))\n",
    "            logging.info('discriminator loss = %f, generator loss = %f, binary training acc = %f at iter %d epoch %d'\n",
    "                     %(nd.mean(errD).asscalar(),\n",
    "                       nd.mean(errG).asscalar(), acc, iter, epoch))\n",
    "        iter = iter + 1\n",
    "        btic = time.time()\n",
    "    dali_iter.reset()\n",
    "\n",
    "    name, acc = metric.get()\n",
    "    metric.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize(img_arr):\n",
    "    plt.imshow(((img_arr.asnumpy().transpose(1, 2, 0) + 1.0) * 127.5).astype(np.uint8))\n",
    "    plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_image = 8\n",
    "fig = plt.figure(figsize = (16,8))\n",
    "for i in range(num_image):\n",
    "    latent_z = mx.nd.random_normal(0, 1, shape=(1, latent_z_size, 1, 1), ctx=ctx)\n",
    "    img = netG(latent_z)\n",
    "    plt.subplot(2,4,i+1)\n",
    "    visualize(img[0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}