~dvshkn/experiment-cifar10-training-order

340456dee5116812cdeb0c907ffae1e380b80740 — David Knight 1 year, 14 days ago
initial commit
63 files changed, 942 insertions(+), 0 deletions(-)

A .gitignore
A cifar10_training_order.ipynb
A cifar10_training_order_test_only.ipynb
A plots/cifar10_training_order_accuracies-entropy-global-asc-all.png
A plots/cifar10_training_order_accuracies-entropy-global-desc-all.png
A plots/cifar10_training_order_accuracies-entropy-grouped-asc-all.png
A plots/cifar10_training_order_accuracies-entropy-grouped-desc-all.png
A plots/cifar10_training_order_accuracies-random-all.png
A plots/cifar10_training_order_losses-entropy-global-asc-all.png
A plots/cifar10_training_order_losses-entropy-global-desc-all.png
A plots/cifar10_training_order_losses-entropy-grouped-asc-all.png
A plots/cifar10_training_order_losses-entropy-grouped-desc-all.png
A plots/cifar10_training_order_losses-random-all.png
A training/cifar10_training_order-entropy-global-asc-0.1.net
A training/cifar10_training_order-entropy-global-asc-0.2.net
A training/cifar10_training_order-entropy-global-asc-0.3.net
A training/cifar10_training_order-entropy-global-asc-0.4.net
A training/cifar10_training_order-entropy-global-asc-0.5.net
A training/cifar10_training_order-entropy-global-asc-0.6.net
A training/cifar10_training_order-entropy-global-asc-0.7.net
A training/cifar10_training_order-entropy-global-asc-0.8.net
A training/cifar10_training_order-entropy-global-asc-0.9.net
A training/cifar10_training_order-entropy-global-asc-1.net
A training/cifar10_training_order-entropy-global-desc-0.1.net
A training/cifar10_training_order-entropy-global-desc-0.2.net
A training/cifar10_training_order-entropy-global-desc-0.3.net
A training/cifar10_training_order-entropy-global-desc-0.4.net
A training/cifar10_training_order-entropy-global-desc-0.5.net
A training/cifar10_training_order-entropy-global-desc-0.6.net
A training/cifar10_training_order-entropy-global-desc-0.7.net
A training/cifar10_training_order-entropy-global-desc-0.8.net
A training/cifar10_training_order-entropy-global-desc-0.9.net
A training/cifar10_training_order-entropy-global-desc-1.net
A training/cifar10_training_order-entropy-grouped-asc-0.1.net
A training/cifar10_training_order-entropy-grouped-asc-0.2.net
A training/cifar10_training_order-entropy-grouped-asc-0.3.net
A training/cifar10_training_order-entropy-grouped-asc-0.4.net
A training/cifar10_training_order-entropy-grouped-asc-0.5.net
A training/cifar10_training_order-entropy-grouped-asc-0.6.net
A training/cifar10_training_order-entropy-grouped-asc-0.7.net
A training/cifar10_training_order-entropy-grouped-asc-0.8.net
A training/cifar10_training_order-entropy-grouped-asc-0.9.net
A training/cifar10_training_order-entropy-grouped-asc-1.net
A training/cifar10_training_order-entropy-grouped-desc-0.1.net
A training/cifar10_training_order-entropy-grouped-desc-0.2.net
A training/cifar10_training_order-entropy-grouped-desc-0.3.net
A training/cifar10_training_order-entropy-grouped-desc-0.4.net
A training/cifar10_training_order-entropy-grouped-desc-0.5.net
A training/cifar10_training_order-entropy-grouped-desc-0.6.net
A training/cifar10_training_order-entropy-grouped-desc-0.7.net
A training/cifar10_training_order-entropy-grouped-desc-0.8.net
A training/cifar10_training_order-entropy-grouped-desc-0.9.net
A training/cifar10_training_order-entropy-grouped-desc-1.net
A training/cifar10_training_order-random-0.1.net
A training/cifar10_training_order-random-0.2.net
A training/cifar10_training_order-random-0.3.net
A training/cifar10_training_order-random-0.4.net
A training/cifar10_training_order-random-0.5.net
A training/cifar10_training_order-random-0.6.net
A training/cifar10_training_order-random-0.7.net
A training/cifar10_training_order-random-0.8.net
A training/cifar10_training_order-random-0.9.net
A training/cifar10_training_order-random-1.net
A  => .gitignore +3 -0
@@ 1,3 @@
.DS_Store
.ipynb_checkpoints/
data/

A  => cifar10_training_order.ipynb +498 -0
@@ 1,498 @@
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment: CIFAR-10 Training Order\n",
    "## To what extent does training order affect image classifier performance?\n",
    "[Code is based on this PyTorch tutorial!](https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The various constants below control the run mode of the experiment.\n",
    "- `ENTROPY_SORT_ENABLED`: This enables or disables entropy sorting of training data. No entropy sorting is the control case and makes this notebook equivalent to the original PyTorch tutorial.\n",
    "- `SORT_GROUPED`: If true this causes each CIFAR-10 image class to be entropy sorted independently. If false this applies entropy sorting globally over all training data. This constant is ignored if `ENTROPY_SORT_ENABLED` is false.\n",
    "- `SORT_ASCENDING`: Entropy sorting is applied in ascending order if true or descending order if false. This constant is ignored if `ENTROPY_SORT_ENABLED` is false.\n",
    "- `TRAINING_SLICE_FACTORS`: For every number (0,1] in this array a net will be trained using that fraction of the CIFAR-10 training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Experiment Controls\n",
    "#\n",
    "\n",
    "ENTROPY_SORT_ENABLED = True\n",
    "SORT_GROUPED = True\n",
    "SORT_ASCENDING = True\n",
    "TRAINING_SLICE_FACTORS = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1]\n",
    "\n",
    "def get_file_suffix(slice_factor):\n",
    "    if ENTROPY_SORT_ENABLED:\n",
    "        grouped_str = 'grouped' if SORT_GROUPED else 'global'\n",
    "        ascending_str = 'asc' if SORT_ASCENDING else 'desc'\n",
    "        return f'entropy-{grouped_str}-{ascending_str}-{slice_factor}'\n",
    "    else:\n",
    "        return f'random-{slice_factor}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import copy\n",
    "\n",
    "class EntropySampler(torch.utils.data.Sampler):\n",
    "    def entropy(self, datum):\n",
    "        arr = datum[0].numpy()\n",
    "        # grayscale coefficients from matlab rgb2gray\n",
    "        arr = 0.2989 * arr[0,:,:] + 0.5870 * arr[1,:,:] + 0.1140 * arr[2,:,:]\n",
    "        hist = np.histogram(arr, bins=256)[0]\n",
    "        hist = hist / np.sum(hist)\n",
    "        hist = hist[hist > 0]\n",
    "        return -np.sum(hist * np.log2(hist))\n",
    "\n",
    "    def __init__(self, data_source, global_sort=False, ascending=True, slice_factor=1):\n",
    "        # some CIFAR-10 magic numbers...\n",
    "        if global_sort:\n",
    "            num_classes = 1\n",
    "            images_per_class = 50000\n",
    "        else:\n",
    "            num_classes = 10\n",
    "            images_per_class = 5000        \n",
    "        \n",
    "        entropies_by_label = {}\n",
    "        source_idx_by_label = {}\n",
    "        pointers_by_label = {}\n",
    "        sorted_idx_by_label = {}\n",
    "        sort_dir_coeff = 1 if ascending else -1\n",
    "                \n",
    "        for i in range(num_classes):\n",
    "            entropies_by_label[i] = np.zeros(images_per_class)\n",
    "            source_idx_by_label[i] = np.zeros(images_per_class)\n",
    "            pointers_by_label[i] = 0\n",
    "        \n",
    "        for i in range(len(data_source)):\n",
    "            if global_sort:\n",
    "                label = 0\n",
    "            else:\n",
    "                label = data_source[i][1]\n",
    "            idx = pointers_by_label[label]\n",
    "            entropies_by_label[label][idx] = sort_dir_coeff * self.entropy(data_source[i])\n",
    "            source_idx_by_label[label][idx] = i\n",
    "            pointers_by_label[label] += 1\n",
    "        \n",
    "        for i in range(num_classes):\n",
    "            sorted_idx_by_label[i] = np.argsort(entropies_by_label[i])\n",
    "        \n",
    "        sliced_images_per_class = int(slice_factor * images_per_class)\n",
    "        \n",
    "        self.grouped_entropy_idx = np.zeros(sliced_images_per_class * num_classes, dtype=int)\n",
    "\n",
    "        for i in range(sliced_images_per_class):\n",
    "            for j in range(num_classes):\n",
    "                sorted_idx = sorted_idx_by_label[j][i]\n",
    "                source_idx = source_idx_by_label[j][sorted_idx]\n",
    "                self.grouped_entropy_idx[i*num_classes + j] = source_idx\n",
    "    \n",
    "    def create_sliced_copy(self, relative_slice_factor=1):\n",
    "        # because the array round-robbins the categories in group sort mode\n",
    "        # we can truncate it the same way as if it was globally sorted\n",
    "        dupe = copy.copy(self)\n",
    "        end_idx = int(relative_slice_factor * len(dupe.grouped_entropy_idx))\n",
    "        dupe.grouped_entropy_idx = dupe.grouped_entropy_idx[:end_idx]\n",
    "        return dupe\n",
    "    \n",
    "    def __iter__(self):\n",
    "        return iter(self.grouped_entropy_idx)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.grouped_entropy_idx)\n",
    "\n",
    "class ShuffleSampler(torch.utils.data.Sampler):\n",
    "    def __init__(self, length):\n",
    "        self.length = length\n",
    "        self.shuffled_idx = list(range(length))\n",
    "        np.random.shuffle(self.shuffled_idx)\n",
    "    \n",
    "    def __iter__(self):\n",
    "        return iter(self.shuffled_idx)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# Data Prep\n",
    "#\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5,0.5,0.5), (0.5,0.5,0.5))\n",
    "])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(\n",
    "    root='./data',\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=transform\n",
    ")\n",
    "\n",
    "trainloaders = []\n",
    "if ENTROPY_SORT_ENABLED:\n",
    "    base_entropy_sampler = EntropySampler(trainset, not SORT_GROUPED, SORT_ASCENDING, 1)\n",
    "    for slice_factor in TRAINING_SLICE_FACTORS:\n",
    "        entropy_sampler = base_entropy_sampler.create_sliced_copy(slice_factor)\n",
    "        trainloaders.append(torch.utils.data.DataLoader(\n",
    "            trainset,\n",
    "            batch_size=4,\n",
    "            drop_last=True,\n",
    "            num_workers=2,\n",
    "            sampler=entropy_sampler\n",
    "        ))\n",
    "else:\n",
    "    for slice_factor in TRAINING_SLICE_FACTORS:\n",
    "        trainset_length = int(slice_factor * len(trainset))\n",
    "        shuffle_sampler = ShuffleSampler(trainset_length)\n",
    "        trainloaders.append(torch.utils.data.DataLoader(\n",
    "            trainset,\n",
    "            batch_size=4,\n",
    "            drop_last=True,\n",
    "            num_workers=2,\n",
    "            sampler=shuffle_sampler\n",
    "        ))\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(\n",
    "    root='./data',\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=transform\n",
    ")\n",
    "\n",
    "testloader = torch.utils.data.DataLoader(\n",
    "    testset,\n",
    "    batch_size=4,\n",
    "    shuffle=False,\n",
    "    num_workers=2\n",
    ")\n",
    "\n",
    "classes = (\n",
    "    'plane',\n",
    "    'car',\n",
    "    'bird',\n",
    "    'cat',\n",
    "    'deer',\n",
    "    'dog',\n",
    "    'frog',\n",
    "    'horse',\n",
    "    'ship',\n",
    "    'truck'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAB5CAYAAAAgYXpDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAZ80lEQVR4nO2daZBcV3WAv9M9PT2bpNFujSRbEsiyvGorWyDAu/GGTCiTmFCgVFyl/CCFnVAJJvzAqsoPqKQgSRUhEWAsHC+xjYNkA7aFYgdwsGOxWNgyRvIieaSRZK2jmdFML3Pz49737p2ZHk3P1tNPOp+q1W/Off3efUvfPu+cc88RYwyKoihK8khNdAcURVGUkaEDuKIoSkLRAVxRFCWh6ACuKIqSUHQAVxRFSSg6gCuKoiSUUQ3gInKjiLwhIrtF5J6x6pSiKIoyNDLSOHARSQN/AK4HWoGXgU8ZY3aOXfcURVGUwagZxWcvB3YbY94CEJFHgNuAQQfwhoYG09zcPIpdKoqinH20tbUdNsbM7C8fzQA+F3g3+LsVuOJ0H2hubmb9+vWj2KWiKMrZx4YNG/aUko/GBi4lZAPsMSKyXkS2i8j2rq6uUexOURRFCRnNAN4KzA/+ngfs77+SMWajMWaVMWZVQ0PDKHanKIqihIxmAH8ZWCwiC0WkFrgD2DI23VIURVGGYsQ2cGNMQUT+EngGSAP3GWNeG+527r333pF24aym/3nT8zgySp03PZcjo/95W7t2TrxcLPa6pVKWVyXkRz86UPa6o3FiYoz5MfDj0WxDURRFGRmjGsDHkjAeXUR/pUdDdC71PCoTSSpVykKr9+RYolPpFUVREooO4IqiKAmlakwo+rg/dui5VKqB0Cza2xstV3MJx/B7U8399KgGriiKklCqRgNXJ6ainGn473HyvtPJ6K9q4IqiKAlFB3BFUZSEogO4oijjjjH2pYwtOoAriqIklKpxYiqKcuaSOB9mQlANXFEUJaHoAK4oipJQ1ISiKErM2ZMILTg+k3ISr88Wi87jKkUAUunq9MCqBq4oipJQhhzAReQ+ETkkIq8GsmkislVEdrn3qePbTUVRRsLpw/eMf7kVRWRY2rcxJn4lCTESvFL2RTZ+FfMZivkMhbyhkDeI4F/uXzVQjgZ+P3BjP9k9wDZjzGJgm/tbURRFqSBD2sCNMT8TkQX9xLcBV7nlTcDzwBfHsF+KoowBxhQAOPzee7EsX7Dacn19fSyrq8sCcPDgQQBqajJxW7Fo7cDnnHNOLKutrR2nHleKXr8oaftG2rc6G7gpoeKaKspUOFIb+GxjTBuAe581dl1SFEVRymHcnZgisl5EtovI9q6urvHenaIoylnDSMMID4rIHGNMm4jMAQ4NtqIxZiOwEaClpWVYzx6lUsxGMhM+Abm2vr4Xt174qBTjHpmC3vS6nzIJ1o/bT+vUGb0zo5QDqJTszA/tUsaaQu4UAK1734plnTl7s09pmhzLfvni/wCwZ89eAD72sY/HbceOHQNg0qRJsSybtSaXpDkvY8IoQldsolj03/1cjzUb1dRFK/rjrKZQy5Fq4FuAdW55HbB5bLqjKIqilMuQGriIPIx1WM4QkVbgK8BXgUdF5E5gL/DJsexUb6/9JczlemJZd7ddrq9vACCbDZwosabcp+fu/zSDEijnvU6lrwm3IQN/3/prHBKq8afRxsvVVPzTxMT/uivJJLzTamqto3Lxkotj2Su/2wnAO3v3xLInn3ocgI/d+gkArrhidbAV+93oWyLNypJ6nxrj+x09zRcLhVgWOW5LD5DVc8zlRKF8apCma8e4L4qiKMow0JmYiqIoCaVqcqGEj2KHDx8G4Nvf3hjLnn/eOlkuvXQ5AJ9dty5uu/SSCwdsrydvH4eOdXTEss7ubgDmTJ8OQEOtj3WtKfFbdqrLOoCyQbxsaoSPjOU+ap46Zff55ptv+r7V2MvU0tIyon0rZxd97rSUvXeMeFNiZCp471BbLJs6tRmAKVOsY7O3t+g3kUr1eYfqcuSNhJRk4+VcwdpQ2tv9WJHLWVnjlDon8eejmg5ZNXBFUZSEUjUaeOggaWxsBGDZZcti2e9ffwOARx57FICntm6N26686moA6hp8mNN+p8UfOHoilnV0W+12cctsAC563/viNslaDWXfW2/Hsn3vWCfPtBnTY9ny5bZPM2fZuUttre/Gbe+8bUO1up2mD7Bo0SK7z8WLY9l5551n++OeDn7+85/HbS+88ALQVwOPtrF+/XqqjTiss4RGVkqWVI0tqYhzadakBzrz2w60xsuzZ80DoKXFvqdLrN9nu4m9ji5IwPin7/YTxwHoduMD+Jmo0UNHtQZLqgauKIqSUHQAVxRFSShVaUJpaLCx3jffckssu/KqqwB46plnAfj3730vbrvvoUcAKBSD2PBGZ06p92YVcbHjv93xGgCPtfv5R+lJ1llRbD/pt3HyyIB+PrDJ/ualM/YRq5grLz1ANHMN/Iy2yNQSxZwCXHihdciuC5y0115rIzaXLFkSy3bu3FnWfseb/nHrQ8W7D9f5dbrtJfcxvnJEpy9MPjVzpjUJhilRJ0+aAkBz87TKdW6ciY4vvINS4oY8489HttaabNNpPxwWCnbeSa4nD0B9xptcKDW720Qmp2BvUui3Uqgvj41RRjVwRVGUhFI1Gngp55cJEp5Ejs0//sQfAXD5FZfHbU/+xDo0H938dCx7Zbd1Rp6q8Zpvymkh2Uk2VErqGv0+XemkyY0+ZPAY9lc4VeMdOtGvesdJq6nX1vn1TdH+4ubz+QHH1dTUFMsiTfrqq63z9SMf+UjctmyZc5LOnEmlGVJ7dlpDGEp5vN06iR988EHAh6MB3H67naAbOsTSMtA51n+vPT1+Bm5d8ORSDkkPbxsvwhDACy5YCsD5S86PZW/t3g1AY2MT/UncOXXjhrgQylQ4G9tYWb7bjy31dfaJOJ3248HRozb97qlOe+y1WT9U1tTYsSL8uhQKtj08RamaXF+ZCfohqoEriqKc1egAriiKklCqxoQS4h1jJWJRXRKdhXPnxqLP3/lnAFy75gOx7IEnrIPysWeei2V79+8HoN45M+sDM0XHYTsrrT7tHQ8dzm/RNNk/VjZPsSaCY8eOApBJ+T5mM3a75557bixbuXIlAFc5JyzAihUrAJg9e/bA43NMRJrOUul7QyLR0fajsSwynWzebM/3XXffHbelXGaw7h4fX/vKjh0AzJzha4DMnWvPV3ROH3roobjtwx9cA8Cs4FpFDrnIrAZQcLMLIwdxXV0dSmmiazvnHD+z90CbjQmfPHlyifUr06+xI0o5bd/TQXWhfI8dP06e9PNDTK89wFTaH2htrb1/enLWHNrd5ceFbJ3dbqHgzTDG5aOWoHp9bHWJTSh9MuX16etIUQ1cURQloVSlBn46xDlj+miobvGiC3yY3Ya/+WsAbr3+mlj2fRdu+PQ2q5WfOLgvbpvcYJ1l9Rn/mzZv3nwApk7zoVWTmqyGt2K5zcmyNAjtW7ncatYXXXRRLJszZw5Qembb6Yo2TITDKOxP5IgNQxzfbbVa2k+e9c7iF/7Xzhy94aM3AbBi2cq47eBBOxv2yBEfjrnp/gcAmD59Rixbs2ZNn35kgpCt91wtR9PrtZ3mZvsU1BvIOjs7+xyDauCW091HixYuipdzrvBDc/PUUlsZ626NA4NrspGGDZDL2/u513hH+ckOe+90dviQ4EWL7Mzpnp7cgLZCIaqhGWjszteeTpd4io2+V30cl+rEVBRFOatJnAYeUUqziGxeAFmXwe9Dl6+KZSsuugCAH19jw/cijRxg+0svAlA322uGCxfYX+GLL/bJ8KPlSy65BIBznZYOfSfr9CfUFidCy44007AfkZYdTSgK26L1w5DIKLzv/PP9U0dLi/VFRE8rkYYD/qnj+PH2WHbllVcBPlwSvN06yroYauT17pxmgkkWYZ8idu3aBcAOZ2O/7rrrBqyjWKLvyaxZvsp8tJyuSeyQMABxBVnSKX9MpzrtvdjZ5SfsRZkXw+9j5GfJ5ex3I9fp7+tij9tuOtB/a11BmEwpO3fE2Pu1htTARWS+iDwnIq+LyGsicpeTTxORrSKyy72XevZSFEVRxolyTCgF4AvGmKXAauBzInIhcA+wzRizGNjm/lYURVEqRDkl1dqANrd8UkReB+YCt2FrZQJsAp4HvjguvRwK96QiQRhQ9LQSOr8aGmw44O1rbY6Vq4Oww60/3QZAJsgZceFSayo4d743k4Sha9DX8Xc6p2Q4E65SlAoL3O9CKQHedLPvFixcOOCzx4/bFJthqOPkyTZfxso5PoSTfjlQwn1Gx3zeuQti2ezZ1qnb1NTQfxOxiSY0kew5bB2gDYFTcpILdYuqpQPsdscybdqZk8tj3Ii+L8E9mZhZlmXQ33dojD/OYtE29nT7sMAodHJSkx8OI/NfOmU3Up/1jvWuUznX5gMTGifZcUNS3tRSFSaUEBFZACwHXgJmu8E9GuRnDfKZ9SKyXUS2d3WVl/hJURRFGZqyPRYi0gT8ALjbGNM+jGxyG4GNAC0tLeM8OyXQhiVWM2JZ7ON0Wvn06V5bu+NPPlnWHuJq3JRwRFaZEhP27ZAL5fvhlidjWVubnby0dKl17j652WdnPHjwINA3T8vCRbYARmeP1168luMql5fQMlKBppLLW+2lq6szltU4Z1BDvdXKozwzAM/91Oa5qc34W/W6668HoDfY14lj9olh/jxbkCB0kpaanKL0J3pSjb4vVXYzDwuXuyX+joal4Oy9OGWynxjW2Njg2vz9FN2nBw7agi31QYBCWuz6NcETf9YVhCmSC/rR/1yG342xOb9laeAiksEO3g8aY55w4oMiMse1zwEOjUmPFEVRlLIoJwpFgO8Crxtjvh40bQGipNXrgM39P6soiqKMH+WYUNYAnwF+JyK/dbK/A74KPCoidwJ7gfJsEOOKlFgKWiOhDPzdKrdwwEQ4I4dLqfSfv37FxkdvefqZWBbV5nz5oYcBOHrwQNx2sZtN+uwzfv3Vaz4MQCG4bU6027jaXM4+OuaLgXlFBvajUHAz4QLn8hRn4pjhigm8+MIv47b3L7QmkY6Tx2PZAw/+BwCXrlzh++GcnTtfs8U6brjhhrgtMqEMlevlbGEiZ/tWgugyRwUawpmYvUV7302e5M2nTc6E0t3tq9J3d1t/3QFXNzSch7B0iZ2FHQ4jubydyRqG0VcinVE5USi/YHCDzbVj2x1FURSlXM6caVej5EzVRiJazrHhgEuXLI5lUbX7t3e9AcD5i31bpJ3v2+fzxexrtQ6dVVd8MJZNcZkaoxDAQpA7Jee08UKglUead/gkM8kVu8ifcvkpen0YYRTC2X7Sl8Y7fNyGDy5wfQRIzbcZDae7XB5RvhTlLCHIMxJ9k9NiQ367O/w92dNlsxB24O/JlNh768hR78bL99jPNGTtvdlbCO5hY5e7On2WzWLahhFOqffhsXEKFCnlxNRcKIqiKGc1OoAriqIkFDWhnIGUMgctWfx+AD73F+tjWZTE6ohzXr70yxfjtsjxt2fPnlg2y83KXLn8slhm+plETMrvO+/MKfm8j42NZlnmC2EsuX2cPHrIpp/d/sIv4rbWd+3+O075uPGoOMbtn/R+8xmu0MZUV+80WyKd7JluJlMczrtYzNvr3RtMjsznrHPy2PG2WPZuq70ne7p9itlZs6xJrnmKdXb2Fvw9fKLdOtRzBb/hTINzmPYp2tDfTHK6tpGhGriiKEpCUQ38jMb/ymfcbMe5c3xuk337rYNyzQdWA5AO4qIiTXnt2rWx7AJXvCIflEiLtVqXO0ICDSTjtPLaeq8N99banBKhBl5w+6qdZVP53nLzjXHbiROu9FWQujNK6RtWr4+eAKKUvkkI91TGkD6THO21z+edYz3QlIvOQd7ReTiW1bjSf42N3lE+w+XUOXSgzW3D32s9HXYb6dr6WJZzhR/CAiiliriMNXqXK4qiJBQdwBVFURKKmlDOaIKZqW6xvt4/9i1csACABefZ9yuvvDpui6q8h/Upowok4SzK/pgSCXtC32Gc/CpMPBYJXbaxW2+6KW7L520/Qt9QKpruFphVaqOZcv3S21qROi/PJsTppT2uzmd34Jw0Loa7o8PXaZ082cZup1I+VXS3c5q3t9s5B/mCNxtmsna9SVm/fmw6qcDsyxDVwBVFURKKauBnCSL2Uod+lXS6dpC1qwetLa+Uh3/Kih6+urpsbpOuzq6gzWrKxbzXytv2W208NzWoW1u0T6pRgYaenN9GQ5MtbJJKDXyyq/TTnmrgiqIoCUU1cEVRzgBCR4vVS6PQv65T7XFTzmnetRn/bNfZYdtPBUVGco2dbn072S2V8sbtdI3TewNtOx37YEZ1EMNGNXBFUZSEogO4oihKQhnShCIidcDPgKxb/3FjzFdEZCHwCDAN+DXwGWNMbvAtKYqijBdhyKzVSyNTR7Hoh6WiC4+tr/OzLvMNdmZlXdaH2EY1Qnu6rSklWx/UxEwPtJPU1GQGyEoVVhlrytHAe4BrjDGXAcuAG0VkNfA14BvGmMXAMeDOceuloiiKMoByKvIYIKo1lHEvA1wD/KmTbwLuBb419l1UlMpSqryeTgYaAWNfv+A0uwp2IDb0L5u1WnGoHfe4qvTplNe2G+ptWGBDgy/GUChYDTxXcNp5yrdFGn7g1yRTY7eb6lOuMZq1FlV2GNYhlUW5VenTrh7mIWAr8CZw3ETTmqAVmDvIZ9eLyHYR2d7V1VVqFUVRFGUElDWAG2OKxphlwDzgcmBpqdUG+exGY8wqY8yq8BdOURRFGR3DigM3xhwXkeeB1UCziNQ4LXwesH8c+qcoSkIJJyrG2t04WaL6bDYVmVCcWSPlpx+nU9YZGaQEIpuNZiT7rUTWgrTLuyNB3h2JTSM+dWxtJtpXkOOnt59Oa8JF0180IobUwEVkpog0u+V64DrgdeA54Ha32jpg8yj7oiiKogyDcjTwOcAmEUljB/xHjTFPichO4BER+XvgN8B3x7Gfyig5XQbBs4XQEXk6p6Q6LMcGE1R+Ny4sT8ZtqmKo+dr3TK3VT+vr/TCXsxMrEeN115z1U9LT7TMOptxH6jJWY5dAs47aUulAlrY79W7B/pk5IdTwe02UqbO8e3IwyolC2QEsLyF/C2sPVxRFUSYAnYmpKIqSUDSZ1VmC1ogsnyNHbHrRpqamWBbV2lTKZ39ba7zcW3R2jTExT4XeQLc9GRh0nsI6FnuC1LGnum3iqrxLUgWQc3aVQlCnNTJniPN/FsjHbZkuW6e1NuNlh4/Yz9Z0DqxeknI5nGszPn1zTcZ6UTOBbCQ1NPVbrSiKklCk1Kyz8aKlpcWsX7++YvtTFEU5E9iwYcOvjDGr+stVA1cURUkoOoAriqIkFB3AFUVREooO4IqiKAmlok5MEXkP6AQOV2yn48MMkn0MSe8/JP8Ykt5/SP4xJKn/5xljZvYXVnQABxCR7aW8qUki6ceQ9P5D8o8h6f2H5B9D0vsPakJRFEVJLDqAK4qiJJSJGMA3TsA+x5qkH0PS+w/JP4ak9x+SfwxJ73/lbeCKoijK2KAmFEVRlIRS0QFcRG4UkTdEZLeI3FPJfY8EEZkvIs+JyOsi8pqI3OXk00Rkq4jscu9TJ7qvp8MVpf6NiDzl/l4oIi+5/v+niNQOtY2JRESaReRxEfm9uxYfSOA1+Ct3D70qIg+LSF01XwcRuU9EDonIq4Gs5DkXy7+47/UOEVkxcT33DHIM/+Duox0i8l9RtTHX9iV3DG+IyEcnptfDo2IDuKvo803gJuBC4FMicmGl9j9CCsAXjDFLsXVAP+f6fA+wzRizGNjm/q5m7sKWwYv4GvAN1/9jwJ0T0qvy+WfgaWPMBcBl2GNJzDUQkbnA54FVxpiLgTRwB9V9He4HbuwnG+yc3wQsdq/1wLcq1MehuJ+Bx7AVuNgYcynwB+BLAO57fQdwkfvMv7oxq6qppAZ+ObDbGPOWMSYHPALcVsH9DxtjTJsx5tdu+SR24JiL7fcmt9om4OMT08OhEZF5wC3Ad9zfAlwDPO5Wqfb+TwY+givZZ4zJGWOOk6Br4KgB6kWkBmgA2qji62CM+RlwtJ94sHN+G/B9Y3kRW/B8TmV6OjiljsEY86zxdc9exBZkB3sMjxhjeowxbwO7SUDFsUoO4HOBd4O/W50sEYjIAmxpuZeA2caYNrCDPDBr4no2JP8E/C0QFcWcDhwPbuJqvw6LgPeA7zkz0HdEpJEEXQNjzD7gH4G92IH7BPArknUdYPBzntTv9p8DP3HLiTyGSg7gpUpxJCIERkSagB8Adxtj2ie6P+UiIrcCh4wxvwrFJVat5utQA6wAvmWMWY5NxVC15pJSOFvxbcBCoAVoxJod+lPN1+F0JO2eQkS+jDWRPhiJSqxW1ccAlR3AW4H5wd/zgP0V3P+IEJEMdvB+0BjzhBMfjB4R3fuhierfEKwB1orIO1iT1TVYjbzZPcpD9V+HVqDVGPOS+/tx7ICelGsAcB3wtjHmPWNMHngC+CDJug4w+DlP1HdbRNYBtwKfNj6OOlHHEFHJAfxlYLHzvNdiHQZbKrj/YePsxd8FXjfGfD1o2gKsc8vrgM2V7ls5GGO+ZIyZZ4xZgD3f/22M+TTwHHC7W61q+w9gjDkAvCsiS5zoWmAnCbkGjr3AahFpcPdUdAyJuQ6Owc75FuCzLhplNXAiMrVUGyJyI/BFYK0xpito2gLcISJZEVmIdcj+30T0cVgYYyr2Am7Gen7fBL5cyX2PsL8fwj5G7QB+6143Y+3I24Bd7n3aRPe1jGO5CnjKLS/C3py7gceA7ET3b4i+LwO2u+vwQ2Bq0q4BsAH4PfAq8ACQrebrADyMtdfnsdrpnYOdc6z54Zvue/07bLRNtR7DbqytO/o+/1uw/pfdMbwB3DTR/S/npTMxFUVREorOxFQURUkoOoAriqIkFB3AFUVREooO4IqiKAlFB3BFUZSEogO4oihKQtEBXFEUJaHoAK4oipJQ/h/EnZ0o0ablPgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "plane   car  bird   cat\n"
     ]
    }
   ],
   "source": [
    "def imshow(img):\n",
    "    img = img / 2 + 0.5\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1,2,0)))\n",
    "    plt.show()\n",
    "\n",
    "dataiter = iter(trainloaders[0])\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "print(' '.join('%5s' % classes[labels[j]] for j in range(4)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `imshow` output above can be used as a quick sanity check that the entropy sorting is behaving. Entropy sorting in ascending order should cause 4 relatively low information images to be shown. Entropy sorting in descending order should cause 4 relatively high information (i.e. busy looking) images to be shown. If grouped sorting is enabled each image should be from a different class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Network Def\n",
    "#\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 16 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Training\n",
    "#\n",
    "slice_factor_idx = 0\n",
    "loss_curves = []\n",
    "loss_labels = []\n",
    "for trainloader in trainloaders:\n",
    "    net = Net()\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",
    "    slice_factor = TRAINING_SLICE_FACTORS[slice_factor_idx]\n",
    "    suffix = get_file_suffix(slice_factor)\n",
    "    running_period = 2000 * slice_factor\n",
    "    loss_samples = []\n",
    "    \n",
    "    print(f'START Training Time [{suffix}]')\n",
    "    \n",
    "    for epoch in range(2):\n",
    "        running_loss = 0.0\n",
    "    \n",
    "        for i, data in enumerate(trainloader, 0):\n",
    "            inputs, labels = data\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            running_loss += loss.item()\n",
    "            if i % running_period == running_period-1:\n",
    "                curr_loss = running_loss / running_period\n",
    "                loss_samples.append(curr_loss)\n",
    "                print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, curr_loss))\n",
    "                running_loss = 0.0\n",
    "    \n",
    "    \n",
    "    PATH = f'./training/cifar10_training_order-{suffix}.net'\n",
    "    torch.save(net.state_dict(), PATH)\n",
    "    \n",
    "    plt.plot(np.linspace(0, 2, len(loss_samples)), loss_samples)\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.title(f'Training Loss ({suffix})')\n",
    "    plt.show()\n",
    "\n",
    "    loss_curves.append(loss_samples)\n",
    "    loss_labels.append(suffix)\n",
    "    \n",
    "    print(f'END [{suffix}]')\n",
    "    \n",
    "    slice_factor_idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Plot Losses\n",
    "#\n",
    "for i in range(len(loss_labels)):\n",
    "    suffix = loss_labels[i]\n",
    "    loss_samples = loss_curves[i]\n",
    "    plt.plot(np.linspace(0, 2, len(loss_samples)), loss_samples, label=suffix)\n",
    "    \n",
    "all_suffix = get_file_suffix('all')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.ylim(1, 2.5)\n",
    "plt.title(f'Training Losses')\n",
    "plt.legend(bbox_to_anchor=(1.52, 1.0))\n",
    "plt.savefig(f'./plots/cifar10_training_order_losses-{all_suffix}.png', bbox_inches='tight', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Test Time\n",
    "#\n",
    "dataiter = iter(testloader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4)))\n",
    "\n",
    "accuracies = []\n",
    "\n",
    "for slice_factor in TRAINING_SLICE_FACTORS:\n",
    "    suffix = get_file_suffix(slice_factor)\n",
    "    print(f'START Test Time [{suffix}]')\n",
    "    \n",
    "    net = Net()\n",
    "    net_path = f'./training/cifar10_training_order-{suffix}.net'\n",
    "    net.load_state_dict(torch.load(net_path))\n",
    "\n",
    "    outputs = net(images)\n",
    "\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            images, labels = data\n",
    "            outputs = net(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "\n",
    "    accuracy = 100 * correct / total\n",
    "    print('Accuracy of the network on the 10000 test images: %d %%' % accuracy)\n",
    "    accuracies.append(accuracy)\n",
    "\n",
    "    class_correct = list(0. for i in range(10))\n",
    "    class_total = list(0. for i in range(10))\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            images, labels = data\n",
    "            outputs = net(images)\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            c = (predicted == labels).squeeze()\n",
    "            for i in range(4):\n",
    "                label = labels[i]\n",
    "                class_correct[label] += c[i].item()\n",
    "                class_total[label] += 1\n",
    "\n",
    "    for i in range(10):\n",
    "        print('Accuracy of %5s : %2d %%' % (classes[i], 100 * class_correct[i] / class_total[i]))\n",
    "        \n",
    "    print(f'END [{suffix}]')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Plot Accuracy vs. Training Set Size\n",
    "#\n",
    "training_set_sizes = np.array(TRAINING_SLICE_FACTORS) * 50000\n",
    "all_suffix = get_file_suffix('all')\n",
    "plt.plot(training_set_sizes, accuracies, '-o', label=all_suffix)\n",
    "plt.xlabel('Number of Training Images')\n",
    "plt.ylabel('Accuracy (%)')\n",
    "plt.ylim(20, 65)\n",
    "plt.title(f'Test Accuracy vs. Training Set Size')\n",
    "plt.legend(loc=4)\n",
    "plt.savefig(f'./plots/cifar10_training_order_accuracies-{all_suffix}.png', bbox_inches='tight', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

A  => cifar10_training_order_test_only.ipynb +441 -0
@@ 1,441 @@
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Experiment Controls\n",
    "#\n",
    "\n",
    "ENTROPY_SORT_ENABLED = True\n",
    "SORT_GROUPED = True\n",
    "SORT_ASCENDING = False\n",
    "TRAINING_SLICE_FACTORS = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1]\n",
    "\n",
    "def get_file_suffix(slice_factor):\n",
    "    if ENTROPY_SORT_ENABLED:\n",
    "        grouped_str = 'grouped' if SORT_GROUPED else 'global'\n",
    "        ascending_str = 'asc' if SORT_ASCENDING else 'desc'\n",
    "        return f'entropy-{grouped_str}-{ascending_str}-{slice_factor}'\n",
    "    else:\n",
    "        return f'random-{slice_factor}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import copy\n",
    "\n",
    "class EntropySampler(torch.utils.data.Sampler):\n",
    "    def entropy(self, datum):\n",
    "        arr = datum[0].numpy()\n",
    "        # grayscale coefficients from matlab rgb2gray\n",
    "        arr = 0.2989 * arr[0,:,:] + 0.5870 * arr[1,:,:] + 0.1140 * arr[2,:,:]\n",
    "        hist = np.histogram(arr, bins=256)[0]\n",
    "        hist = hist / np.sum(hist)\n",
    "        hist = hist[hist > 0]\n",
    "        return -np.sum(hist * np.log2(hist))\n",
    "\n",
    "    def __init__(self, data_source, global_sort=False, ascending=True, slice_factor=1):\n",
    "        # some CIFAR-10 magic numbers...\n",
    "        if global_sort:\n",
    "            num_classes = 1\n",
    "            images_per_class = 50000\n",
    "        else:\n",
    "            num_classes = 10\n",
    "            images_per_class = 5000        \n",
    "        \n",
    "        entropies_by_label = {}\n",
    "        source_idx_by_label = {}\n",
    "        pointers_by_label = {}\n",
    "        sorted_idx_by_label = {}\n",
    "        sort_dir_coeff = 1 if ascending else -1\n",
    "                \n",
    "        for i in range(num_classes):\n",
    "            entropies_by_label[i] = np.zeros(images_per_class)\n",
    "            source_idx_by_label[i] = np.zeros(images_per_class)\n",
    "            pointers_by_label[i] = 0\n",
    "        \n",
    "        for i in range(len(data_source)):\n",
    "            if global_sort:\n",
    "                label = 0\n",
    "            else:\n",
    "                label = data_source[i][1]\n",
    "            idx = pointers_by_label[label]\n",
    "            entropies_by_label[label][idx] = sort_dir_coeff * self.entropy(data_source[i])\n",
    "            source_idx_by_label[label][idx] = i\n",
    "            pointers_by_label[label] += 1\n",
    "        \n",
    "        for i in range(num_classes):\n",
    "            sorted_idx_by_label[i] = np.argsort(entropies_by_label[i])\n",
    "        \n",
    "        sliced_images_per_class = int(slice_factor * images_per_class)\n",
    "        \n",
    "        self.grouped_entropy_idx = np.zeros(sliced_images_per_class * num_classes, dtype=int)\n",
    "\n",
    "        for i in range(sliced_images_per_class):\n",
    "            for j in range(num_classes):\n",
    "                sorted_idx = sorted_idx_by_label[j][i]\n",
    "                source_idx = source_idx_by_label[j][sorted_idx]\n",
    "                self.grouped_entropy_idx[i*num_classes + j] = source_idx\n",
    "    \n",
    "    def create_sliced_copy(self, relative_slice_factor=1):\n",
    "        # because the array round-robbins the categories in group sort mode\n",
    "        # we can truncate it the same way as if it was globally sorted\n",
    "        dupe = copy.copy(self)\n",
    "        end_idx = int(relative_slice_factor * len(dupe.grouped_entropy_idx))\n",
    "        dupe.grouped_entropy_idx = dupe.grouped_entropy_idx[:end_idx]\n",
    "        return dupe\n",
    "    \n",
    "    def __iter__(self):\n",
    "        return iter(self.grouped_entropy_idx)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.grouped_entropy_idx)\n",
    "\n",
    "class ShuffleSampler(torch.utils.data.Sampler):\n",
    "    def __init__(self, length):\n",
    "        self.length = length\n",
    "        self.shuffled_idx = list(range(length))\n",
    "        np.random.shuffle(self.shuffled_idx)\n",
    "    \n",
    "    def __iter__(self):\n",
    "        return iter(self.shuffled_idx)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "#\n",
    "# Data Prep\n",
    "#\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5,0.5,0.5), (0.5,0.5,0.5))\n",
    "])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(\n",
    "    root='./data',\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=transform\n",
    ")\n",
    "\n",
    "trainloaders = []\n",
    "if ENTROPY_SORT_ENABLED:\n",
    "    base_entropy_sampler = EntropySampler(trainset, not SORT_GROUPED, SORT_ASCENDING, 1)\n",
    "    for slice_factor in TRAINING_SLICE_FACTORS:\n",
    "        entropy_sampler = base_entropy_sampler.create_sliced_copy(slice_factor)\n",
    "        trainloaders.append(torch.utils.data.DataLoader(\n",
    "            trainset,\n",
    "            batch_size=4,\n",
    "            drop_last=True,\n",
    "            num_workers=2,\n",
    "            sampler=entropy_sampler\n",
    "        ))\n",
    "else:\n",
    "    for slice_factor in TRAINING_SLICE_FACTORS:\n",
    "        trainset_length = int(slice_factor * len(trainset))\n",
    "        shuffle_sampler = ShuffleSampler(trainset_length)\n",
    "        trainloaders.append(torch.utils.data.DataLoader(\n",
    "            trainset,\n",
    "            batch_size=4,\n",
    "            drop_last=True,\n",
    "            num_workers=2,\n",
    "            sampler=shuffle_sampler\n",
    "        ))\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(\n",
    "    root='./data',\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=transform\n",
    ")\n",
    "\n",
    "testloader = torch.utils.data.DataLoader(\n",
    "    testset,\n",
    "    batch_size=4,\n",
    "    shuffle=False,\n",
    "    num_workers=2\n",
    ")\n",
    "\n",
    "classes = (\n",
    "    'plane',\n",
    "    'car',\n",
    "    'bird',\n",
    "    'cat',\n",
    "    'deer',\n",
    "    'dog',\n",
    "    'frog',\n",
    "    'horse',\n",
    "    'ship',\n",
    "    'truck'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Network Def\n",
    "#\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 16 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAB5CAYAAAAgYXpDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO19aZAl2VXedzPz7a9e7V1d1XtPd88uzYxGIwmEEBLYIwESYQssTMCELcdEOFAYHEQYYX5gRfgHhB1gHIHlmEBCAhMIWRJIFjJGjHZgpOlZpZmeXqbX6q6u6tqr3v4yr3+cc/OcV0t39UJXP7hfREdl38yXee/Nm5nnnO8sxloLDw8PD4/eQ7DdHfDw8PDwuDH4F7iHh4dHj8K/wD08PDx6FP4F7uHh4dGj8C9wDw8Pjx6Ff4F7eHh49Chu6gVujHncGHPcGHPKGPORW9UpDw8PD49rw9yoH7gxJgRwAsCPAZgE8CyAn7XWvnrruufh4eHhsRmim/jtYwBOWWtPA4Ax5tMA3g9g0xd4sVi0AwMDN3FJDw8Pj398mJqamrXWjq5tv5kX+C4AF9T/JwG85Wo/GBgYwJNPPnkTl/Tw8PD4x4ePfvSj5zZqvxkbuNmgbZ09xhjzpDHmqDHmaK1Wu4nLeXh4eHho3MwLfBLAHvX/3QAurT3IWvuUtfZRa+2jxWLxJi7n4eHh4aFxMy/wZwEcNsYcMMZkAXwQwBdvTbc8PDw8PK6FG7aBW2s7xpgPA/h/AEIAn7DWvnK959m39AUAgLFJ2pbNULdMIN+XVqsJAOjEbTomm033xQn91iZiwTFBDAAIQtXndon2gfZlso10Xwh3TTlHnHQAAO2O9C1J2HJkIu6PWJKavE/blhIelzHS2mrRGOI4Wjf2gPvWSqStSt1ArRWnbaX7noDGhz/84XS70+msu+atwHWfz675q5sC3UatgWvUhjjj5i9Rx7t5lpNczZtqo3674z/2sY+t27fvh3lu407aNnflMgCg2ZA1c/CuQwCAgf4KACATSn+yGVp4Wd3G6zkyao116gCAcinD55C+RrwdqkW8sDAPAOjr60vbMpkMn5eOM4Gco5O0AADBBqJaYKSxViXzZhTRmszn8+m+VovO0eFnEAAK+QJfS/r2u7/9W13n371nR7pdHjlCvwvlua30lQEAK01Z19XlOe4v3e9ELYaIB1GIcmlbPuRXmHpu0weQm+JEzu/aEtXmruHGTtfnudxg7Ri+fybQ74V4g+Pot7kc9TcbSL9hadtkZf5qc8cAAF975vvrzrUZbobEhLX2ywC+fDPn8PDw8PC4MdzUC/xWoMVSlLV1aWTpM4dS2hSAvlRRxJK1lij4q2oy0th0UkMiX7iIJbyQmyJ1DpOQVIyOSBlOGk7UOVqGJJM4pC9oS++LAz6XfI0NS/F51beIJZ8goo7H7bbqSIeHJOdwEmcYbm7xCsNw0323Cjcq0ev5SOUkJSUmTmSyPAYr+5xGZCDSjpzl5iXwjVAu0r0NrDwezSq1JS0h4vNZOm+pQMdF6jJu7eTUIitk+b6rsTRjdxytq6xaJ26KokjurZPsAyXFu7nJsVaql0m11uZrCpz2aiHnDfhiGZZCnVQPAO1mk8enxsJSJa6yJhIrUnwnHKRzZeSZjkOSwIOMksDrq9S3uMr9kPM1LR3XVpJvg+dXCeVotUlLCviZqNfk3eKeEz0+pxEHgTyH1mkuPJla4+90Yj5GrmmMez/JmhkcpDHnCn18frlniVvXOelHvFrG9cKH0nt4eHj0KPwL3MPDw6NHse0mFMsmBlgxXVgmj0wsKl7SJpUmLLCZQqmhznqgiYQsq0gdKypK0g67jnOqEAAYu4ZIA2CYcLGhqIL1mHS1y3OkblVbohatrlJbaOW8fXkmsxQJVykSAVTI0TiToJXuC1JziYzdjaCdbK72a5PA31eZvK2ct8tc4Y7v0jXdLm3yoTlvtmk+Iq03x/Tb0Gx07WSDtq3hamOJ2IwVKDNWNqRrZQJpywVsHnP7FAHZrJOpJQwV4RbRfW83hQgNwCazDrVZI49kzKaibKYgx7t5UGvMkbkxmwF1vMXclSsAgLGRQTmezSVhVq4V8rXcPCtLDiI+vqlIXUewttvSthaBlX0x9zdWz0FsaMz5PunH8L4x+u3SAgCgXFtN97Ua9I6Iy/I8Jv0U2d2Xlbl31w3YztpqyvPlHB7yebkv6ZSqNeHWsfsbKJtth8ec6OXHl89GsnYLBSZ64cyAYqJJnHlWy9A3YKL0EriHh4dHj2LbJfAoZsk7lK9fwJJELlRfd8cQ8Zcw0EwN/7SjJVRHymRFetm5/24AwPLiLABgdk4klUxE0nYA+TK3OjQ9dSsBSMfOkURjc8MAgHYopEyLJYPVpfm07eI0SxJ5JVlNLQIA9u6kaw73aSnNuRbK2J1wEdv1rkoOWvK9Fe6Dt0SKT/uttAN2tewo8aXNmtDJ06cBAGM7xf0sYTJ6dEgkyDwTP8lN9PFqc5RlKTvpiOQWsvSUUQRahtuCmNZRNqOkupBdVZV2lQno3iZGaVwJu8c2mMxU66nBYy8WZQ2HjtnU4h/PQ5VdHJ977vl0V5s1gcHKm9O2XI7JfDUFqSsra6eBct8z1pH5siZt4oi8zSXwDsTVMQCt9SRUBC5rYaHSxkrMRlaKfI+ffzbd15olaXz8gbulb1fomWsambcyD2ylTkRoXo0lxxp5MCyEYcAkpn6lNIt03qjNmklbJmulRPclt7SUtkV77gMA1Ab607aEtaqY71k+ESI01fhjaQvj65envQTu4eHh0aPwL3APDw+PHsW2m1Ccnm0iSTPr1NuOjlBkwqjFam1WkUNx7NQ5ZWLgc2i/2rf86I8BAJ77278DAFxiUwoAVDsuslJUq3OTMwCAM5MX07bc4DgAYPfYAbpmTtTEFqt/mbJkfew0SO2bm5E0McVBMr9MrlJ0X0Opw2N9pOIVM6JWxm1Sg3Ww2Vr6biMS83ZEYl7d1MJkWUZFzbKPd31VSOvFJVJ1p2fJ9FToE3V4mCMOddSgI+10dOYGnV3Ti60jy+Y6q86RcZMfS79DOLKd2jLKr7rt1OdEzhFWaB6MVX7/7G+cuGjfWNb16jKZ2spFIe0Cnm8dFRlx5PIik5fzy2IaLLCfdEtZOlptulaU1WuG2mKOdO4o85GLgs4qH2fLazaJNzfr6Zl3JsFAjT3u8FiV7cKwiaNh6L5nElkLZoRMa7UV6Vv7zAnqrxEzU8LTVXX+5er5yrY5fuOCItF5PrRjRIPNoWGD50ouieZO6mP9sphK+ww986Z/RMbH120HjhhWsQ8836EixaPg+k2CXgL38PDw6FFsuwTeDOhLu1RTEVosvQyWRWyoMCkUsQSiCabUDUgRKo7krNUW0ravfonyrkwvkkQxvSrfr3MX6bhzlyTFeZgnaTwOK2lbqUJf2kyR9kV5+fLnWErMBzKW2RZFgY3v3pu2NZhcOX2aJPD5RZWTZRedd/+oaAIZdqUzyo1L5C8er/q62+T6ZM408HEDAUBL3cEGEnjMUlbC0oaOFnURblfmltO25SqNta7zX9RoNEGOyOJqXe5tucgSp+qbk+e3qmBcryaSM87lTebbkZcbugAmHPmnXAAj1hgjxRSGhubDxvru8fiYuI+Vq9nqCs3beX3NyEUui7S4p0Lz5lwGX3r55XTfG+6/HwCQaBfHmOY3r11sWROo11jDjeT8HdYAw0jI/Dbn22k2N08RHSvpPOE1bLXMyE4HLe1uyNftX+G5Gh1L9xV27KP+WCEPwa6QdmRn2lTPcG6Ty5RXBcolt8rPqx0bTtsyCfWpoTT4EmuBrRUaX1PnqClwxGtV7ks0TNqBySg3Sc530sc/DZWE3zE09yZQLrO4/mhqL4F7eHh49Cj8C9zDw8OjR7HtJpQrdVIb5ttCYn7jb74OALjviJgifuR+IgcG2V9ckycuaU2g1JGYyRLFfeHMOfIznq+TamOLQ+m+sMxk2ZCo+wWu39lSKURbTJxVBqlvlbL0ceYymUSWFxS5wSpeviCmlvMLRJ5mKqQezkxJtaTy5RUAwM6KHF9wqWsTRX6tQbWmk4GxCqlUR5dqN1SJkdy2S4+pckghSNZ/212UqLZdrLJ678jMgiK6GhyxNqVMKDMLtJ0ogqvN9pHaChG+M7Myf5MXpwAA9x0+mLbdtX839V/5xadkqouk1VYT120dJnAVajNkE17SFvNAwCa7+pKMBWw+sJwEKSzI2LN8r7Jqvk2bTGexNjtwtLFJiVMxH1WrZCqYnpbjS5UyX1Ml8uI5b63ScXnlj35lkYjQ578vZpVSjq556KDMacSmnGaN1l8hUomXmrS2YpVWOXaPWkPNx1qoKXYpXZOuWA3ep57lDJuvcqdO0umf+1a6r/NmNj2ptKyWYzSyK/JsNEDzUOZ4izAnxyclOr+xiljnZHJ9w/IOylxk88sqrcnMmDgr4ALtiypi5mxcofkNi9KWHCHf8AYnwgoU6Z7t0OREyjZor8LJbwYvgXt4eHj0KK4pgRtjPgHgJwDMWGsf4LYhAH8KYD+AswB+xlq7sNk5rtqBfpICanPyLWlniSicr6lk5y1y66lk2e1KER9O4gxDIVkaLZJgryi+aHaFvr7FASIwBkeFWKwmJEmMQEW9MeHRyohU1KiShNJYpeP3KTKkxtL2TEukYcPS0NK8krpYGqnz1z3MSr+nl2kap5ZE6t83whrGVb7Qi3UZaLlIWkGg8jK44hRdgrUjV1yQa1ca1w2+7Ru4J16eIhfLoSHSZgp5kWyaDRpzMSdtO0dJk7JKPKvWaKwlllRaDZX+kwe92pTxddI8FcqtLXVndPvWDbNLIrya92PeJexXBzkJPKek/jKTxf1MPgXsDgkAOb7HeS1wspYUNGQtpEn+uTBIa1nWWl+J9g0OiaZ4ZpK0vNMXLqdtJ049DQBYmCWJc7Uh56i1qcZKBOUWyJL9g3cfSdve9+OPAwB28Xpu5mWcjWqVfyfXrHCBdFNfwWbIhLL+XDpoR2YCklI1UnJkeYGu1Zkkt9uK0iZWLtH1W3mJdrSg94K5PJO2lSaYgKywZgl5lgrsvppdlH43mDjuzE6lbVmew84yzVVuXhwZ2nXWlgqiwSyeIeeHbEEk8L5xIl1dKiWrXAabjrxWa7iVXL8IvhUJ/JMAHl/T9hEAT1trDwN4mv/v4eHh4XEbcU0J3Fr7TWPM/jXN7wfwTt7+FICvA/jVG+nA3W94DAAw+czxtK3cT1/3x972lrStGJKduMUSsJYuDWdri63ky+jbQfWWX3z5pJx3gKS/XfvItcoqW1qGpeykOZe2tVrJumuF/MV85aWXAAAVlZC9WKIvf0nZwS5dngbQnaclZKliiN2/FhfEfrcwT9tnpsRVamKMXKSirIomWIOoIppAzNJzW9eTY9ti+hdil3TBIVritBv4FDoBXXkspgElLl8GlCvnALtitdvqXCyVFctiU3QSuOHgLKNctnIF526lyoQxsdFlM1zXN7lmpvsQ3r25CH7h7Fnut8z3yjKtu7gtmsDFi6R9LPAaqK6KPXjHMEnN5ZIE4YRcjKSlMvhFnKsn4Fw8VSWdN9xgVGGJ85eIPzkzKTxBtUW/zfezK1tJJsatxFJWZLWpcxT8cunSdNr2rW/9DQDgXuYaRgdE4qyvkmTvyp0BQPteykeyurS54p3Lytitk8YTpRKzBhMot9dVDrxbffSNAIBK9KZ0X22F7kFb5U0yOZ4bVW4wU6DrVtldUru/tjnfSEY9G3WeG+3EV2e7fG2VrlkqyFgafHyuLM/5UB+9e2L1rljltQt2ayy0VUZD7pP2+G3fQG6fG7WBj1lrpwCA/+64xvEeHh4eHrcYf+8kpjHmSWPMUWPMUZ2n2MPDw8Pj5nCjboTTxphxa+2UMWYcwMxmB1prnwLwFABMTEys0xGK/aT67zsohEqdLQp7DxxK20ZYDV88cxYA0NbRWx0yRTz2jp9K2/YefBQAcODBs2nbcy+Q2WOwTCaJSzOSCyVit6KcLibAvV2tCjm1OE9q5FA5ow+hfrCZZGRUcqG4IgWzC2ISMRyt2McuiFGoiAxWoV+/MJm2jQ6Smn14t3JlWoNP/OH/kvNzPzJKnSv3kQp46IAQt29+A7k5ubKNVpl5HClotb3E5ahRZhJHsGVzdH5NTmazZBIZHlTujK62qaoxmObYyNA5Gh05/yKTuosqdefKEqn0be06ycTjMLuCHT4kBFPGRevpwuVBl0GlC9/622d4uKqgiCOe67IWzl4moi2tXanEoUGuVF9SpG6Oj8so18KIXdwCrolZUwRkxOewKu/P5XkivtuKjS72Ofc3zhe0qtwf+X40GtLvSh+d961vejBtq3IK5Aa7zJ4/L6aR119/ncauXN7OzdHc12ty3ignZDwAlEriENDheWjH+p5xYRVF3hk2KRXGiKhcrspYrizR2I1yj21xzc+sJgMX6Tcul1IuK8/BMq/xfEa9+lyaXxWJ2eToYHDN26W6rEmXhqaoolX7dpPJNtRmvbSeK98rXbvBvTnUokxuwI/wRiXwLwJ4grefAPCFGzyPh4eHh8cNYituhH8CIixHjDGTAH4DwG8C+Iwx5kMAzgP46RvtQJgjIuDS9LG07aE3UfL5Ur980cMVIoxilgIiVQ7q9AUiGt4+eEBOXKRgj76SqiIe0bUK7LaXz6pS1vz13TUxnja9ypJHVpExy0ykHNhDGsORe+5L983Pc/GGigQEXGL3JqNIk4FBklqXWLrU+UMKRfptfUX6ffI8B1coImpMUj/Q8TUVbFSn7YwKqllhAbao2uJ77wEANCyTPUoCz7EkpKVWV5hBZ+nrHyJtIyWKlPuhc4sKlbTtIqu0rJGwNHKWA60uzohCNz9HGk+9LpJb3GRJU+VMcTk5du+h4Ki9e3an+0rpWtEk7eYS+IsnqR/Fgmg8ljW+ZkfuSz9nlXRkXUtJuVdW6R6Eaq768qRxdWIhrQ2TdiH7mplIAsNyVZIcW20hR+fnHXmpy3/R3xbnWFmpyly12L10z6i4Ig4P0uJxgUIAML9AeVSGB6gfj77x/nTfJLuKLtVlDb82SfclUOv6gKQtAQBEKhNooY+euVVVIi1ilSVWWfgiDnYJeE0myv3RcIGXSF3TbbVbKgMja9ERS9Za43HkZay0PFeqraNWZabAJGO8Pqupy52S6ShNgBl+ndEwH7sMlnwtteRcIFu3V+/1Zw/dihfKz26y693XfTUPDw8Pj1sGH4np4eHh0aPY9lwomTwRKo2GVoe5/qCKUCyWHClEqr2ul1mOSAX65FMfT9t+8l98mM6hoseyXAvQFYc4cHBXum9mngipxqqowTt3kN+4TpDf5DqFBw8RwXrXISFfl16gWoTVFVETHQnTURFodTZxDHD9vNhKVFj/IKl/HZWBPwxofJOXxLQw9gZ04Wf+2T+XPjK5V1L5VxxpUlCmJ5eaYXmZ85N0RLXPMKkWKf9Xy6poXflH24TO56p2a+I04uMzGR3hud4M4/xfG5w/pKRyTAxyPpq4JX3LhzSuxTkxAUxePAsAOMTEdxgoU5F1FddVyt2ruNwus5nOaqKQffsLoczH7j13Uf9d2tzLstZm2fQzNiYetrkRMutUF8WfOuFI0/5Bsj/kchLL0OAh1zpiQsnzcxC3ZY2FTAa6IieZrCoskaftxx4Rk8iRfRN0/pas9TOv07heP/4qAOBtbxaCc88eOv78y5Kzpx27nESb18TMqn5kuSZsYsVsWWDSuqPS9q5wJGrMRGW+X0w/YyU2aSmyTyq+q7S9cDU/6a8uRLERLD+b2oQSs6+5S9sbqGtmneFGJVpq8jtF516K2IQYcwX6rrq1/NzouqTalLpVeAncw8PDo0ex7RK44QitmpJ8GyxBZnQehDl28eF8JxkspvvGB+iLePKYRF1emjxFGzUpZXZu8iwA4OGdFP25a58wgRMzJAFVT4mUMZQj6a9vQMokvf76GbrmBEnvi8siHbX5Sz59RUlYjtxQroI1lsAN50bQ1EXJZTdMJLIya2g+WrOXsRmStkgIqQSi9pezdN5CXua0zpnkam3qx9nTZ+WaTGLuPbAvbTtzgebyS3/5dNrW5gyQec53UlTnd9Fr/RWJ6hvoJynq4YdFhRgdIanzrt00p4Fy33NSlCOaACGn6jtEOpsYp3s1sYtIaJ3hrsauZl0ayVVElwwT66M7JtK2PBPIs7Pi3lnlqGAXTtdQEZb9o7S2dilX2L5+GmdlRKTyOSa+Y5bI2qpCmXNZrCnir9V2BKVoJFmX8TJH9zhjRUPawXM/Oij3IM+E3OigsI4VdrWbO38eAHDu9bPpvp1DtP6Xpp9J2zJMXrfCzV8hkcr9EXKWxbzKj7I4Q4Ts/KrkILkyRfM72Efr/4H7RBPIsPbdVARumzUATcC79e+KnASKWHdSsC4FGKfEqWYZu3Pr6EynSM8hz1zEx+u1636TcZqRftD59IFyiYyv4tq6GbwE7uHh4dGj8C9wDw8Pjx7FtptQ0lSwSh0ZHyH1SavjX32ZfLIHOan84SFRafI5JnEi8YW+MnOWTt+UiLK9d5GfeMjnLVaEMBoZI4Jpbl7U1SUmL3Xh7R07SP2N2LzTUGSjS1JUV+p+h3/cUSdpNDlVZYe+n8NKpTZcKy9rZCw5Jnli2x3ppvHn/+ev0u2EE9QHyoe2zIRwnzJn7D9MYx4dJpPB8LhEaQ5xn/IqGdPiMTIvfe+Y1A2tW1c8gv4fKfW2wr89tFfMMG977BG6Vkl8rEushjsNtqXmtMO+zbUlMZm12Y+6oKq1DwyQ+WCak4fNqqIQBY4IHNsp81wsqhiANRhkk1mozANNLlxhlMwzP0d9Wl7mtMDK5BdyBN+5i5IwqrJM5o/+fokTcP7fTSbxjSL0ci5asCT3vWBd5KbOjUvPRKnA5kVV+X33MM1LURGKVa5231GmGVfs4gCbfI69djrdd+QIJa6CIiwvXSLf8PygmLEAvd1N2rniIokyZ6xwTMWVK2IaXFyg8554+bsAgNde+rt036FDFHOx/9C9advgCJuBlPnBpU52xT20YSJMfchV39LCJqpqPBOQUjhGkaR8vObB08jlDdjxlCTtShbHZ1X3W79LtgovgXt4eHj0KLZdAndRUv1lIZgG+mjbqJwby5YkidkF+hKO9EnXS0zAxIFIHmcvnQUAjA1K8vd9/AV37lnffU6iPy9OkaTeVxapPMNuTq+cOq967CIJ6W9TfTVXOQJuQCXg77BYOTWtEs73UZ8idlUqFkXCcvlD0BYiNK5S38Z2bJ4L5dkXvp9uFzJEKDabQrBmmYR7y1vfnLadu0iS9BxzSA/cL65mWSYga02R4jOsuTzyiBCQDY70y7K0ePigRMPezylHJ0ZE4qwU6d4mym30wmWKApxZ4GIWs1fSfVUmtxcXRQJvcUrXjHKJdLlYXKRuWxGKxQGatwcg4+vv33wunSRdU5GeoXEl6UTqjzk1acQRvokVeSibo/OPjEhkb5nXeF65ZvZzvyO+Z9q90rKrXke5d/azi2WgohcTTpsauejFpkjW/ZyAxXZEK4xZq2mpSMI6348ir81zl2X9vfo6aXfNpkR4ths0vzbUVPnmcFJrPi9jv+duigQ+dK+489ZWSBp/5XlyyX3hqBCn3/omaYDHXpW1fuTehwAAh+8WqXxgkNabI3fDrj66+d0gF7EmR10JuM76MoYuOjNWpGeSujNujq50zcaVgZQ1rFNObxVeAvfw8PDoUfgXuIeHh0ePYttNKC46bucO8cl2NfISRQaO7ybV/CibRhaNpGy1IanZ/SNCFPZX2AczL6ryfjahlDmF7R984o/SfTW+1nJdyK8a++HqzJM7OVKyMU/qXDWnr0lmnteOiz/69DSZA5ZVdObAAJ2wUiJ1OFSkU4aj48LaxbRttET7+/OioKmknACAKxeU//oQmYF27xbS7r43HKbz5+Qcr7xIRNEYq7VlVa1nhusDlipighqu0HHve/wdaVvADtX9/XTcyLD4r89z6t0z52Q+lhbJrLO8JNGnK0wWL3La3vllibDsMCGbUWl+s1wBJ1CRa/0VGtcAR24OKnNTjk1U2YKYqlbrQhKvxTD7cGvf+jJXV0lUOtRMQPOxg/3FjYpCzbLPsjPtAECeoxFDlXfWmUzSKkTKhOJ84GtVWTsuIjCnFqVlc0ptieb74lmZ73l2Ph4oyPFjnHI3n9c1ZNkkEpH5KCoK2X2F61PuGZdnro+rVS03NyfeEpUm1iW9soFuo76Fyjd8YJjSsr79nbR2Dx0Sk9y3v/F1AMCZM/JsVF/g53ZZTGwPvoGq+ezZQ+fS6ZrjDq3xWPUtYVNtVxWqtP6r+yu7XL1YTWg764f2OXeEZnqtLhKT33HKDKNNMluFl8A9PDw8ehTbLoE70q4yKBJ4J6Zu5SJxyzrChQiOPkeS1XJGItwSQ9Lc2C75kr96jNyPfuCH/1Xa9necqL9aJSmw3ZKCDjOXnWucfNNWuYZdpKLeBgOS0HcV6BxLV0Ta6YQk+Y7tECI0ZterupL4GnWSOKtMlnUSkbDaDYpE25ERSW+iTJJSsyNtayXwiydeSbeXmej6yX/yb9O2xx+n5JF//VVxN9zB5N4OrmJfUK5peY5OG+sXSayPt/PKfa/DUouTNHXOl8vHSVI6PyOudC0uzBHlJW1qXx+RvjtYImy31hNHGZWU3+WM0Lkj+vpoLJVKH+9TdRY5H830tNzvRmPz6lBFlj7bimgtsEvkQEW0miRNbUwEZEHV+UxJKiX9JZbbtNzkimm4v4pc6/D97sTS1+U5GoN+cDMsga8ukbY3dUmij8eGaCwDJYkmrrH0nChNoMNndMTpLi5QAAB3c53Mh+6TIhknTtPz8sL3xBFgLXQK5YALLgSRaNUZJvFjFb3o0rEGTOoePiKEecJut1NTn0vbFmZprCeborVNX6T6uncdJpL03vvlHDvGiFSO1Lul0+ZiEyrFbMw1Xt193LAASFdOlvX705TFPA/6FGnxFCXad0V7bhFeAvfw8PDoUWy7BO5yfwyOiITQ4a91I5BCAPkySxKcwe/8BXH+f/ubyT2ssSpfxGIfue1NXZTcFYSO61cAACAASURBVKdOUDXujqtWrbyLqmx37RsWt6+lJZJ8+ssicd59hHIzPPvSawCA54+dkX78yHsBdGdRPH2KJPRFldHQuSA26iR57xsTya3AQRtDQyL52ogkg05rczejhipt9eAbqY/veve70rbhAbJN/+BblP2aJbc+1gQqZZGKQy5S4KqmA2Jr1Un2lxbI7lphiSZRGVgO3v0AAGDHbsnYOL9AmkvfgLgWusx2xq6vGO7sqK7UFwCssk3YqhJYrlDAhSmy3TstBwDaXOxC50cpljYP5KmyttSnCjq4oJ4ZledmmYOLEs5aeMgFvAAY4PwhYUZLl7SttZQW1+eqMffRaEq/Oy2aK6MKQNgmHV9SGsnAAGkwhSzZqCMj62SAtbf+PlmTLT5HTWVbbHEG0IADSwaV5lXkLJ6TimdxheHvv/tw2nZFuX/SubQ9n+3dqm9Z3p3oB5ElU2cjbiltbPee/QCA/fv3p23PTtP97qhyb1dmFrk/JJ0fO/Zyus8FKt11l/R7bIzcGPv6hO8BB9Q1uNp9rJ69DGtcOmjHuRHqOB5rtKsijSo9fVoAQhDeQEGHa0rgxpg9xpivGWOOGWNeMcb8ErcPGWO+Yow5yX8Hr3UuDw8PD49bh62YUDoAfsVaey+AtwL4RWPMfQA+AuBpa+1hAE/z/z08PDw8bhO2UlJtCsAUb68YY44B2AXg/aBamQDwKQBfB/Cr19uBhGsM9g9JEv9qndSWWiwqhyOsXK3DE68o17QaqSrlkuTy4Fz7OHdC1L6LTO687W2UTlan6ezj9LBDE+K2dH6ezCT1pkrmXiJ1tTJKJM/DfVJ78Qqr12fPvShjqZG5YXFJrrWDq9b3W+rPvrK43u2ocBEEIyYRl0K0pFRSccIjHLznoXT7g7/wb2h8sajZx08RkZgYlUOGyc42q3PziyrpS+LywAhd6gp/JxAiamWZehJOk6p7SdWzdIU5koaQQyUmTE+fFNPWGU5h6tzwhkZkPpy6v6Sq0s/NEpFnlUkkYPc0E7i8ICqylwnTvE6lu7qWBhbk2GVxblbG8voCXdNFMQLAwCApnePjlI+jpaL22i0ywyRW+rjMZq66Mu/EHCEZsnlK1150ZpK8qu5eYPfBhlq7CRN/pTK7pap1kuUoRE34OkK4oUg7V+ndkYhtVbRjco4iZGuqhqYjAXeOy/pfi1CZENJtdU0Ynq8u9zr3G7Nun4vi7OsT805KLnYV63AmObrWyoLcxxc4JfMrLz2btg0N033cuVOI253j+/maZFYZVqbVUS5IaxRR7u5zR5n1Okxypm6E2hWRzVdWmdNsstbkcm1cF4lpjNkP4GEA3wEwxi9395LfsclvnjTGHDXGHK3VNmf+PTw8PDyuD1smMY0xZQCfA/DL1tplnbnrarDWPgXgKQCYmJhYx8KtcCKOgsrklmZmS1T5LyY/RoZIOjsRSLa0mXmSbOZC+YL1l+krec8DQkycPkuSnkuar4nFw4eJ1Dh84K607dwUSRyvvPK9tG1uloNCOOn/oHIdm3yFJPapWclBYpiIDVVA0fgecsfax1O4t08krDyXZmo2dKABSUzazWktPvBz/zLdHtxJUtFL3xcp15FBLfWVj5lUc6XDNIniSlXFWkLgtqDrs8+5RzhL5OycuAw6NzgVu4GBygD3RyTZ+TnWNlgKnJ0VwrLJ2kdHuWHGXNYuVLlQinma55xzMdQVw13yG4h0VFBZFtdikYnZSxfFHa/E5PI9qsCAy9hY5PwujbpoTQsL5G7abss4a5yrpKjcMPsrtO5LOfpbUORkxM9YrEjMTqfF51XZLV05r7T4gCoSwFpsWz15UcgkXKJcWznb4twV0jRm58Tl0mUNXFD5aJwmlesTbWktjNUSOP3VxJ5hqVXnCEklaf7rCEMAqK9SPy5flgIQly7R9lJRjsvwOnKkfEnlXylGdJwmtC9yEYmTZ+WdUq9T0ZJOTOcaGZXiHg8+SAGBhw+JxD46Smuh0i/OGLkCaQoWfH317HXSJIeKSP77IDEBwFCO088B+GNr7ee5edoYM877xwHMbPZ7Dw8PD49bj614oRgAHwdwzFr722rXFwE8wdtPAPjCre+eh4eHh8dm2IoJ5QcB/DyA7xljHDv3HwH8JoDPGGM+BOA8gJ++kQ6cPkVqy97Dkg4yH3BazJYQTRGrQUJkCOlZ5iIF99wjfrh//VdfBgDUlsRfvDhMZNOpSVIW9uwW0vPA3VRoIKfU8oN7af/ivBSFeJXrbiZMkEwuCNmzzORrIxZz0PIimWl2KILk3By1De0hc8JcTvkkJ0x6KnOJjbgWYCLq+Fov5hdePJpuv/w9uk0GYppx+SYiXXQgTY2a4WNE9Y44/axO/+nykWRVfwP2Ew8t7atkxZs0YDNTO1TqPkemKrddZDlXSbvG/slVMUG1mOQzbRWdyTacliK5Y462rK7Q8UV1H0f7qR+RMl04S8VGVObQKK2TQVVowxUkiNR8rKwSkbi6Sv3N5cT84UhAnY50YozI61xe1H1HXlrOx1FtSI8aTBAvLkh+nrl58rWuK3PNvZy2N8O+9d0FDLhep1pPTa7lOZlGH4sPd4vNU7WqnH9pkUyJWRVV6sb+9Fe/mra94y0PowuqWEHi/Ls7KgKSTSzKHR0mNe/QvlBFpr70/HMAgNUF8TcfZv/2C1PSVmEf9iw/N4mKYK6U2R9d+ednIy6EkVNxEAGbZRfIbHT2jEQ6Ly7QvD1/VOW+4biJPXskWnWCC6SMT9CzPzEm75sSp602BVWvM9g8NmEzbMUL5dvYPM3tu6/7ih4eHh4etwTbHon54imShvc+8FjaloC+fkaTdvwFX2ZCZXFRSJbhIXKhe+/jP5K2PfRGyoPwmc//WdpmOK9BP1cH3zUhLlBlJtfCjkgeQztpesYPiBS1xMn4n3+RpNypVeW+lCHCtH9ciJ2RQ9TWVQiA3faOc5GKU5dFQs0y21NXkYdVnoZOIlLDe9b4/HzrG19Jt2ucmS2bUaW4io5ElVseWs5/4ap4Z7QETv3I5xTBym54WZXFLirRWPNZGmdO5XNwqTaMyqLoyOi2KhTRYIIylVp1BBsfr0u1pSG0SuIdKNF2f4nGVC6IlJvL0PkyRu6jUe6Aa9FmUk27HUbs4hh3EXOunBzPnxJz8ixl16syzjpnYKwrH1Cn6QQZ51Yma/74sVcBAOfOnk3bXBSxVe6JE+NE2A9xRsi68vZy24sLQkDOMUlbVxquy9njPMUWl0ULCnjui5GsHZdv5fJl0XDXSuBtVUTCkeimI+dwUZ/aec6C2hzpuboqk+WKh9x9RLT1Rx56FADw3MtS5OGZZynL5iIXA4k7cg92jBMZ+fa3vz1ti/g+nz0nLsfPPEO5lB64j6K8K/3iDDHNY56eFsLerd2dY+JueODAfro+OwJUV8QN0zkEZCKR+hsb5AC6FnwuFA8PD48ehX+Be3h4ePQott2EcmKJVPTZWKXizJBKHbSUypG4GnL0d2JcbAg/9ANEQOYzolYe2EeRlT/+gQ+mbZ/9s7+ga12m804tifLWaJwCAGQhKux8nbZPnRM1Eazm2FEy0QyOiTkhrYunoh0TNjckRlR6l7xpiSMl8xmVtItTulaNSsbE5KFNtIrVrW6NjUp02lSdCJ04FrW5wnU6I9W35VkiZ1eWq9wvUTUTp/5uFB2mzCSZAt0Hm6Hru0RkABCwDaWoknu5yulxe715DJw0yWTFFpFnMrKgzBlDfaR27lE++LvHyf/W8ZTNhqjegaX1FKnIuYEKrbua5KZKceIEpUi9//770rYCm0T0dARMDSUcfTetolBdcrRmXZkp2CQYKzPJwUP7AQCjO6j/utBAhs02AyqxlCNAdZlH58P92nFKo7qqCkC4fTqGIGETUXVF5qjG/axxtGhLmbhc8Yjz00IUuhql8VXqONquCEvrNlK4KEoVJIrEEZ98qwqqXuwPvfPdvEt+4Io1HHlITLAPvInqvrqyoYGi8FzBkYMHJd4j4jndf1jSzk7sJWK4wBG9/cqE4sblCpYAYibZMSppsV1yrJBNT4Fia2N2SGgru1tiNp/LzeAlcA8PD48exbZL4McX6RvyhW9LtOND+0ga2ZkVA3+RpYDxnfSFGx8RqeSug0xGWpEapjgvySc+/Rdp23MvEinkIj27AhutI5HkHHGOrhFrYo5d8zpMiHYCRfK52VSlkRotPq/60kZMaIYsbVmVK6TDlE5Gfa1daa1We/NILdsWib2/RBLFiiJC2zFJZffc+4D8ZoKkkRmOvptR0XernBdFpz9wkqON5byliKSMe95IaTovqVJpV5ZJwq+3RCKscyEFHfWZY9fGEmsaAyr3xyhXGB+fEMnm0C5y89uREzF0lV0P59nNLszK/BVLRFqXVcTrMOe/uHRGiCuHNkvvjVXRYAJHHioR0hVriNlV8OTJE+m+lSVHJMsj5opeREp8TjgkL+BIVijXyGHWmjQ5WuMUxPW6zOmFC5Ndx6ngPlh2uay15J456bk6KxpuhvvpSth1VKRild0IO8p1USIZN5ca60r7CNklMrIqQpaf146KkO3wPLjz67JsTqDvKA3GlTdrqRwkE3s5n1HCKVsTVTSBn/Mz58U1s95yeXRUgZD+A13XX1iSa0YsUZcq+2WwLp/Qkoz50vQ8n4M6nlPpsV2AqSnL+mgsbF7mbzN4CdzDw8OjR+Ff4B4eHh49im03oayyWvHXz4v6eeJ1is58z5uERLprglT1M6cpEvIdbxZTQJ5V75WWqGef+UtKF/n8q5KQqOaiwNiEEajUnU7NCVT0mDN7xEo9a7Jpo80qnlG+xU2OaNTkTRStr99Y5MQ7WbgK2ekuxEwC6iRSHSb8sn1SxWZt6pm5S5K4Km6TKlZX6m3tAiXyGlIVwEc5zWqGq8AUVNapeugqjGg703q1uVYns8s7uCrS/fdKsqfz58k8MbcokaxNR44p8itiYrrArNOIIiwHSiW+styDy7M0luOzktTIMBFV2UFmoUJFCM4ik546TW1ZkVJrUeB71lJmCkcud9V5dP7fbH6oVCQ6OM8+9eWSkHAhj6uoojmdyeLka5QIbWleVPsljpiMlc93JssRoWo95VgfN646vYrmnGGirdYU9TzkMQz2y3pqsbmtxk7qHZUsK0nNJTofKs+H2VwG/OY3vyZj6VBVnFIk8xHzumsrM4kj0l0CL/0stdlUpZ9HRxA2mtIWpxWeODWzqn85NEDm2XJZV4RyFeL18EzXX11t3o05UCaRiJNkBWb9cW4IXeENht8fRTk+aLD5TxHU14KXwD08PDx6FNsugQ+PUH6I+QX5/E1x1Njfct1JAIjb+3iLvnSjOyWK0oT0hf3uUYnG+ouvUiRVM5EvPvhLHATrv1sxS4ZWfYade5iWAlwUZYa//EZ/LjmPgyapXC1Fnbsl5OuHliUKqzQBluK1WD6+k6TFvoqSGmvdEvjO8aF0e/L8JI9JJ8+n7TMnjqdNS+ze565eVW6KVZZ2kriL6aXjVSrhVpMktue/TdXu31mScT7A46z3izTsSDsdZdtggm2JoyM1mXruNYp2m61LZGAjQ9cv7JAxD+4kiSpXoTGFKhKzyG54uaKQ4ibcfOk7V9W4I/fARfEmHaWN8dgdiVlQkYoBa4V1lVOkOU/a4HldjIHnwaVUdflmACG7M3kl9fMlWi2Zv5UFkrgbjVX+K8Szu1N5tebbdU5Jq+qXOsLR/dXkoXP36yjtw7LUms1sTqznVSRwO+T7olJE59hJIFGup86NMuBratI44XwxWup3EamJVVG2PGrr6k6qqvdOeA9UXdco5BTOTYkcTQlNHp6uudlmjVhr1W7NmK4q893vmZaKKrV8joZ6feRC0pYmJvZhq/ASuIeHh0ePYtslcCetZlSWvE6DpKcz0yJ1NasUXPGOR6jCeWFAVY/n4gff+I5k5Kuz7batssHl2I3LSRcbVQgKlTSQfkyVbSzHkptxolCgjs+RlFFQ5bycy1FbBa6ssFTmgiCaStLrH2QXynFJDF9m/8S6CrxY++nde0QynS2zS111clYdwVnplHvYPF83y2NuKXu32F3Xu4l1JeBnnHyZ8k9cWBHJZjSg+ejSYFgqWVX29suWpL5TbBOdVDk0akXWYPZKQv2xAySh5AfElTS9DywVlcuiCRTZHh6oNWavYrtd5jw7tRVxI5y5RGuy0ZC+uXJoLg+GvsdOkwtU8FCGA80cLwJIBsiIbebaZbDNdmCdT6XZpLWzotzV3G0rVdg9VUl+tk3z3FxV1e45N8iSkjid5O3sy0bZuxO7PpjL5YYxyeZFRhJ1H1erxIMUQ30P6G+sFrMLOGqxW2yno1zruHCFVdK2ZH2U57DDNvDYaXvqXrsgJi0cW0v9bDZ0bpi463itmduUj4lVmwvi00VRuq8ZtnS/OffMoC70QtsT8BK4h4eHxz94+Be4h4eHR4/imiYUY0wewDdBNQQiAJ+11v6GMeYAgE8DGALwPICft1aFQm4RKSmkibyQVMGWIlmmV0nNef44EUHvrYlKs2LJtHBxQUwMeVahOzU5R4NVRlfDMFJRcm5fl5uYcW5IcpwNulOwZnLiErbKrlctlZLWmVO0GcGZTKocEVoeEHPJIOdSaKkUmK+xi1lGuU+9aY2WVRkUQm90jPKTTCkTSqrOqd802Uzi6iVqV734KhF2XXv4xG1Wwauzki8jyHGKXuXCdomv8aKqbH8q4vkok1pe2iNFIUYnKKfNMBdZAIAcu+a1VE8sq/m5iKuwR5pIdm2KZLyKr9bls+TSqquEO5Xa6IhaTmfrqpNr9TnL5hqdB8bt1wRhh00Gq6tcs7Spc5awC5vRLn20LrKq+MDYrgk+B0VMLi+I22aHCzRYXYGeb1qtpc0qzjzhfN6w7viMGrsrtFCrKbPeGly4IE4FJ6eoHyVV4zJi20/cVW6A5tRFWyaKWM9yrhzd5kwusU4NxPPsSEajcow4clTbqlw+FX1fnLtrErsoTUVOssmxK+eRK1hh10eOul+2VZ6leIjWxa4HxVW6393S60iJshUJvAngXdbaNwJ4CMDjxpi3AvgtAL9jrT0MYAHAh7Z+WQ8PDw+Pm8VWKvJYAM7vKcP/LIB3AXCl0D8F4D8B+Nh198CRAzpRPgebJCpvgstHcmaGvvif+MyX033veicldT9zSaS/qnPOV9+ojMvkxlJAUbkBZblQQ31FpGdHNFhFMmaYUHQSniaunKSXKMKjzi5jus0dN8BS87BKAn9ljgI5FmclA+LiOQpeOnTwADZDIS8SWY4DRjIqH0jMZJb+uHdSyYTHp3deRQroorRY2lnl8b2mpLp+Lrf2WkMS37/C2slcRSTT4T00rvEDJG0PKJfIHLslBiqfRZvXShip0mQs8UZpUIscn0rP2sXrKiRmmLArnXLlTN399HlZGwusk8jkHE12iey0ZT05iVpXRHdwZHcmq0vecRk8TQLzWsznlDtegX4zP0fX1FkGM6xRhrr6OWubHS0triHhugJXXIELpdWsctGQWlXyqaxFYFU5PieNxiK1Omm/KxgoZDdC61z1lCbFkq+Ka0rn3ipXQXcjrPgMpnBStnb17fD124rET/gdZF3JO/U8pHmNVEcM1o/FMlnd4YDBisrns/tBcsaIjNzvxROcD2q3aJvXwlar0odcD3MGwFcAvA5g0UqY3iSAXZv89kljzFFjzNGNvD48PDw8PG4MW3qBW2tja+1DAHYDeAzAvRsdtslvn7LWPmqtfbSocvt6eHh4eNwcrssP3Fq7aIz5OoC3AhgwxkQshe8GcOmqP94Ew1xJu6ES8Fc5Uiwbij+1SzPpfHm/8d2X031nuD7fYlWYjPlVUoMVF4gSq+MdVqNyqrq6U73zBZVnIXA+uqKqO5/VDpsMjPYPZZUqVhXUW+ynWlD5L1xS+aERMp20FIHb5AIG9ZxcM+HoPF2xfC3aKmKyyvks+gbkmo0qqc26YEDM6l6awVSlMjXrtfwUVqXLtUwAVdlH91uqCMe5GrXNqXwP0RhV6B7fPZq2HRil7eF+mpdARXNWWS5oKCIqYlVe16zMc5RlxNXB8wURFnI89zrK8WpINsjD4ZRNq0w5ltnf1ESjzuEi+WJtAuB1pNedW2OOVO2yYiVuPQkJHDNZ3MrIvXUV6p3pJNGEJedOaSjt143Lal9od7wzP6h+RDwW2xLieWGOzGLt1uZrsqP8wGM+rhVoAtflxdFFQLiJn6VA3QOXMjbRpg42cyUq/bIjkJ01Qx/vTGDaapM4/2xlMnNmo9TUov272cwDTbA6M4x6H7Q5rfPQ3VQ8Ytf+Pem+BtfTfP01iV0ptNlSLUHm18Q1JXBjzKgxZoC3CwB+FMAxAF8D8AE+7AkAX9j6ZT08PDw8bhZbkcDHAXzKUEKBAMBnrLVfMsa8CuDTxpj/DOAFAB+/kQ40WKrMqU9JkyWgTChSaIc/hC5BfVAQKe0sk5eBIlk6LB11FAHZ4IxrVY6E1ESNk4pKWZHSCkxsBkpqcARhoUjX1zkprnAmuUS5C0VMYAxWhGTcOURax86dRNYtVkVSWebMfatLEgU4wIn9Z6/oyMoRaLRVlfUwS2MfHJVrtss0l522yvyWuL9McCoJ3A1ZR+Sl0plm6xzRxtn62ioHSbOf+n3XgJAyg0MUPVmuyNIrF+m+5Zggbqh8Iy12O7RKeg6d+6fuB29nWJPSboSuWIEmxOxVWNoGu95F2n3UuaZpV0QeuyvsoNfTWsmaO0Bd1ZGSPPfOjS9WkY1tnodQaV5tzqcRK3fXUpM0Fyd561w1zTpL7xuUPks2iKh1/Yj0fHO/56cl/06bI0L1LVgHPXTOmRJk5ZoZlw007qpAwT/luVKnsy6Dn9IA86xhDFaE+HYl1FwBEj2nIbt85pSG6/KcdEWf8n1xkakryyqPCS/PJJI5WuJUg9GI9GPfESIqBzm6+uJrp9J9s6co42qk+pa/Sl6ZzbAVL5SXATy8QftpkD3cw8PDw2Mb4CMxPTw8PHoU257Myql4OZX0p+iIjLaojs7NM2EvZJ1gJ2F1q9NSpFPsUkpqIoq2kzRlpXy/FubJdDGvrlnhQgD9Ksqxwr7jeZB5xVWXBoCIVbxQ1WpscvIjVxBAH9epca3Bmkr6szjHYxf2Nc8Rf42rRA+GSv0aGCbzTrmk/MCbbFJSJpRO7HzDne+vSszF3/agKz0mmwVUMqaIVeIimyz6+lSEICfNL+eEjC6xb3g2J+pnizdX2W+9rghZR7TmlbqaDZ3PtKjBwRrzhL7vLSapsllFOmU2n0sXXRsoM0XGme60+YP75maoq6h4Gpmnkj3F64lkF4nsCju0WnLf62w6iesqYpJJzJIyMxX6SUXv8DjbDTlHsIGNI/WH14R2WjSeNkoqRqLKtU2Xl8Ws5yxQes2sRdhRc8x1JxMVgWtB/Q2hUujytkStKgLS2K6/AJBwsrpaJInvJJrapYNW883R0o229M2tddPlS552ks+kQj35+pqgrnBq49EjEqsR8Lvq+LPfoWvOiAk05PunC3NsZNK6FrwE7uHh4dGjMPYG3vo3iomJCfvkk0/etut5eHh4/EPARz/60eestY+ubfcSuIeHh0ePwr/APTw8PHoU/gXu4eHh0aPwL3APDw+PHsVtJTGNMVcAVAHMXuvYOxwj6O0x9Hr/gd4fQ6/3H+j9MfRS//dZa0fXNt7WFzgAGGOObsSm9hJ6fQy93n+g98fQ6/0Hen8Mvd5/wJtQPDw8PHoW/gXu4eHh0aPYjhf4U9twzVuNXh9Dr/cf6P0x9Hr/gd4fQ6/3//bbwD08PDw8bg28CcXDw8OjR3FbX+DGmMeNMceNMaeMMR+5nde+ERhj9hhjvmaMOWaMecUY80vcPmSM+Yox5iT/Hdzuvl4NXJT6BWPMl/j/B4wx3+H+/6kxJnutc2wnjDEDxpjPGmNe43vxth68B/+e19D3jTF/YozJ38n3wRjzCWPMjDHm+6ptwzk3hP/Oz/XLxphHtq/ngk3G8F94Hb1sjPkzV22M9/0aj+G4Meafbk+vrw+37QXOFX1+D8B7ANwH4GeNMffdruvfIDoAfsVaey+oDugvcp8/AuBpa+1hAE/z/+9k/BKoDJ7DbwH4He7/AoAPbUuvto7fBfCX1tp7ALwRNJaeuQfGmF0A/h2AR621D4Bq1XwQd/Z9+CSAx9e0bTbn7wFwmP89CeBjt6mP18InsX4MXwHwgLX2DQBOAPg1AODn+oMA7uff/A/TlV/2zsTtlMAfA3DKWnvaWtsC8GkA77+N179uWGunrLXP8/YK6MWxC9TvT/FhnwLwU9vTw2vDGLMbwI8D+H3+vwHwLgCf5UPu9P5XALwDXLLPWtuy1i6ih+4BIwJQMMZEAIoApnAH3wdr7TcBzK9p3mzO3w/gDy3hGVDB8/Hb09PNsdEYrLV/ZSVJ+zOQEsLvB/Bpa23TWnsGwCn0QMWx2/kC3wXggvr/JLf1BIwx+0Gl5b4DYMxaOwXQSx7Aju3r2TXx3wD8BwAuq/0wgEW1iO/0+3AQwBUAf8BmoN83xpTQQ/fAWnsRwH8FcB704l4C8Bx66z4Am895rz7b/xrA/+XtnhzD7XyBb1SxsydcYIwxZQCfA/DL1trlax1/p8AY8xMAZqy1z+nmDQ69k+9DBOARAB+z1j4MSsVwx5pLNgLbit8P4ACACQAlkNlhLe7k+3A19NqagjHm10Em0j92TRscdkePAbi9L/BJAHvU/3cDuHQbr39DMMZkQC/vP7bWfp6bp52KyH9nNvv9NuMHAbzPGHMWZLJ6F0giH2BVHrjz78MkgElr7Xf4/58FvdB75R4AwI8COGOtvWKtbQP4PIAfQG/dB2DzOe+pZ9sY8wSAnwDwc1b8qHtqDA638wX+LIDDzLxnQYTBF2/j9a8bbC/+OIBj1trfVru+COAJ3n4CwBdud9+2AmvtUPp/igAAAUpJREFUr1lrd1tr94Pm+6vW2p8D8DUAH+DD7tj+A4C19jKAC8aYu7np3QBeRY/cA8Z5AG81xhR5Tbkx9Mx9YGw2518E8AvsjfJWAEvO1HKnwRjzOIBfBfA+a21N7foigA8aY3LGmAMgQva729HH64K19rb9A/BeEPP7OoBfv53XvsH+vh2kRr0M4EX+916QHflpACf579B293ULY3kngC/x9kHQ4jwF4H8DyG13/67R94cAHOX78OcABnvtHgD4KIDXAHwfwB8ByN3J9wHAn4Ds9W2QdPqhzeYcZH74PX6uvwfytrlTx3AKZOt2z/P/VMf/Oo/hOID3bHf/t/LPR2J6eHh49Ch8JKaHh4dHj8K/wD08PDx6FP4F7uHh4dGj8C9wDw8Pjx6Ff4F7eHh49Cj8C9zDw8OjR+Ff4B4eHh49Cv8C9/Dw8OhR/H86g/sGL68EWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GroundTruth:    cat  ship  ship plane\n",
      "START Test Time [entropy-grouped-desc-1]\n",
      "Accuracy of the network on the 10000 test images: 36 %\n",
      "Accuracy of plane : 10 %\n",
      "Accuracy of   car : 15 %\n",
      "Accuracy of  bird :  6 %\n",
      "Accuracy of   cat : 26 %\n",
      "Accuracy of  deer : 36 %\n",
      "Accuracy of   dog : 37 %\n",
      "Accuracy of  frog : 50 %\n",
      "Accuracy of horse : 63 %\n",
      "Accuracy of  ship : 29 %\n",
      "Accuracy of truck : 93 %\n",
      "END [entropy-grouped-desc-1]\n",
      "START Test Time [entropy-grouped-desc-0.9]\n",
      "Accuracy of the network on the 10000 test images: 55 %\n",
      "Accuracy of plane : 40 %\n",
      "Accuracy of   car : 67 %\n",
      "Accuracy of  bird : 36 %\n",
      "Accuracy of   cat : 40 %\n",
      "Accuracy of  deer : 41 %\n",
      "Accuracy of   dog : 55 %\n",
      "Accuracy of  frog : 72 %\n",
      "Accuracy of horse : 59 %\n",
      "Accuracy of  ship : 70 %\n",
      "Accuracy of truck : 71 %\n",
      "END [entropy-grouped-desc-0.9]\n",
      "START Test Time [entropy-grouped-desc-0.8]\n",
      "Accuracy of the network on the 10000 test images: 52 %\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/home/user/anaconda/lib/python3.7/multiprocessing/queues.py\", line 242, in _feed\n",
      "    send_bytes(obj)\n",
      "  File \"/home/user/anaconda/lib/python3.7/multiprocessing/connection.py\", line 200, in send_bytes\n",
      "    self._send_bytes(m[offset:offset + size])\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/user/anaconda/lib/python3.7/multiprocessing/queues.py\", line 242, in _feed\n",
      "    send_bytes(obj)\n",
      "  File \"/home/user/anaconda/lib/python3.7/multiprocessing/connection.py\", line 200, in send_bytes\n",
      "    self._send_bytes(m[offset:offset + size])\n",
      "  File \"/home/user/anaconda/lib/python3.7/multiprocessing/connection.py\", line 404, in _send_bytes\n",
      "    self._send(header + buf)\n",
      "  File \"/home/user/anaconda/lib/python3.7/multiprocessing/connection.py\", line 368, in _send\n",
      "    n = write(self._handle, buf)\n",
      "BrokenPipeError: [Errno 32] Broken pipe\n",
      "  File \"/home/user/anaconda/lib/python3.7/multiprocessing/connection.py\", line 404, in _send_bytes\n",
      "    self._send(header + buf)\n",
      "  File \"/home/user/anaconda/lib/python3.7/multiprocessing/connection.py\", line 368, in _send\n",
      "    n = write(self._handle, buf)\n",
      "BrokenPipeError: [Errno 32] Broken pipe\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-f7cc59df6325>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtestloader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mimages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mpredicted\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-1ca181200171>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m16\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m5\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.7/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 345\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2d_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    346\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda/lib/python3.7/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mconv2d_forward\u001b[0;34m(self, input, weight)\u001b[0m\n\u001b[1;32m    340\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[1;32m    341\u001b[0m         return F.conv2d(input, weight, self.bias, self.stride,\n\u001b[0;32m--> 342\u001b[0;31m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[1;32m    343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#\n",
    "# Test Time\n",
    "#\n",
    "def imshow(img):\n",
    "    img = img / 2 + 0.5\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1,2,0)))\n",
    "    plt.show()\n",
    "\n",
    "dataiter = iter(testloader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4)))\n",
    "\n",
    "accuracies = []\n",
    "\n",
    "for slice_factor in TRAINING_SLICE_FACTORS:\n",
    "    suffix = get_file_suffix(slice_factor)\n",
    "    print(f'START Test Time [{suffix}]')\n",
    "    \n",
    "    net = Net()\n",
    "    net_path = f'./training/cifar10_training_order-{suffix}.net'\n",
    "    net.load_state_dict(torch.load(net_path))\n",
    "\n",
    "    outputs = net(images)\n",
    "\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            images, labels = data\n",
    "            outputs = net(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "\n",
    "    accuracy = 100 * correct / total\n",
    "    print('Accuracy of the network on the 10000 test images: %d %%' % accuracy)\n",
    "    accuracies.append(accuracy)\n",
    "\n",
    "    class_correct = list(0. for i in range(10))\n",
    "    class_total = list(0. for i in range(10))\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            images, labels = data\n",
    "            outputs = net(images)\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            c = (predicted == labels).squeeze()\n",
    "            for i in range(4):\n",
    "                label = labels[i]\n",
    "                class_correct[label] += c[i].item()\n",
    "                class_total[label] += 1\n",
    "\n",
    "    for i in range(10):\n",
    "        print('Accuracy of %5s : %2d %%' % (classes[i], 100 * class_correct[i] / class_total[i]))\n",
    "        \n",
    "    print(f'END [{suffix}]')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Plot Accuracy vs. Training Set Size\n",
    "#\n",
    "training_set_sizes = np.array(TRAINING_SLICE_FACTORS) * 50000\n",
    "all_suffix = get_file_suffix('all')\n",
    "plt.plot(training_set_sizes, accuracies, '-o', label=all_suffix)\n",
    "plt.xlabel('Number of Training Images')\n",
    "plt.ylabel('Accuracy (%)')\n",
    "plt.ylim(20, 65)\n",
    "plt.title(f'Test Accuracy vs. Training Set Size')\n",
    "plt.legend(loc=4)\n",
    "plt.savefig(f'./plots/cifar10_training_order_accuracies-{all_suffix}.png', bbox_inches='tight', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

A  => plots/cifar10_training_order_accuracies-entropy-global-asc-all.png +0 -0
A  => plots/cifar10_training_order_accuracies-entropy-global-desc-all.png +0 -0
A  => plots/cifar10_training_order_accuracies-entropy-grouped-asc-all.png +0 -0
A  => plots/cifar10_training_order_accuracies-entropy-grouped-desc-all.png +0 -0
A  => plots/cifar10_training_order_accuracies-random-all.png +0 -0
A  => plots/cifar10_training_order_losses-entropy-global-asc-all.png +0 -0
A  => plots/cifar10_training_order_losses-entropy-global-desc-all.png +0 -0
A  => plots/cifar10_training_order_losses-entropy-grouped-asc-all.png +0 -0
A  => plots/cifar10_training_order_losses-entropy-grouped-desc-all.png +0 -0
A  => plots/cifar10_training_order_losses-random-all.png +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.1.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.2.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.3.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.4.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.5.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.6.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.7.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.8.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-0.9.net +0 -0
A  => training/cifar10_training_order-entropy-global-asc-1.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.1.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.2.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.3.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.4.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.5.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.6.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.7.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.8.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-0.9.net +0 -0
A  => training/cifar10_training_order-entropy-global-desc-1.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.1.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.2.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.3.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.4.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.5.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.6.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.7.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.8.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-0.9.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-asc-1.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.1.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.2.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.3.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.4.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.5.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.6.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.7.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.8.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-0.9.net +0 -0
A  => training/cifar10_training_order-entropy-grouped-desc-1.net +0 -0
A  => training/cifar10_training_order-random-0.1.net +0 -0
A  => training/cifar10_training_order-random-0.2.net +0 -0
A  => training/cifar10_training_order-random-0.3.net +0 -0
A  => training/cifar10_training_order-random-0.4.net +0 -0
A  => training/cifar10_training_order-random-0.5.net +0 -0
A  => training/cifar10_training_order-random-0.6.net +0 -0
A  => training/cifar10_training_order-random-0.7.net +0 -0
A  => training/cifar10_training_order-random-0.8.net +0 -0
A  => training/cifar10_training_order-random-0.9.net +0 -0
A  => training/cifar10_training_order-random-1.net +0 -0