{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Training a FFNN in dCGPANN vs. Keras (classification)\n", "\n", "A Feed Forward Neural network is a widely used ANN model for regression and classification. Here we show how to encode it into a dCGPANN and train it with stochastic gradient descent on a regression task. To check the correctness of the result we perform the same training using the widely used Keras Deep Learning toolbox." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# Initial import\n", "import dcgpy\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "from tqdm import tqdm\n", "from sklearn.utils import shuffle\n", "import timeit\n", "\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Data set" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# We import the data for a classification task.\n", "from numpy import genfromtxt\n", "# https://archive.ics.uci.edu/ml/datasets/Abalone\n", "my_data = genfromtxt('abalone_data_set.csv', delimiter=',')\n", "points = my_data[:,:-1]\n", "labels_tmp = my_data[:,-1]\n", "\n", "# We trasform the categorical variables to one hot encoding\n", "# The problem is treated as a three class problem\n", "labels = np.zeros((len(labels_tmp), 3))\n", "for i,l in enumerate(labels_tmp):\n", " if l < 9:\n", " labels[i][0] = 1\n", " elif l > 10:\n", " labels[i][2] = 1\n", " else :\n", " labels[i][1] = 1\n", "\n", "# And split the data into training and test\n", "X_train = points[:3000]\n", "Y_train = labels[:3000]\n", "X_test = points[3000:]\n", "Y_test = labels[3000:]\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# Stable implementation of the softmax function\n", "def softmax(x):\n", " \"\"\"Compute softmax values for each sets of scores in x.\"\"\"\n", " e_x = np.exp(x - np.max(x))\n", " return e_x / e_x.sum()\n", "\n", "# We define the accuracy metric\n", "def accuracy(ex, points, labels):\n", " acc = 0.\n", " for p,l in zip(points, labels):\n", " ps = softmax(ex(p))\n", " if np.argmax(ps) == np.argmax(l):\n", " acc += 1.\n", " return acc / len(points)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Encoding and training a FFNN using dCGP\n", "\n", "There are many ways the same FFNN could be encoded into a CGP chromosome. The utility *encode_ffnn* selects one for you returning the expression." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Starting error: 2.655116669472566\n", "Net complexity (number of active weights): 1460\n", "Net complexity (number of unique active weights): 1460\n", "Net complexity (number of active nodes): 81\n" ] } ], "source": [ "# We encode a FFNN into a dCGP expression. Note that the last layer is made by a sum activation function\n", "# so that categorical cross entropy can be used and produce a softmax activation last layer. \n", "# In a dCGP the concept of layers is absent and neurons are defined by activation functions R->R.\n", "dcgpann = dcgpy.encode_ffnn(8,3,[50,20],[\"sig\", \"sig\", \"sum\"], 5)\n", "\n", "# By default all weights (and biases) are set to 1 (and 0). We initialize the weights normally distributed\n", "dcgpann.randomise_weights(mean = 0., std = 1.)\n", "dcgpann.randomise_biases(mean = 0., std = 1.)\n", "\n", "\n", "print(\"Starting error:\", dcgpann.loss(X_test,Y_test, \"CE\"))\n", "print(\"Net complexity (number of active weights):\", dcgpann.n_active_weights())\n", "print(\"Net complexity (number of unique active weights):\", dcgpann.n_active_weights(unique=True))\n", "print(\"Net complexity (number of active nodes):\", len(dcgpann.get_active_nodes()))\n", "\n", "#dcgpann.visualize(show_nonlinearities=True, legend=True)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Start error (training set): 2.582368721500477\n", "Start error (test): 2.655116669472566\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 100/100 [00:03<00:00, 29.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "End error (training set): 0.722099548734631\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "End error (test): 0.7430558265559517\n", "Time: 3.3997908270102926\n" ] } ], "source": [ "res = []\n", "\n", "# We train\n", "n_epochs = 100\n", "print(\"Start error (training set):\", dcgpann.loss(X_train,Y_train, \"CE\"), flush=True)\n", "print(\"Start error (test):\", dcgpann.loss(X_test,Y_test, \"CE\"), flush=True)\n", "\n", "start_time = timeit.default_timer()\n", "for i in tqdm(range(n_epochs)):\n", " res.append(dcgpann.sgd(X_train, Y_train, 1., 32, \"CE\", parallel = 4))\n", "elapsed = timeit.default_timer() - start_time\n", "\n", "print(\"End error (training set):\", dcgpann.loss(X_train,Y_train, \"CE\"), flush=True)\n", "print(\"End error (test):\", dcgpann.loss(X_test,Y_test, \"CE\"), flush=True)\n", "print(\"Time:\", elapsed, flush=True)\n", "\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Accuracy (test): 0.6508071367884452\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3hUVf7H8feZmRRqgBBaQui9SRGComBBkXXFtrvY14Zt7Wvbgm7TdW2rgoVVF8WfqCuswqpYqKK00EMJhB5CCQlJICH9/P6YIaaSCAPDnfm8nodH5t6bme8B/OTke+8911hrERER53MFugAREfEPBbqISJBQoIuIBAkFuohIkFCgi4gECQW6iEiQ8NR2gDHmHeBSYL+1tnc1+68DHvO9PAzcZa1dXdv7Nm/e3LZv3/6nVSsiEuKWL19+wFobU92+WgMdmAxMAN6rYf82YLi19qAx5hJgEjCktjdt3749iYmJdfh4ERE5yhizo6Z9tQa6tXaBMab9Mfb/UO7lYiDupxQnIiL+4e8e+q3Al35+TxERqYO6tFzqxBhzHt5AH3aMY8YB4wDi4+P99dEiIoKfZujGmL7AW8AYa21GTcdZaydZawdZawfFxFTb0xcRkeN0woFujIkHpgM3WGs3nXhJIiJyPOpy2eJUYATQ3BiTCjwJhAFYa98AxgPRwGvGGIBia+2gk1WwiIhUry5XuVxTy/7bgNv8VpGIiBwXx90pmrz3EC98nUzG4YJAlyIiclpxXKBvST/Mq3NSSFegi4hU4LhAD3d7Sy4sLg1wJSIipxfnBbpHgS4iUh3HBnqBAl1EpALHBXqEZugiItVyXKBrhi4iUj3HBXrZDL1EgS4iUp7jAj3c7QbUchERqcx5ga4euohItRwc6CUBrkRE5PTi3EBXD11EpALnBbrvTtGCIgW6iEh5jgv0MLfBGM3QRUQqc1ygG2MId7t0UlREpBLHBTp4++i6sUhEpCJHBnqEx6WWi4hIJY4MdLVcRESqcmagexToIiKVKdBFRIKEYwO9QHeKiohU4MhAj/C4dVJURKQSRwa6ToqKiFTlzEBXD11EpArHBrpuLBIRqcixga4euohIRY4M9Aj10EVEqnBkoKuHLiJSlWMDXT10EZGKHBnoEZqhi4hU4chA10lREZGqnBnobjclpZaSUhvoUkREThvODPSjD4pW20VEpIwCXUQkSNQa6MaYd4wx+40xSTXs726MWWSMKTDG/Nb/JVZ1NNALSrTioojIUXWZoU8GRh1jfyZwH/C8Pwqqiwi3ZugiIpXVGujW2gV4Q7um/futtcuAIn8WdixquYiIVOXIHnrE0ZaLAl1EpMwpDXRjzDhjTKIxJjE9Pf2430czdBGRqk5poFtrJ1lrB1lrB8XExBz3+5QFum4uEhEp48iWS7hOioqIVOGp7QBjzFRgBNDcGJMKPAmEAVhr3zDGtAISgcZAqTHmAaCntTbnZBWtlouISFW1Brq19ppa9u8F4vxWUR2E66SoiEgVjmy5RKiHLiJShSMDPdztBtRyEREpz5GBHhF2tOWiW/9FRI5yZKDrKhcRkaqcGei6ykVEpAoFuohIkHBkoHtcBmN0lYuISHmODHRjDOFuPShaRKQ8RwY6eNsuurFIRORHjg30CI9LLRcRkXIcHOhuCooU6CIiRzk20MM1QxcRqcC5ge52Uag7RUVEyjg30D26ykVEpDxnB7paLiIiZZwb6LoOXUSkAucGulouIiIVODrQdWORiMiPHBvourFIRKQixwZ6uMelG4tERMpxbKBrhi4iUpFjA11XuYiIVOTcQNdVLiIiFTg70NVyEREp49xAd7spKbWUlNpAlyIiclpwbqDruaIiIhU4NtAjFOgiIhU4NtCPztALtISuiAgQFIGuGbqICDg40MtaLrrSRUQEcHCgh7vVQxcRKc+5ga6ToiIiFTg/0NVyEREBnBzoarmIiFRQa6AbY94xxuw3xiTVsN8YY14xxqQYY9YYYwb4v8yqIsLcgAJdROSouszQJwOjjrH/EqCL79c44PUTL6t2R2foug5dRMSr1kC31i4AMo9xyBjgPeu1GGhijGntrwJrouvQRUQq8kcPPRbYVe51qm/bSaVb/0VEKvJHoJtqtlW7BKIxZpwxJtEYk5ienn5CH6qrXEREKvJHoKcCbcu9jgPSqjvQWjvJWjvIWjsoJibmhD5UV7mIiFTkj0CfAdzou9olAci21u7xw/sek24sEhGpyFPbAcaYqcAIoLkxJhV4EggDsNa+AXwBjAZSgDzg5pNVbHkKdBGRimoNdGvtNbXst8A9fquojjwug8uohy4icpRj7xQ1xhDucemyRRERH8cGOnhPjKrlIiLi5exA97g1QxcR8XF0oEd4NEMXETnK0YEe7nHppKiIiI+zA93tolCLc4mIAE4PdLVcRETKODrQI9RyEREp4+hA1wxdRORHjg90XbYoIuLl7EDXjUUiImWcHehquYiIlHF8oKvlIiLi5ehA11UuIiI/cnSgq4cuIvIjRwd6RJhbgS4i4uPoQA93q+UiInKUswPd46Kk1FKsUBcRcX6ggx5DJyICTg90tx4ULSJylLMD3aNAFxE5KigCXTcXiYg4PNAj1EMXESkTHIGuGbqIiLMDXT10EZEfOTrQIz1uAA4XFAe4EhGRwHN0oHdv3RiAVbuyAlyJiEjgOTrQmzUIp3urRizakhHoUkREAs7RgQ6Q0DGaxB2Z6qOLSMhzfKAP7RRNflEpq1PVdhGR0Ob4QB/SoRnGoLaLiIQ8xwd6k/rh9GjVWIEuIiHP8YEO3rbLip0HyS8qCXQpIiIBExSBntAxmoLiUl2+KCIhrU6BbowZZYxJNsakGGMer2Z/U2PMf40xa4wxS40xvf1fas0Gd2iGS310EQlxtQa6McYNTAQuAXoC1xhjelY67HfAKmttX+BG4GV/F3osUfXC6NUmikVbFegiErrqMkMfDKRYa7daawuBD4ExlY7pCcwGsNZuBNobY1r6tdJaDO0UzaqdWeqji0jIqkugxwK7yr1O9W0rbzVwJYAxZjDQDojzR4F1ldCxGYUlpTw+bQ3r03JO5UeLiJwW6hLoppptttLrvwNNjTGrgHuBlUCVFbOMMeOMMYnGmMT09PSfXOyxnNMlhhsS2jFr3V5Gv/IdV7/+Azsycv36GSIip7O6BHoq0Lbc6zggrfwB1toca+3N1toz8PbQY4Btld/IWjvJWjvIWjsoJibmBMquKszt4i+X92bJExfyh5/1IHnvIZ6asc6vnyEicjqrS6AvA7oYYzoYY8KBscCM8gcYY5r49gHcBiyw1gak7xFVP4zbzunIPed3Zm5yuq58EZGQUWugW2uLgd8AXwEbgI+tteuMMXcaY+70HdYDWGeM2Yj3apj7T1bBdfXrs9rTOiqSv8/aiLWVO0QiIsHHU5eDrLVfAF9U2vZGud8vArr4t7QTExnm5sGRXXn0kzV8sXYvP+vbOtAliYicVEFxp2hNrhoQR9eWDXnuq40U6UHSIhLkgjrQ3S7DY6O6sz0jjw+X7ar9C0REHCyoAx3g/O4tGNiuKa/NTaGgWDcdiUjwCvpAN8bwwIVd2JOdz8eJqYEuR0TkpAn6QAcY1rm5ZukiEvRCItA1SxeRUBASgQ7eWfogzdJFJIiFTKB7Z+ld2ZOdz7TluwNdjoiI34VMoAOc3TmaLi0a8vnatNoPFhFxmJAKdGMMF/RoyZKtmRzKLwp0OSIifhVSgQ7e69KLSy0LNx8IdCkiIn4VcoE+IL4JUfXCmL1xf6BLERHxq5ALdI/bxfCuMcxL3k9pqVZhFJHgEXKBDt62y4HDhazZnR3oUkRE/CYkA3141xhcBuao7SIiQSQkA71pg3AGxDdlzsZ9Zdt2ZeaRmVsYwKpERE5MSAY6wPk9WpC0O4ftB3J55ssNnPf8PK6ZtJjCYq2bLiLOFLqB3r0FAKNeXsCb87cytFM0yfsO8dbCrQGuTETk+IRsoHdr2YhuLRvRtml9PhqXwJRbh3Bxr5a8MnszOzPyAl2eiMhPZgL1AOVBgwbZxMTEgHz2UYXFpXhcBpfLALAn+wgXvjCfQe2bMfnmMzHGBLQ+EZHKjDHLrbWDqtsXsjN0gHCPqyzMAVpH1ePhi7oxf1M67y3aQW5B8TG/vljPKRWR00hIB3p1bjqrPWe0bcKTM9bR909fc9mEhUxZtL3Kcc/O2sjw5+ZpTRgROW0o0Ctxuwwfjkvg3VsGc/eITgD88bN1zEraW3bMoi0ZvD5vC7uzjvB/S3YGqlQRkQoU6NWIDHMzvGsMD1/UjU/uPIt+cVE88slqdmXmcbigmEc+WU276PoM7tCMtxduI79ID8wQkcBToNci3ONiwrUDALjngxX8eeY6dmcd4flf9OOBC7qQfqiAT5brsXYiEngK9Dpo26w+z13djzWp2XycmMptwzpwZvtmDO0UTb+2TZi0YKtOkIpIwCnQ62hU71bcf0EXhnRoxsMXdQO8D8y4e0Qndmbm8fnaPQGuUERCnSfQBTjJgyO7Vtk2skdLurRoyOvztvDzvm0qXAYpInIqaYZ+glwuwz3ndWbj3kPMXKNnlYpI4CjQ/eCyfm3oHduYf8xK1hUvIhIwCnQ/cLkMvxvdg91ZR3jn+221Hn8ov4jJ329j/6H8U1CdiIQKBbqfnNWpORf2aMFrc7eQcbigxuMWbErn4pcW8NTM9fzqzcWkZR05hVWKSDAL6cW5/C1l/2Eu/ucCxpzRhoSO0SzeksH6PTk0bxhBbJN65BWVMHN1Gp1iGnDbOR15+vMNRNUPY+rtCbRtVj/Q5YuIAxxrcS4Fup+N/yyJ9xbtACC6QTh946I4mFdE6sEj5Bwp4uZh7Xnwwq5EhrlZk5rFDW8vpX64m4/GDSU+umKoZx8pon64mzC3fpASES8F+imUW1DMl0l76RsXRZcWDSsswWutrbIk74Y9OYydtJh20fWZdtdZZeG9MyOPyyYupFXjSCbdMKhK2ItIaDrh5XONMaOMMcnGmBRjzOPV7I8yxsw0xqw2xqwzxtx8okU7VYMID1cPjKNry0ZVwru69dV7tG7Ms1f1YU1qNv/8dhMA+UUl3Pn+ckpLLWlZR/j5hIUs2JR+SuoXEeeqNdCNMW5gInAJ0BO4xhjTs9Jh9wDrrbX9gBHAC8aYcD/XGrRG9W7Nrwa15bV5W1iyNYPxnyWxfk8OL4/tz8x7h9E6KpKb/r2UdxbWfgWNiISuuszQBwMp1tqt1tpC4ENgTKVjLNDIeKegDYFM4NhPh5AKxv+8J+2a1efWdxP5ODGV+87vzHndW9AuugHT7z6Li3q25M//W1/t2uwiIlC3QI8FdpV7nerbVt4EoAeQBqwF7rfWVlmtyhgzzhiTaIxJTE9XC6G8BhEeXh7bn/yiEs7p0pz7L/xxmYH64R4mXDuAC3u04I+frdPqjiJSrboEenWLk1Q+k3oxsApoA5wBTDDGNK7yRdZOstYOstYOiomJ+cnFBrt+bZsw++HhvHXTINyV1oQJc3uX8T27czSPfrKaWUlaDExEKqpLoKcCbcu9jsM7Ey/vZmC69UoBtgHd/VNiaGkX3YAIj7vafZFhbv514yD6xEbx+/8m1frMUxEJLXUJ9GVAF2NMB9+JzrHAjErH7AQuADDGtAS6AVv9Wah41Q/38ORlvcjILWTyD9sDXY6InEZqDXRrbTHwG+ArYAPwsbV2nTHmTmPMnb7D/gKcZYxZC8wGHrPWHjhZRYe6AfFNOb97CyYt2EqOHx5SfeBwAVMWbeehj1ZpfRkRB6vTeujW2i+ALypte6Pc79OAi/xbmhzLQyO7cumrC3n7u23VrtNeF3uyj/DYtLUs3JxOqe+sSEFxKROvG+DHSkXkVNE95Q7VOzaKUb1a8fbCbRzMLTyu93hqxjqWbcvk7hGdmfXAOTw8siufr93D3OT9fq5WRE4FBbqDPTiyK7mFxbz4zSZKSiteeJRfVMK+nJrbJ9+nHOCrdfv4zfmd+e3F3ejeqjHjhnekU0wDxn+WxJFCresu4jQKdAfr1qoRY8+MZ8riHYyZuJDlOzLJyS9i4twUzv77HM55di5TFu+g8no9xSWl/Hnmeto2q8etwzqUbY/wuPnbFX3YlXmECXM3H3ddn63azdyNmuWLnGp6pqjDPX1FbxI6NuOZLzZy1euLqB/uJq+whBHdYii18MdPk1i2LZNnruxDgwjvX/cHS3eSvO8Qb1w/kMiwipdIJnSM5qoBcUxasJUrB8TRKabhT6pnV2Yev/3PasLdLub+dgQtGkf6bawicmwKdIczxjDmjFgu7NGSSQu2kpZ1hJvOak/v2ChKSy2vz9/CC18ns2RbBv3bNqVzi4ZMWbyDsztHc3GvltW+5xOjuzNzTRqTv9/OXy7v/ZPqeXXOZowxFJVYnp2VzAu/7OePYYpIHSjQg0SDCE+Vq12OPsB6YLum/Pv7bWzaf4hvNuzDbQzjL+1V7eqPAM0bRnBpn9b8d+VuHr+ke9nMvjxrLe8v2Unf2Cj6tW0CwPYDuUxbsZsbh7YjwuPmjflbuD4hnv7xTf0/YBGpQoEeAhI6RpPQMRqAguIS8gpKaNrg2IthXpcQz/SVu5mxOo1rBsdX2f/C15uYMDeFyDAXb914JsO6NOeV2ZsJcxvuGtGJ+uEepq1I5U8z1zP9rrNwuar/5iEi/qOToiEmwuOuNczBe/NS91aNeL+ak6qTv9/GhLkpXNk/lvbRDbhl8jLe+m4rn67azU1D29OiUSQNIzw8Nqo7q3Zl8fr8LWxJP0xhcZX12soE6kErIsFEgS7VMsZwXUI71qXlsCY1u2z7zNVp/Ol/67moZ0v+cXVfPhyXQI/Wjfjr5xuoF+bmjuGdyo69sn8sg9o15bmvkrnghfl0/+OXXD7xe+Zu3F8W4Kt3ZfHLNxbRc/xXPPrJapJ2Z1epRUTqRo+gkxodyi9iyNOz+Vmf1vx5TG9e+nYTb323lUHtm/HeLYPLrpA5lF/E49PWMrRTNNcntKvwHvlFJaxLy2H7gVy2Hcjls9W72ZV5hAHxTYhrWp8Zq9No3jCcszs35+t1+zhSVMLg9s1444aBNKvDTxI1ySss5r6pq7iifyw/69v6hP4cRE4neqaoHLcnpq/lvytTadU4ku0ZeVwzOJ7f/6wHDas5UVoXhcWlfLI8lVfnbCYjt5DbhnXgrhGdaBQZRvaRIj5ZnsqzszbSo3Vjpt4+hPrh3s/5et1e3lywlfGX9iw7CXvsutcwdekumtYPY94j5xFVL+y46q1sxc6DfLR0F3+7ojcePbxbAuCEnykqoeu6IfHkF5VSYi0f3DaEZ67sc9xhDhDucXHtkHjmP3Ieq8aP5NFR3WkU6Q3bqHph3DqsA69e05+1qVnc9f4KCopLePGbTYybspyVOw9y/dtLWJOadczPmJW0l6lLdzG6TyuyjhTx6uyab5Jatj2Tb9fvq3MPf+KcFD5K3MVX6/bVfdAip4gCXY6pd2wUn983jK8eOJezOjf32/uGe1xls+/KLu7Viqev6MP8TekM/8c8Xpm9masHxvHtQ8OJqhfG9W8tYW1q9b32vdn5PD59DX1io/jnr/rzq0FtmfzDdramH65ybH5RCXdMWc5t7yVy9RuLWLHz4DFrPnC4gHm+h3W/tbDi6tDWWnZm5OnkrgSUAl1q1atNVI3he7KMHRzPIxd3IzO3kD+P6cVzV/elY0xDPhyXQON6YVz31mL+/uVGlm7LpLiklP2H8pmVtJd7PlhBQVEp/xx7BuEeFw9d1JUIj4unv9hY5TM+XbmbTF/bZ0dGHle+9gNPzVhXY00zVqVRUmq5PiGelTuzWL7jx28Aby7YyrnPzeXGd5aSvPfQMcdWVFLKsu2Z/GvB1lqPDaRZSXuPe+E3CQz10OW0VlhcSrin4rxjV2YeT0xfy+KtGRSXWiLDXOQXeS+JDHe7ePbqPlzRP67s+NfmpfCPWcm8f+sQhnXx/pRhreWilxYQ5nbx+X3DyC0s4ekvNvDBkp28ck1/LuvXpkotl776HQbDR3ckMPSZOZzdOZrXrhtI0u5srnjte3q2bsz2jDwO5ReVnWso/40wK6+Q3/13LfOT08n1LX4W7nEx/tKeXDckvsYbvQB2ZOTy6co0YhpFcM3gtsc81h/Wpmbz8wkLueXsDoz/ec+T+lny0xyrh64bi+S0VjnMAdo2q8/7tw0hJ7+IhZsPsHRbJnFN69E/vim92jSusj7NLWd34KNlu/jtf1bzv/uG0bxhBAs2H2Dz/sO88It+GGNoGOHhz5f1Yn1aDn/8NInB7ZvRKurHdWg27TtE0u4cxl/ak/rhHq4dEs+b87ewed8h7v9wJc0ahDP55sEAvDx7M+8t2s6GPTn8+9eDiaofRnZeEde/vYRNew/zyzPjGNa5OV1bNuJPM9fzh0+T+D7lAM9e3ZfGkRVP3i7cfICXZ29i2fYffxpI3J7JM1f1qfFRhf7w3qLtAHy1bi9/vLTHSf8GIv6hlos4VuPIMEb3ac1Tl/XitnM6MrBd0yphDt5nsU68dgAH8wr5zQcrKC4p5e2F24hpFMHPy83EPW4XL/6yHwXFJTw6bU2Ffvj0FbtxuwyXneE9/qah7XEZw9hJi9mSnssLvziDpg3CadognKcu6+Wbuefwq0mLSNl/mBvfWULy3kO8ecNA/np5H0b1bk3HmIb8+9dn8sQl3flm/T6e/nxDhboLi0u5d+oK0rLyeXRUN354/HweGtmV6St3c8Nb3tbON+v38fK3m3m3mscR5hUW80PKgZ/c18/KK2TG6jRaNY5kd9aRCvchnI6stUxdupOU/adv++pUUaBLSOgdG8XfrujD4q2Z3P/hKhZsSufGhHZVfgLoGNOQ34/uwYJN6fzru60Ul5RSUmr5dOVuRnSNoXnDCABaRUVyad/WZOQWcuuwDmWtnKNG9W7FO78+k52ZeYx8aT7r9+Tw+nUDOa97iwrHuVyGO4Z34or+scxcnUZe4Y8P/p6XvJ+DeUX89fLe3D2iM22a1OO+C7rw8tgzWJWaxcX/XMDt7yXy0rebeHLGOhb4Ttge9fi0tVz71hJe+mbTT/qz+mR5KgXF3vMQHpfhi6Q91R5XWmp5+OPV/P3LjRW+aVhree6rjcc8H+FPs5L28sT0tYydtISdGXmn5DNPVwp0CRlXD4zj+oR4Pl+7hwiPi+sq3QR11PUJ7RjeNYanv9jImX/7llvfXcbenHyuHBBX4bhHR3Xn/gu68MjF3ap9n2FdmvP+bUPo3SaKCdcO4MKe1a9uCfCLQW3JLSzhi7V7y7ZNX7Gb5g3DOafSN4sxZ8Ty6d1n88yVfZh211ms/ONI2kfX58kZ6ygo9vbm5ybvZ8bqNDo0b8Arc1J45RiXbpZXWmqZsngHZ7ZvSkLHaIZ2imZW0t5qZ/lvLtjKtBWpvDF/Cy99++P7v/TNJibO3cLkH7azbHtmha9ZviOTv32+ntSD/gneI4Ul/OV/6+kY04Di0lJufGcJBw4X+OW9nUiBLiFl/KW9uKhnS+45r3ONd6IaY3jzhoG8ft0ARnRrwYodB4lpFMEFPSrOrts0qceDI7tW2+Y5akB8U2beO4yLe7U6Zl1ntm9K++j6/CdxF+Bte8zeuI8xZ8RWewNTzzaNuWZwPAPbNaVpg3D+NKY32w7k8q8FW8krLOYP/02ic4uGfHn/OVw1II4Xv9nExLkptbZfvks5wI6MvLI7fi/p3ZodGXls2FOxnbFy50Fe+DqZ0X1a8ctBcbwyezNTFm1n8vfbeGVOClcOiKV5wwie+yq57DNz8ou45/9W8q/vtnH+8/N58rMk9mbnn9ClnhPnppCWnc+zV/Xl7ZvOZG9OPjf/exn7c479vvlFJfyw5QCTv9/GE9PXMmHO8T/Q5ShrLevSssnKC9yVQTopKiEl3ONi0o3VXiBQQWSYm0v6tOaSPq0pKimlsLj0mMF9oowxXD0wjue/3sTOjDzmb06nqMRy5YDYOn398K4xjO7TiglzU9i07zC7s47wnzuHEhnm5h9X96WktJTnvkrmhy0HGH9pL7q1akRa1hFen7eFT1fupkebxlzUsyXzktNp3jCcUb2934Au6tWSP3y6li+T9tCzTWPAG8z3fbiSlo0jeebKvjQId5OZW8R4X4tlZM+W/OOqvry/eAdPzVzPwpQDnNMlhr9/uZH9h/J584aBzEtO5/0lO3l30Q7C3S6aNgijcWQYLmMwBu/ibpd058z2zWoc87YDuUxasJUr+seWHffadQO4/b3lDH56No0iPLRrXp87zu1U4VyJtZYb317KUt9PD0cfChMf3aDC1U1rUrP49/fbycor5FB+MfXC3Tw2qju9Y6Oq1FJQXMLvpicxbUUqxkDf2CjO6RLDzWe3J9rXpjsVdNmiyGkiLesIZz87h3vP78J3m9M5UljCl/efU+crTPZkH+GCF+aTV1jCtUPiefqKPmX7SkotUxZt56VvN3Mov4hhXWJYvCUDi+WiXq3Ysv8wG33XxN9zXiceubh72deOnbSIA4cL+fah4aQfKuCJ6WuYm5zOx3ckMLCdN0jzi0q4/b1EjDFMusH7JKyC4hLOf34+zRtF8Pio7lzzr8WMO7cjvxvdA/Cun//1+r1k5hZxMLeQnPwiSn15tC4thz3Z+Tw0sit3De+EMZC87xA/pGRQVFKKx+3iq6S9rN+Tw5yHh1d4Mtba1GyWbc9kR0YuP2zJIPXgEb5+8FzaNqsPeO8/eOCjVTw6qhtXD4ijWYNwfvGm9+T1rAfOJbZJPdalZTN20mJcxhDfrD6NIj1s3n+Yg7mF3Ht+F+4+rxNhvp+cMnMLuXPKcpZuz+SO4R2pF+bmu80HWLUri1aNI5l040B6tan6TeB4aS0XEYe44e0lJO3O5mBeEb8b3Z1x53aq/YvKmbp0J1OX7mTKrUOqXb/mYG4h//x2EzPX7GFU71bcPaITcU29QbcjI5cl2zIZ3ad1heUd3v1hO0/OWMf1CfFMX7GbguJSfj+6B7eUex4teGe+lb/5fLRsJ49NW0ujSA/NGoQz6/5zqRde+086h/KL+N1/k5i5Oo1+cVEcOFzI7qwjFY4xBv4ypneVBeHK2511hJEvzkTsTK0AAAYaSURBVGdIh2a88+szySss4fwX5tGiUSSf3XN22Tr9OzJyGf3yd2Unz8dOWkS428V/7jqL2Cb1AMjOK+LJGUl8uiqNzi0a0rZpPVzGsGFPDgdyC3n+F/0qzPDXpmZz+3uJZB8p4sVf9uOSPv5ZJE6BLuIQM1ancd/UlbgMLHriAlqeBs9k3ZeTT8Izs7EWLu3bmodGdqVjHZ81W1RSysgX57M9I48Pbh/CWZ3qvnyEtZaPlu1iwtwUurdqxIU9WjKiWwsaRXooLrFgqNOia299t5W/fr6BidcOYOPeHF6dk8K0u4aW/XRx1MeJu3j0kzVEhrloEO7h4zuHVvtM3S/W7uGdhdsoKC6l1Frqhbl5YnQPBrar+mSu/Tn5jJuynFW7sji3awwje7bkwh4taB1Vr85/DpUp0EUcIr+ohCFPz6Zf2ya8d8vgQJdTZvaGfbRsHFlt/7g269Ny2Lz/EGPOqNv5AH8rLinl8te+Z292PofyixnVuxUvj+1f5ThrLfdOXcn8TelMvT3huMZanfyiEibMSeF/a9LY7rus8u4RnXh0VPdavrJ6CnQRB9m07xBN6oVV6AvLiVmbms2YiQsJ97iY8/AI2jSpfoZcWmrJKyo5oRVFa2KtZUv6Yb5ev49+cU04+zgXu9Ot/yIO0rVlo0CXEHT6xEXx3NX9aBDhqTHMwXuj18kIc/BeydS5RSM6tzh5f78KdBEJCVcNjKv9IIfTjUUiIkFCgS4iEiQU6CIiQUKBLiISJBToIiJBQoEuIhIkFOgiIkFCgS4iEiQCduu/MSYd2HGcX94cOODHcpwiFMcdimOG0Bx3KI4Zfvq421lrY6rbEbBAPxHGmMSa1jIIZqE47lAcM4TmuENxzODfcavlIiISJBToIiJBwqmBPinQBQRIKI47FMcMoTnuUBwz+HHcjuyhi4hIVU6doYuISCWOC3RjzChjTLIxJsUY83ig6zkZjDFtjTFzjTEbjDHrjDH3+7Y3M8Z8Y4zZ7Ptv1YcYOpwxxm2MWWmM+Z/vdSiMuYkx5hNjzEbf3/nQEBn3g75/30nGmKnGmMhgG7cx5h1jzH5jTFK5bTWO0RjzhC/bko0xF//Uz3NUoBtj3MBE4BKgJ3CNMaZnYKs6KYqBh621PYAE4B7fOB8HZltruwCzfa+Dzf3AhnKvQ2HMLwOzrLXdgX54xx/U4zbGxAL3AYOstb0BNzCW4Bv3ZGBUpW3VjtH3//hYoJfva17zZV6dOSrQgcFAirV2q7W2EPgQGBPgmvzOWrvHWrvC9/tDeP8Hj8U71nd9h70LXB6YCk8OY0wc8DPgrXKbg33MjYFzgbcBrLWF1tosgnzcPh6gnjHGA9QH0giycVtrFwCZlTbXNMYxwIfW2gJr7TYgBW/m1ZnTAj0W2FXudapvW9AyxrQH+gNLgJbW2j3gDX2gReAqOyn+CTwKlJbbFuxj7gikA//2tZreMsY0IMjHba3dDTwP7AT2ANnW2q8J8nH71DTGE843pwW6qWZb0F6mY4xpCEwDHrDW5gS6npPJGHMpsN9auzzQtZxiHmAA8Lq1tj+Qi/PbDLXy9Y3HAB2ANkADY8z1ga0q4E4435wW6KlA23Kv4/D+mBZ0jDFheMP8/6y1032b9xljWvv2twb2B6q+k+Bs4DJjzHa8rbTzjTHvE9xjBu+/6VRr7RLf60/wBnywj/tCYJu1Nt1aWwRMB84i+McNNY/xhPPNaYG+DOhijOlgjAnHewJhRoBr8jtjjMHbU91grX2x3K4ZwE2+398EfHaqaztZrLVPWGvjrLXt8f69zrHWXk8QjxnAWrsX2GWM6ebbdAGwniAfN95WS4Ixpr7v3/sFeM8VBfu4oeYxzgDGGmMijDEdgC7A0p/0ztZaR/0CRgObgC3A7wNdz0ka4zC8P2qtAVb5fo0GovGeFd/s+2+zQNd6ksY/Avif7/dBP2bgDCDR9/f9KdA0RMb9J2AjkARMASKCbdzAVLznCIrwzsBvPdYYgd/7si0ZuOSnfp7uFBURCRJOa7mIiEgNFOgiIkFCgS4iEiQU6CIiQUKBLiISJBToIiJBQoEuIhIkFOgiIkHi/wGAErA59AFZhAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.plot(res)\n", "print(\"Accuracy (test): \", accuracy(dcgpann, X_test, Y_test))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Same training is done using Keras (Tensor Flow backend)\n", "IMPORTANT: no GPU is used for the comparison. The values are thus only to be taken as indications of the performances on a simple environment with 4 CPUs." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n" ] } ], "source": [ "import keras\n", "from keras.models import Sequential\n", "from keras.layers import Dense, Activation\n", "from keras import optimizers\n", "\n", "# We define Stochastic Gradient Descent as an optimizer\n", "sgd = optimizers.SGD(lr=1.)\n", "# We define weight initializetion\n", "initializerw = keras.initializers.RandomNormal(mean=0.0, stddev=1, seed=None)\n", "initializerb = keras.initializers.RandomNormal(mean=0.0, stddev=1, seed=None)\n", "\n", "model = Sequential([\n", " Dense(50, input_dim=8, kernel_initializer=initializerw, bias_initializer=initializerb),\n", " Activation('sigmoid'),\n", " Dense(20, kernel_initializer=initializerw, bias_initializer=initializerb),\n", " Activation('sigmoid'),\n", " Dense(3, kernel_initializer=initializerw, bias_initializer=initializerb),\n", " Activation('softmax'),\n", "])\n", "model.compile(optimizer=sgd,\n", " loss='categorical_crossentropy', metrics=['acc'])\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "End error (training set): [0.7097578446865082, 0.671999990940094]\n", "End error (test): [0.7359503806396978, 0.6525063514709473]\n", "Time: 9.497980389976874\n" ] } ], "source": [ "start_time = timeit.default_timer()\n", "history = model.fit(X_train, Y_train, epochs=100, batch_size=32, verbose=False)\n", "elapsed = timeit.default_timer() - start_time\n", "print(\"End error (training set):\", model.evaluate(X_train,Y_train, verbose=False))\n", "print(\"End error (test):\", model.evaluate(X_test,Y_test, verbose=False))\n", "print(\"Time:\", elapsed)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3RU5dbA4d+ekp7QEnrvUkNHEARUxIqCCggIqCCWa/d+6lVUrv1iw4ZYKDZUBJUiRVBAqaE3KVJDDSUkIXVm3u+PMwmBFAJkEsLsZ60sZ07dJ8HZ83YxxqCUUsp/2Yo7AKWUUsVLE4FSSvk5TQRKKeXnNBEopZSf00SglFJ+ThOBUkr5OU0EqsQRkfEi8nJxx6HUpUITgSrxRCRCRN4VkT0ikiQi273vI7Md01dElonISRE57H39gIiId/94EUn3nn9MROaKSEMfxtxFRGKzvQ8QkSki8peIRPjqvkrlRhOBKtFEJACYBzQGegARQAfgKNDWe8wTwHvA/4CKQAVgONARCMh2uTeNMWFAVeAwML6IniEQmAKUBrobYxLO4VwREf3/WF0Q/QekLnoi0kJEVolIooh8BwRl230XUB241RizyRjjMcYcNsb81xgzU0RKASOBB4wxk40xicay2hjT3xiTdub9jDHJwDdAk1xiaS8iB0XEnm3brSKyzvu6rYjEiEiCiBwSkbfP8mwhwDTACdxgjDmZ7T6LRSReRNaKSJds5/whIq+IyF9AMlBbRIaIyGbv72iHiNyX7fhIEZnuvdYxEVmkyUNlp/8Y1EXN+43/J+BLoCzwA9A72yFXA7OMMUl5XOJyIBD4+RzuGQb0B1afuc8YsxQ4CXTLtvlOrMQBVsnjPWNMBFAH+D6fWwUCvwKpwM3GmBTv/asAM4CXsZ75SeBHEYnKdu5AYBgQDuzGKsHciFUiGgK8IyItvcc+AcQCUViloWcBnVtGZdFEoC527bG+Lb9rjMkwxkwGVmTbXw44kM/5kcARY4wrc0O2b9opItI527FPikg8sB0IAwbncc1vgX7ea4UD13u3AWQAdUUk0hiT5E0ceQnHSlQTziiZDABmGmNmeks4c4EY730yjTfGbDTGuLy/lxnGmH+8pZ0FwBygU7aYKgE1vMcuMjrJmMpGE4G62FUG9p3xwbU72+ujWB9yeTkKRIqII3ODMaaDMaa0d1/2/wdGGWNKG2MqGmNuNsb8k8c1vwF6eev2ewGrjDGZMd0D1Af+FpEVInJjPrEdAfoCE0Tk2mzbawC3e5NVvDc5XXHGc+7NfiERuU5ElnqrfuKxkkZmY/n/sJLbHG+10dP5xKT8kCYCdbE7AFTJ7N3jVT3b69+Aa0UkNI/zlwBpQM/CCsgYswkrGV3H6dVCGGO2GWP6AeWBN4DJ+cSGMWYKMNR7XFfv5r3Al96klPkTaox5PfupmS+8CelHYBRQwZvkZgLivUeiMeYJY0xt4CbgcRG56gJ/DeoSoolAXeyWAC7gYRFxiEgvvL2BvL7E+uD8UUQaiohNRMqJyLMicr0xJh54CfhIRG4TkTDvMdFAnh/QBfAN8DDQGavdAgARGSAiUcYYDxDv3ezO70LGmG+Bh4CfRaQj8BVwk4hcKyJ2EQnydjetmsclArDaG+IAl4hcB3TPFtONIlLXm0wTvPHkG5PyL5oI1EXNGJOOVf0yGDgO9MHqapm5Pw2rwfhvYC7WB91yrGqRZd5j3gQeB/6N1ah6CPgE+D9g8XmG9i3QBZhvjDmSbXsPYKOIJGE1HPc1xqQW4DknYDXqzsCqAuqJ1agbh5XoniKP/1+NMYlYSel7rN/RncAv2Q6ph1VySsJKrB8ZY/4o4HMqPyDaZqSUUv5NSwRKKeXnNBEopZSf00SglFJ+ThOBUkr5OcfZD7m4REZGmpo1axZ3GEopVaKsXLnyiDEmKrd9JS4R1KxZk5iYmOIOQymlShQR2Z3XPp9VDYnIF9553zfksb+/iKzz/iwWkea+ikUppVTefNlGMB5rcE1edgJXGmOaAf8FxvowFqWUUnnwWdWQMWahiNTMZ3/2EZ1LsRYDUUopVcQuljaCe7DmZc+ViAzDmnud6tWr53WYUkoBkJGRQWxsLKmpZ53d45ITFBRE1apVcTqdBT6n2BOBd8bFe7Cm2c2VMWYs3qqj1q1b65wYSql8xcbGEh4eTs2aNTl94tpLmzGGo0ePEhsbS61atQp8XrGOIxCRZsBnQE9jzNHijEUpdelITU2lXLlyfpUEAESEcuXKnXNJqNgSgYhUx5pFcqAxZmtxxaGUujT5WxLIdD7P7bOqIRHJnKY3UkRigRewlhzEGDMGGIG1zOBH3sBdxpjWvopny8FEpq/bz+AONSkXFuir2yilVInjsxKBMaafMaaSMcZpjKlqjPncGDPGmwQwxtxrjCljjIn2/vgsCQDs3buTdX/8yJHjx3x5G6WUAiAsLCzr9cyZM6lXrx579uwpxojy5jdzDVU4vooJAW9gjl2cfwil1KVp3rx5/Otf/2LWrFkF7vXodhftAnJ+kwgcziAAMtJTijkSpZS/WLRoEUOHDmXGjBnUqVMHgK+++oq2bdsSHR3Nfffdl/WhHxYWxogRI2jXrh1Llixh5MiRtGnThiZNmjBs2DAyFxEbPXo0jRo1olmzZvTt27dQ4iz27qNFxRHoTQRp/tevWCl/9tK0jWzan1Co12xUOYIXbmqc7zFpaWn07NmTP/74g4YNGwKwefNmvvvuO/766y+cTicPPPAAX3/9NXfddRcnT56kSZMmjBw50rpHo0aMGDECgIEDBzJ9+nRuuukmXn/9dXbu3ElgYCDx8fF53v9c+E+JIMBKBK50TQRKKd9zOp106NCBzz//PGvbvHnzWLlyJW3atCE6Opp58+axY8cOAOx2O71798469vfff6ddu3Y0bdqU+fPns3HjRgCaNWtG//79+eqrr3A4Cue7vN+UCJzeEoE7QxOBUv7kbN/cfcVms/H9999z9dVX8+qrr/Lss89ijGHQoEG89tprOY4PCgrCbrcD1jiIBx54gJiYGKpVq8aLL76YNTZgxowZLFy4kF9++YX//ve/bNy48YITgt+UCAICQwBwaxuBUqqIhISEMH36dL7++ms+//xzrrrqKiZPnszhw4cBOHbsGLt355wdOvNDPzIykqSkJCZPngyAx+Nh7969dO3alTfffJP4+HiSkpIuOE6/KREEBGSWCNKKORKllD8pW7Yss2bNonPnzrz77ru8/PLLdO/eHY/Hg9Pp5MMPP6RGjRqnnVO6dGmGDh1K06ZNqVmzJm3atAGs3kQDBgzgxIkTGGN47LHHKF269AXH6DeJwBkUDGgiUEoVjezf1KtVq8bOnTuz3vfp0yff4wFefvllXn755RzH/fnnn4UYpcVvqoYCvW0EHk0ESil1Gr9JBI4Aq0RgXJoIlFIqO79JBDis+YWM9hpSSqnT+E8isHsTgZYIlFLqNH6UCKzVesStiUAppbLzn0QgQjpOcKcXdyRKKXVR8Z9EAGSIE9GqIaVUMXjxxRcZNWoUAKNGjaJhw4Y0adKE5s2bM3HiRABcLhfPPvss9erVIzo6mujoaF555ZWsa9jtdqKjo2nSpAm33347ycnJhRKbnyWCAK0aUkoVqzFjxjB37lyWL1/Ohg0bWLhwYdbMos899xz79+9n/fr1rFmzhkWLFpGRkZF1bnBwMGvWrGHDhg0EBAQwZsyYQonJbwaUAbjEic2jVUNKqaLxyiuvMHHiRKpVq0ZUVBStWrXi1Vdf5ffffyciIgKAUqVKMWjQIJKTk/n000/ZtWsXQUHWuKfw8HBefPHFXK/dqVMn1q1bVyhx+lUicEsANnfG2Q9USl06fn0aDq4v3GtWbArXvZ7vIStXrmTSpEmsXr0al8tFy5YtadSoEYmJiVlrE2S3fft2qlevTnh4+Flv73K5+PXXX+nRo8d5P0J2flU15LY5sRktESilfG/RokXceuuthISEEBERwc0334wxpsCLy48bN47o6GiqVavG3r17AUhJSSE6OprWrVtTvXp17rnnnkKJ1b9KBLZAHC5NBEr5lbN8c/elMz/0Q0JCCA0NZceOHdSuXfu0fXXr1mXPnj0kJiYSHh7OkCFDGDJkCE2aNMlaxSyzjaCw+VWJwGMLwK4lAqVUEejcuTNTp04lJSWFxMREpk2bBsAzzzzDgw8+SEKCtWpaQkICY8eOJSQkhHvuuYeHHnooaxpqt9tNerrvP7P8qkTgsQfg8Jws7jCUUn6gZcuW9OnTh+joaGrUqEGnTp0AuP/++0lKSqJNmzY4nU6cTidPPPEEYDUuP//88zRp0oTw8HCCg4MZNGgQlStX9mmsktltqaRo3bq1iYmJOa9zd7zTg8Tjh2k+clUhR6WUuphs3ryZyy67rLjDKDa5Pb+IrDTGtM7teL+qGjKOAALIwOX2FHcoSil10fCrRIA9kAAySHVpIlBKqUz+lQgcgQRKBqkZ7uKORCnlYyWt2ruwnM9z+1cisAcSgEsTgVKXuKCgII4ePep3ycAYw9GjR7NGJheUX/UaEm8bQUKGVg0pdSmrWrUqsbGxxMXFFXcoRS4oKIiqVaue0zn+lQicQVoiUMoPOJ1OatWqVdxhlBh+VTVkcwYRSDppLk0ESimVyc8SQSB2MaSm6ehipZTK5FeJwO60GlDS01KKORKllLp4+FkisBawz0hLLeZIlFLq4uFXicARYJUINBEopdQpPksEIvKFiBwWkQ157G8oIktEJE1EnvRVHNk5AoIByEgvnHU+lVLqUuDLEsF4IL/lc44BDwOjfBjDaTJLBK50XbdYKaUy+SwRGGMWYn3Y57X/sDFmBVBka0c6AzMTgVYNKaVUphLRRiAiw0QkRkRiLmSkoNNbIvBoIlBKqSwlIhEYY8YaY1obY1pHRUWd93XEYfUacru0akgppTKViERQaByZJQIdR6CUUpn8LBEEAODJ0BKBUkpl8tmkcyLyLdAFiBSRWOAFwAlgjBkjIhWBGCAC8IjIo0AjY0yCr2LCblUNebRqSCmlsvgsERhj+p1l/0Hg3OZKvVDeNgKjiUAppbL4WdWQJgKllDqTfyUCb9UQmgiUUiqLfyUCb2OxuDURKKVUJv9KBN4Sgbh1PQKllMrkX4nAoYlAKaXO5F+JwGbHjV2rhpRSKhv/SgSAyxaA3aMlAqWUyuR3icAtTk0ESimVjd8lAo8tAJsnA2NMcYeilFIXBf9LBPYAAiSDNJenuENRSqmLwlkTgYiEiojN+7q+iNwsIk7fh+YbHlsggWSQlqGJQCmloGAlgoVAkIhUAeYBQ7CWoSyRPPYAAnGR6nIXdyhKKXVRKEgiEGNMMtALeN8YcyvQyLdh+ZA9gAAySM3QRKCUUlDARCAilwP9gRnebT6btdTXjD2QAFykatWQUkoBBUsEjwLPAFONMRtFpDbwu2/D8iGH1VisJQKllLKc9Zu9MWYBsADA22h8xBjzsK8D8xVxBBFIBic1ESilFFCwXkPfiEiEiIQCm4AtIvKU70PzEYe3jUC7jyqlFFCwqqHM5SNvAWYC1YGBPo3Kh2yOIG8bgZYIlFIKCpYInN5xA7cAPxtjMoASOyzX5gzUNgKllMqmIIngE2AXEAosFJEagO8WmPcxm9PqNaQDypRSylKQxuLRwOhsm3aLSFffheRbNmcwgWTogDKllPIqSGNxKRF5W0RivD9vYZUOSiR7QKAOKFNKqWwKUjX0BZAI3OH9SQDG+TIoX3I4gwgUF6npmgiUUgoKNkK4jjGmd7b3L4nIGl8F5Gt2p7VcZUZ6ajFHopRSF4eClAhSROSKzDci0hFI8V1IPuYIAiAjXZerVEopKFiJYDgwUURKed8fBwb5LiQf8y5g704vublMKaUKU0F6Da0FmotIhPd9gog8CqzzdXA+YQ8AwJ2hVUNKKQXnsEKZMSbBO8IY4HEfxeN73hKBR6uGlFIKOP+lKqVQoyhKWiJQSqnTnG8iKLFTTGQ2FntcmgiUUgryaSMQkURy/8AXINhnEfmawyoRmAytGlJKKcgnERhjwosykCJjt9oIjEsTgVJKwflXDZVcDk0ESimVnc8SgYh8ISKHRWRDHvtFREaLyHYRWSciLX0Vy2m8jcWaCJRSyuLLEsF4oEc++68D6nl/hgEf+zCWU7yNxeJOL5LbKaXUxa4gs48+JCJlzvXCxpiFwLF8DukJTDSWpUBpEal0rvc5Z97GYptbSwRKKQUFKxFUBFaIyPci0kNECmsMQRVgb7b3sd5tvuVtLBZNBEopBRQgERhjnsOqvvkcGAxsE5FXRaTOBd47t4SS6/gEERmWuR5CXFzchd3V21hsNxm4PSV3OIRSShWWArURGGMMcND74wLKAJNF5M0LuHcsUC3b+6rA/jzuP9YY09oY0zoqKuoCbklWY7EuYK+UUpaCtBE8LCIrgTeBv4Cmxpj7gVZA73xPzt8vwF3e3kPtgRPGmAMXcL2C8TYW6yplSillKcg01JFAL2PM7uwbjTEeEbkxr5NE5FugCxApIrHAC4DTe+4YYCZwPbAdSAaGnM8DnDO7E4BAySDVpQvYK6VUQaahHiEiLUWkJ1Yd/l/GmFXefZvzOa/fWa5rgAfPMd4LJ4LbFkCgVg0ppRRQsKqh54EJQDms0sE4EXnO14H5kscWoFVDSinlVZCqoTuBFsaYVAAReR1YBbzsy8B8ydgzE4FWDSmlVEF6De0CgrK9DwT+8Uk0RcURRAAujp3U0cVKKVWQEkEasFFE5mK1EVwD/CkiowGMMQ/7MD6fsDsDCZQMdh89WdyhKKVUsStIIpjq/cn0h29CKTo2ZxAhNhd7jiUXdyhKKVXsCtJraIKIBAD1vZu2GGMyfBuWb4kjgFIBht1HNREopdRZE4GIdMHqNbQLa1qIaiIyyDupXMnkCCLcoSUCpZSCglUNvQV0N8ZsARCR+sC3WCOLSyZ7AGGONGKPJeP2GOy2wppHTymlSp6C9BpyZiYBAGPMVrwjhEssRyAhNhcZbsP++JTijkYppYpVQRLBShH5XES6eH8+BVb6OjCfsgcSJC4ArR5SSvm9giSC4cBG4GHgEWCTd1vJ5QggQKz2bm0wVkr5u3zbCETEBqw0xjQB3i6akIqAIwiHycBpFy0RKKX8Xr4lAmOMB1grItWLKJ6iYQ9AXGlUKxPCnmM6qEwp5d8K0muoEtbI4uVA1qemMeZmn0Xla45AcKVRvUKIVg0ppfxeQRLBSz6PoqjZA8CdTo2yIazcdRxjDIW3FLNSSpUsBWksvt4YsyD7D9aCMiWXt0RQrWwIiWkujieX6IHSSil1QQqSCK7JZdt1hR1IkXIEgXFTs4y1kL1OPqeU8md5JgIRuV9E1gMNRGRdtp+dwPqiC9EHvAvY1yht1YxpzyGllD/Lr43gG+BX4DXg6WzbE40xx3wala85rJJAtQg7oGMJlFL+Lc9EYIw5AZwA+omIHajgPT5MRMKMMXuKKMbC5y0RBImLChGBmgiUUn6tILOPPgS8CBwCMtd2NEAz34XlY94SAa40apQN1bEESim/VpDuo48CDYwxR30dTJFxeFfedKdTvVwIC7fGFW88SilVjArSa2gvVhXRpcNbNYQrlRplQzicmEZKurt4Y1JKqWJSkBLBDuAPEZmBtX4xAMaYkjv3UFbVUDrVy5UFYO/xZOpXCC/GoJRSqngUpESwB5gLBADh2X5KLmew9d+0E9SKDAVg7d74YgxIKaWKT0HWLM4xxYSIFKQkcfGq2AzEBnuX06RzV+pXCOPjBf9wa4sqOOwFyY1KKXXpyG9A2Z/ZXn95xu7lPouoKASXtpLBzoXYbMJjV9dnR9xJfl6zv7gjU0qpIpff19/QbK+bnLGv5M/QVqszxK6AjBSubVyRxpUjeG/eNjLcnrOfq5RSl5D8EoHJ43Vu70ueWp3BnQ57l2GzCY9fU589x5L5cWVscUemlFJFKr9EUFpEbhWR3t7Xvbw/vYFSRRSf71RvD2KHnQsB6NawPNHVSvP+/O2kubQrqVLKf+SXCBYANwM3el/f5P25EVjo+9B8LDAcqrSCnYsAEBEeu6Y+++JTmLXhYDEHp5RSRSe/uYaGFGUgxaJWJ/jzXUhLhMBwOtYpR4DdxqYDCfSMrlLc0SmlVJHw776StTqDccOepQA47DZqRobwz2Gde0gp5T/8OxFUa2dNN7FzQdamOlFh7DiSVIxBKaVU0fJpIhCRHiKyRUS2i8jTuewvIyJTvQveLBeRM7up+pYzGKq2zWonACsR7DmarN1IlVJ+46yJQERuF5Fw7+vnRGSKiLQswHl24EOsZS0bYa1r0OiMw54F1hhjmgF3Ae+d6wNcsFqd4MBaSDkOQJ3yobg8RtcoUEr5jYKUCJ43xiSKyBXAtcAE4OMCnNcW2G6M2WGMSQcmAT3POKYRMA/AGPM3UFNEKhQ4+sJQuytgYNwNEPMFdUtZY+X+idPqIaWUfyhIIsjsVH8D8LEx5mesCejOpgrWFNaZYr3bslsL9AIQkbZADaDqmRcSkWEiEiMiMXFxhbx2QPV2cMsYsNlg+mM0+a4d7W2bNBEopfxGQRLBPhH5BLgDmCkigQU8L7dpKM4ckfw6UEZE1gD/AlYDrhwnGTPWGNPaGNM6KiqqALc+R9H94L5FcM9vSHBZXgmcwI5DCYV/H6WUuggV5AP9DmA20MMYEw+UBZ4qwHmxQLVs76sCp83qZoxJMMYMMcZEY7URRAE7CxJ4oROBam3gmpeoY/ZSY+/PxRKGUkoVtYIkgkrADGPMNhHpAtxOwWYfXQHUE5FaIhIA9AV+yX6AiJT27gO4F1hojCner+KNbmFvSCPuSJqASdfxBEqpS19BEsGPgFtE6gKfA7WAb852kjHGBTyEVZrYDHxvjNkoIsNFZLj3sMuAjSLyN1bvokfO4xkKlwjrGz1JBY5zcuH7xR2NUkr5XEEWmPEYY1wi0gt41xjzvoisLsjFjTEzgZlnbBuT7fUSoN65BFwUwht0Zs6yVnRb9j5cfi+ERhZ3SEop5TMFKRFkiEg/rDr86d5tTt+FVPzqRIXxhqsvNlcKLHqruMNRSimfKkgiGAJcDrxijNkpIrWAr3wbVvGqGBHEAWd11pXtATFfQMKB4g5JKaV85qyJwBizCXgSWO+dAiLWGPO6zyMrRjabUDsqlK8C+4A7A/58p7hDUkopnynIFBNdgG1Y00V8BGwVkc4+jqvY1YkKY+nxcGjRH1aOgxP7ijskpZTyiYJUDb0FdDfGXGmM6Yw1zcQl/xW5TlQY++JTSG3/OBiPthUopS5ZBUkETmPMlsw3xpitXOKNxWAlAmNgh6sctBgIqyZC/N6zn6iUUiVMQRLBShH5XES6eH8+BVb6OrDiVqd8KADLdx7Fc8UT1sZlY3Ict2BrHDG7jhVlaEopVagKkgiGAxuBh7EGfG3ybruk1SwXSkSQgxenbaLth1vYEtKC9M2nDYlg77Fkhk6M4c7PlrF8pyYDpVTJlG8iEBEbsNIY87Yxppcx5lZjzDvGmLQiiq/YBDnt/PFUV96+oznta5flh4QmBMTvIOXA31nHvP7r39hFqFI6mHsnrGDLwUQAktNdfPHnTr6P0aokpdTFL99EYIzxAGtFpHoRxXNRKRsaQK+WVfngzpZc13sQAPN/+RKAZTuOMmP9AYZfWYcv72lLkNPOoC+W88H8bXR+83dGTt/EiJ83kJyeYzJVpZS6qBR00rmNIjJPRH7J/PF1YBebVs2jORxchzKx85m14QAjp2+icqkghnWuTdUyIUy4uy0n01yMmrOVyypF8Oz1DUnN8LBgSyGvn6CUUoWsIHMNveTzKEqIci1upuzi0bT55k+Oe0IY3a8FwQF2AC6rFMHUBztwMs1N82qlcbk9jFmwg183HOS6ppWKOXKllMpbnonAO9toBWPMgjO2dwb8cnSVveF1sPgdujjWsafSddzU7PQP+Lrlw7NeO+w2rrmsAjPWHyDN5SbQYS/qcJVSqkDyqxp6F0jMZXuyd5//qdoagsvy38v2MW5IG0RyW4TtlB5NK5KU5uKv7UeKKECllDp3+SWCmsaYdWduNMbEADV9FtHFzGaHet0J2zOfCKfAySMw8ymIGZfr4R3rRBIe6ODX9QcLdHmP58yVPJVSyvfyayMIymdfcGEHUmI06AHrJsGc52Dtt5AaDzYHVG8P5S877dAAh42rLivP3M2HcLk9OOw58+6OuCSmrzvAjHUH2H3sJCNubMyd7fyyk5ZSqpjkVyJYISJDz9woIvfgByOL81Snm/XBv+xjqNAYBs+AwAiY9gh4PDkO79GkEvHJGSw7Y8DZiZQMHvtuDd3eWsDbc7cSEeygedXSPDt1Pf+Zup50V85rZdofn8Iva/fz1pwt/BOXVOiPqJTyL/mVCB4FpopIf0598LcGAoBbfR3YRSuoFNz0HtgDoOnt1qL3174CP91vzVLa5p7TDr+yfhTBTjvT1+2nba2yOO02lu44yhPfr+VgQioPda3LgPY1qFgqCLfH8Obsv/lkwQ62HUpi/N1tCAk49Sc6fjKdfp8u5e+Dp5puvl2+lx+GX06tyNAi+xUopS4tYkz+9dIi0hVo4n270Rgz3+dR5aN169YmJiamOEPIyRiYeDPsXwMPLoeI03sTPfD1SmZ62wnCgxwkpbmoUTaEd/pE06J6mRyXm7wylid/WMt/b2nCwPY1srZ/tmgHL8/YzNPXNaRjnUicDuHOT5cR5LDx/fDLqVom5CxhmrM2cCulLk0istIY0zrXfWdLBBebizIRABz9Bz66HEpVhfrXQrV2UPtKCC7DwROpzNl0kOMnM3Ae3UyDtHW07/M0oYG5F8iMMdww+k8MMPPhKxARjDF0f2chYUEOpj7QMevYjftP0G/sUsqEBvDqrU1pXDmC0iEBpKS7idl9jCX/HGXroUR2HU1mz7Fk2tQsw9iBrfO8t1Lq0qSJoKhs+hmWfQL7VoIrFSKqwn0LIDTS2p+aAB93hBN7YPBMqNkxz0t9tXQ3z/20gakPdKBF9TKs3H2c3h8v5vVeTenb9vTG5FV7jnPX58tJSrOms6gYEcSxk+mkuz04bELd8mHUKBdC2dBAvluxh/QpSrgAACAASURBVPa1y/HF4DYEOXVsg1L+Ir9EoF8LC1OjntaPKx12LoRJd8KUYdB/MthsMPtZSIiFwFLWQjf5JIKe0ZV5deZmvlm2hxbVy/D9ir2EBNi5sXnlHMe2rF6GP/+vK2tjT7D5QAJbDiYSFR5IhzrlaFOz7Gnf/tvULMMTP6zl/q9W8snA1gQ4CjLLiFLqUqaJwBccAVDvarjuDZj+qPWhX7EprP4SrnjM6mU07yXYtwqqtMz1EuFBTnpGV2bq6n083r0+09bt58ZmlQjLo0qndEgAV9aP4sr6UfmG1qtlVVIy3Pxn6gae+2k9b97W/IIfVylVsunXQV9qNRia9YE/XoWfhkOFJtDlGWhzr1Uq+PPtfE+/s20NUjM83P/VKpLT3fRpU61QwurfrgaDO9Rkyqp9HE265GcUV0qdhSYCXxKBG96GcvUgLQluHQOOQAiKgHbDYPM0OPx3nqc3rVqKplVKsWZvPHXLh9Eylx5G56tv22q4PIbp6w7k2Jfh9hB7PJkVu45lrbGglLp0aSLwtcAwa9DZ0HlW9VCmdveDMwT+fCff0/t7Rxn3aV2tULt+NqwYwWWVIpiy+vT5Aycs3kXD52dxxRu/c/uYJfR4byELtupU2kpdyjQRFIWwKKh0Rl18aDloNQTW/wBxW/M89daWVXjhpkb0b1/40070alGFtXvjs0YnH05M5Y1Zf9Oqehle79WU8UPaUL98OI99t4YDJ1IK/f5KqYuDJoLidMVjEBAKc5/P85BAh50hHWudNsK4sPSMroxN4CdvqeC937aR7vLwxm3N6Nu2Ol0alOfD/i1JzXDz8LercbnznvZCKVVyaSIoTmFR0OkJ2DoL/vm9YOckH7NGMheC8hFBdKwbydTV+9h+OJFJK/YyoH2N06arqFs+jNd6NWXFruO8PGMz/8QlEZ+cnutMqfvjU3jg65Vs3H+iUOJTShUNTQTFrd1wKF0dZv8HPO68j0tLhBlPwJu1YMbjuU5wdz56taxC7PEUhn25khCnnX91q5vjmJ7RVejXtjrjF+/iqrcWED1yLtEj55zWdpCS7mbYlzHMXH+QYRNXcuxkeqHEp5TyPU0Exc0ZBNeMhMMbrXEGqSdgy6+w8H+wcjxsnwebp8NHHWDF51CzE8R8YU1y53Zd8O27N6pIsNPOjriTDO9Sh3Jhgbke9/ItTfhmaDve7RPNiBsbUaVMCEMnxDBn40GMMTw5eS0b9yfw1LUNiEtK4+FvV+PW9RWUKhF0QNnFoNEtUK09zPw3TH8MTC7f9svVhbtnWeseLPwfzH8ZMpKh9+fWALbzFBro4JYWlVm07Qh3d6yV53F2m9ChTmTW+94tqzJo3HLu/3oVV19WntkbD/H0dQ0ZfmUdosIC+feP6xg1Zwv/16PhecemlCoaOtfQxeLQRpg30updVLMTVG4BKccgfi+kHIe6V4Ez23pASz6C2c9A67vhxmxdUDNSYeowa+xCl2fAfvZcn+H2kO7ynPNEdElpLu4ev4LlO49xS3Rl3ukTndXF9Zkp6/l2+R7uurwGQzvVplrZEIwxbNyfwPR1B7iyfhSX1yl3TvdTSp0/nXTuUjX7P7DkA+j7DTS8wdr2y8OwaoL1ulZnuG3cqUnvfCAl3c2cTQe5tnHF0yaxS3O5eeHnjfy4Kha3x3D1ZRXYcyw5ay2FALuN9+9swbWNK/osNqXUKfklAp+2EYhIDxHZIiLbReTpXPaXEpFpIrJWRDaKyBBfxnPJuWoEVGwGPz8ICfth1UQrCVzxOPT8CPYuh086W3Ma+UhwgJ2e0VVyzGQa6LDzeu9mLPp3N4Z2qs3yXccIdNr5b8/GLHiqC40qR/DA16uYujrWZ7EppQrGZyUCEbEDW4FrgFhgBdDPGLMp2zHPAqWMMf8nIlHAFqCiMSbPLidaIjjDkW3Wh31kPWu6ihodYMCPYLPDgbUwaYC1rvKgX6zqpotEUpqLoRNiWLrzKO/2iaZndJXiDkmpS1pxlQjaAtuNMTu8H+yTgJ5nHGOAcLEqlsOAY8CFd4XxJ5H1oMfr1od+WHmr8djm/XZeqTkMmQlBpeHLW+HghuKNNZuwQAfjhrShWdXSvDlriw5WU6oY+TIRVAH2Znsf692W3QfAZcB+YD3wiDE5u8yIyDARiRGRmLg4nfcmh5Z3wc3vw8Cp1tQV2ZWuZpUGHMEwsSfEbSmeGHMR5LTzYJc67ItPYc6mQ3ke5/YYTRRK+ZAvE0FuM6SdWQ91LbAGqAxEAx+ISESOk4wZa4xpbYxpHRWV/3z7fknESgaR9XLfX7aWlQzEBj8MKbTBaIXhqssqUK1sMF/8uTPPY/49eR29P15MSevYoFRJ4ctEEAtkn0C/KtY3/+yGAFOMZTuwE9CO574QWQ96vGYNXNs4pbijyWK3CYM71CJm93HWxcbn2L/76Emmro5lbewJthzSKbGV8gVfJoIVQD0RqSUiAUBf4JczjtkDXAUgIhWABsAOH8bk3xr3shbH+f0VcGdc+PWMgf1rYPmnkHH+s5Pe0bqq1Wbw164c+8Yu3IHDZsMmMH1tzrUTlFIXzmeJwBjjAh4CZgObge+NMRtFZLiIDPce9l+gg4isB+YB/2eMOeKrmPyezQZd/wPHdsCab87/OunJ1sjm91vC2Cth5pPwx2vnfbnwICe3tarK9HX7OZyQmrU9LjGNH1bG0rtVFTrUiWT6uv1aPaSUD/h0HIExZqYxpr4xpo4x5hXvtjHGmDHe1/uNMd2NMU2NMU2MMV/5Mh4FNLgOqrSGBW+C6zyXqZw3EhaOgtI14KbR1nKciz+4oF5JgzvUxOUxjJ6/LWtm0/GLd5Lh9jC0U21ubFaJXUeT2bAv4bzvoZTKnU46529E4KrnISEWVnyW+zH5fes+vBmWj4XWQ+Cun6DVIKv7anAZmPZI/jOo5qNmZCj92lbnq6V7GDRuOTvikvhyyW6ubVSR2lFh9GhSEYdNmL7uVDPTriMn+XnNvnyuqpQqCE0E/qh2F6jdFWY/C1Pug4QDVk+iv2fCZ9fA241g58Kc5xkDv/4bAsOh63OntoeUtRqi98VYM6Oep1dap/Lh1cGs2HWMa95ZSEKqi+Fd6gBQOiSATvUimb7uAMYY9sWn0GfsEh6ZtIY/thwu8D0ytBuqUjloIvBXfb6yFsXZOBXebwUftYNJ/SDpoDW53cSeVvVR9m/4m3+xEkS353KOV2h6u5Vg5o2ExIPnHk/yMeTLW7lhzf3MuK8FTaqU4ppGFYiuVjrrkBubVWZffAp/bIlj0BfLSU5zU7VMMC9N20Sa6+wlkW+W7aHxC7P5dKH2R1AqO510zt8d2wm/vQgnYqHtMGjSG1yp1nTY67+32hNqdYLIBlZvo6BSMGxB7rOaHv0HPmgNHR+Fq184tzjmjoC/RgMGOj4C14zEGJM1mylAQmoGrV/+LWs0yoS725LmcjN43Ar+r0dD7veWHg6eSGX+34e5skEUVUoHY4zh4wX/8OasLUSGBXIkKY0nu9fnoW55jLs4R/M2H+JESga9WlYtlOsp5Qv5TTGh6xH4u7K14I4Jp2+zh0GvsVYCWPIhLH4fPN6ZP24dk/fU1uXqQP0e1gI7XZ7Je52Ebb9BVH1rZTaAxEOwbCw0uwNsTmuK7RZ3IZGnr5YWEeSka4MoZm88xAd3tsiaxvqaRhV4f/42bmlRmZhdx/nP1PUkpLqwCXRrWJ7IsEAmrdhLz+jKvNG7Gc9OWc+oOVtJd3l47Jr6pyWb7NJdHkb8vIGOdSO5qXnlPI95esp6UtLd3NCsEoEOe67HKXUx00Sgcpc5WrnlXdaYg2M7rYVwKkfnf16be2DLTKsaqeltOffHfGGVNkKjrCkxKjaFRW+BOx2u/D+r/WHzLzDraej/gxVHNi/f0pRhnWvTqkbZrG0jbmzEVW8v4JYP/+JQQhrR1UrzzHUNWbgtju9W7OVIUjp3XV6DF29qjM0m/O/25jjswuj527HZhEevrp8jTGMMz0xZz4+rYpm5/gCd6kVSOiRnYvt1wwHiEq3eV0v+OUqXBuUL8MtV6uKibQTq7OxO6xv82ZIAQO1uUKaWtazmmTZOhemPW20J9gAYf4O1beU4aNHfKlGElYcuT8P2ubB2EqQlnXaJqPDA05IAQLWyITzcrS5HktJ59Op6TB5+Oe1ql+Opaxuy+OmrmPHwFbx0s5UEwBrN/HqvZtzWqirv/raN8X/lnN7ioz/+4cdVsfRuWZWkNBfvz9+e6+OOX7yLmuVCCA2wM3tj3vMlKXUx00SgCpfNZnUt3bMYDm06tf2f+fDjUGupzb7fWstuBpeFHwZb+zv/+9SxbYdBVEP4aTi8VgVG1YfvBlgzrGY6vsuaN2l0C5j/Mg+2DGLV89fw6NX1cdhP/bMOcNhoXLlUjuofm014vVdTujeqwIvTNvHTaqsbakq6m8krY/nf7C30jK7MqNubcXurakxcsos9R5NPu8bavfGs3hPPoA416dKgPHM3HcoaA6FUSaKJQBW+6AFgD7SqgdwuqxH4234Q1QD6TYKAEKt94O5Z1lrNnZ60ZknNZHfCkF/h9glw1QtQ9xrYuchad2FSf5j1DHzQBrb8CuGVYeEo5L1mlJp2b44SRH4cdhuj+7Xg8trleOKHtbR95TeajZhO+tSHuLviTt7o3QwR4fHu9XHYbLw5++/Tzp+weBehAXZua1WV7o0rcCQpjdV7c86XpNTFTtsIVOELLQeNb7WqdvbFwP7V0OB6axRy8KnuoIRXhHtm536NkLLQ+JZT71NfhaVjrMbrtASI7g/d/gMRla3SQcw4q1E79QTc+R04Aq3zdv1pnXP1i1YiOkOQ087Yu1rx2q9/k+HycOeJqbSI/R2P+29sZggQSoWIIIZ2qsXo+dsZ2P4o7WqX40hSGtPXHaBv22qEBznp2rA8TrswZ9NBWtUok/N5Dm6A9T9Yic2m37/UxUW7jyrf2LsCPr/aahS+7k0rMeTRO+ecpJ6wvvWXymVFs9Vfw88PQKNb4LYvYNknMOc5MG4ILQ+DpkH5fCa33T4PvuoFta6EnQus6qpu/wGsFdW6jvqDuMQ0akWG0ipgD7sPxPHaY/dRt3wYAAM/X0bs8RTmP3Flzp5I3w2AzdOsElGD6y7896DUOSq2NYuVH6vWxvrgfXA5NOlVOEkArHEMuSUBsBqcr/kvbPoJPmwLs5+xPnSHzrfWYphwozVFRm6S4mDqcKttot8ka4Dc4tFwfDdgraj204Mdee6Gy6hXLoB/Hx3BD4EjqTtnEBzaCED3xhXZeeQk2w+fUT2VfAzP37MAyFg0Ovd7K1WMNBEo36nV2ariKUodH4YrHrMGt3V7Hu74Eqq0gsHTQeww/kZrxPT+1da0GinHYftvMHmIVdro/bnVhnH1S1bymDsi69JVSgdzb6fajG2xi/JyHNPiLohdAWOugN9epHujCgA5VlvbOm88NpPBj+5OOGMXE7992amdyz6BUXWtto+j/+T/bB63Ne33is/Ofmxx2v4bpGhbSUmiVUPq0pR6wio9ZHdku1V1tHc5YCCwFKSdsPaJDW58B1oNPnX8H2/AH6/C4BlQ8wprmzHWB7/xwP2LrUQy+z+w9hu4YyK3/B5JXGIaL9zUiKsuq8DqPccJGHc1YQ4Pu27+kTZTOhLjbE3TR38kMmkbfNoVIutb7RyuVKvHVLfnrWSUKSXemuNp62xI9X7AOkPg+v9xosEdhAQ6cNrz+E4Xvxc2TIawCtC8X+GVzPJyYK3VqN/hX9D9Zd/eS52T/KqGNBEo/5MUB//Mg91/Wb2XqraByi0h6IxVUtOTrTmYDDDsD6sR/J/f4ctboOeH0GKAdZw7Az67GuL3sOy6GTw24wD7T6RSvWwIZZJ38DOPk9RlJGFdHmH/909QfuMXDA55n3d4iwiTwL4+v7H10AmiVrxFi6PT2BbYmPHVXyOibBT3t4+i9OQ74MA6a+R1rSuh/GXWhIG7FjFLOvJpqUd4e+AV1CgXeir2nYtgwRtWY3nmnBwtBsAN7+Q94rsw/PyQNbK8dHV4ZJ3vE48qME0ESp2vfavgix7W+IcBU+Dbvta33sc2nOqZBBC31fomXPMKXH2/Y/amw4z7aye9jn1KP9fPyBN/W4PlTsTiebc5JwijjIlnUPr/scDTHIBgp527y6zm0YRR7Jaq3Jf2EONLf0611O3WNCANbzh1P4+bJROfo83OMfxEV0bKfbzbN5puDSuAKx3ebgiOIKuE0/R2ayGihW+yN6IlowPv475mNup6dlu9uNoOPf2Z00/CvpVQs1OeH+R/bDnMyGmbePO2ZrSu6a3+Sz5mzVwbXBoSD1jJs3KLQvtTFDpjYNVEqH65NWDyEqeJQKkLkb030qafrCm4r3wq53HLP7VWa7v6Jbj8Qau66Z3GULEZ9P/+1HE/3gvrf8DV5j7WNnmanUeSaVgxnIYVw63BcNvnwXcD8GSk4DI2Ttz4GVFtep12q+R0Fx1fn897wZ9zRfoibgsdz6qDGdzdsRZP1NhG6JS74M7vof61Wedsnfs5Nf78N4HiOj3ugVOhTrdT7yffY1UndXrSmmn2jGQwbe1+HvtuDS6PoVGlCKb96wrsNrG67855DgZNt0pNlz8E17yU8/fk8cDPD0JYlLctxnt9Y6zZa9OT4Pr/FehPc0E2/Qzf32X1KLtnjjXv1iVMew0pdSFa9Ic291pJwBEMre/O/bg291qD3357Af5XB77qbX0zju53+nFXvwhdnsHRfSStapTltlZVaVKl1KkR0XWvgkHTcFdozqOex3htR+0ct/pm2R6OJ2dQ/sp7sGWc5LsrDjGgfXXGLd7J0ikfkBJQjoxaXbOOT0jNYHBMTR4MGUXSNaP4tulntHePZZepRNKURyDDu0To1tlWEihXDxaNyrEE6dfLdvPwpNW0rFGGV29tyqYDCUxeudf6cF/xGSmV2tJtiptdEa0xm37OfZGjv96x2lT+es9a8jTTvJHw59vWwke7/jr9nN2L4df/s5ZZLQxpSdbAxMj64MmwElei/04RoiUCpQrClQ5T7oVKza11HPI8Lg22zbEW+dk6y6o+engNOIPO67avztzMp4t2MOfRztSrEA5AmstN5zd/p1ZkKJOGtrem/g4tD3f/yt87dlF3YivGubrzben7ePLaBlzXpCJPTV7HlFWx/Hh/B1pUtwa87YtP4YuJX/D8sWeZFTWEjgNGEDS2A8kSysRm4+m5722q755C8uVP8EPYQH5cvY91sSfo1rA8H/VvSaBduO2Tpew+msyiWzMI/qEvI5xPMCm5Dbcyjzecn5Jx7wKcVbPNUbVnKYy7Hhr1tBrzV46Da1+1Gt/nPMf37i50tq0lpEIdIu7/zSotpByHD9tba2XYHNZgwo6PWEul5jUT7tnMfQH+ehfungM2O0y4GcrWhj5fWtfNa9Bf+kkrKR3eRMaBjaSHVSH02hEX1hZiDMTGWKPrwyue/3XOQquGlCoOHrc1q6oz+LwvcexkOp3f/J1O9SL5eEArwPpW/p+pG/jqnnZcUS8S/nzHWlPioZXWnE6/PsWS7r8wYilsO5xE3fJhbD+cxENd6/LktaePrnZ7DNs+uoNacb/zuyea7rYYeqe/yGpTD8HDm46x3O5YSIynPl+Vuo/m7a9iQD0XzsXvwLrvSS57GR8daMBtZbYRcXI3Xd0f8OWwK1i6YRt3L+nOjIg+dLzvPSLDAq02hDGdSDM2Xqr8CQ2qlWfQ/pFWFQ2wOuxKBpwYzgDnfJ7xfAp3/gD1u8NPD8Lab6HfJMy2OXhixmE3VvVWAqGclFACnU5CAx0EhJZCrnnp9Kqu7H8Sj+HA9rVUnnQV0qwv3PKhtWP7PPimj1U6sAdCmRpWN+ToO7OfDJ9fY42WB04QRimS2N35bWp0u+fUcbuXwNIPrd5eqSesHl7XvGS1M50pPZnE74YR/s806335RtbqgR0eskbNFyJNBEqVYO/M3cp787bRrWF5UjPcbNyfQM3IUH56oIM1gjnxoNVI2/ERKxEYDwxfhNtjmLIqlnfmbiUqPJAfhncgwJHLN93Eg7hGt8KRkcT2WgOQ69+gSulgth9OYtP+eCK3/UDnvWNwpMRBtXbW2Al7gLWI0ZGt1nvgfXcvmg98k871owA4+MG1pMTtolv6W1xdMY3nPB9T5cRqbk17kY3UxmPg4z5NuG7LsySkumj9dz+Gdm1IZLCNbr/dQGS5SEKvHwlf3wadniD+8qd59Ls1bN2ymfsrb6WyM5kQ1wlMWgJxCSl4jKGVcxfVPfus30W35wHBtWcZG/6aTuyRBPbEp9PBE0NDZxxBj6222ikyHd5slViO77R+j3Fb4YEl1qy4YDUs//IvNjd/liGr62ALjOAj9wvU8+xg7x1zaNioOexdjntCTzLswTii6uEILgWHN1kLP3V4yGpfyiwdJuzH/U0/5OBa3svoRboEMLjSbiocW2k1uPf5Cqq1LbR/R5oIlCrBElMzuGd8DIlpLkID7IQHOXiiewOaVMk2TuLrO2DPEmseph6vQ/v7s3Z5PAaPMafNyprDuu+tbp99v7HWhDhTWiIsehs2/AiNbobL/wXh1gC6w/t2M27SNzTqcgc3tapz6pyYcTD9UbZEXUvtuHl4jPCW816iOg+jV8sqDPtyJRv3n+CH+zrw9twtrNoTz8J/dyXQYePV119kpHu09W26VDVW9viZR37czKGEVF64qTH921U/bRqPEykZzN5wkA/mrOMR1zh6m7lQrh7m5BEk9bj1e0CwYXBh5z+uYdz/6PPUjDzV5dbtMZjM31PiQfigLVRqZo2QT0uA91tx0FGF9oeeIrpaGcYObIUnfi+hX1zJTiqzu+2LdF1xH3HuUO5IH0FyQCS3tarKrU1KUXHpK1Tc9i3JwRUJLFsdu92OObKNtJSTPJL+AH0HDmfswh0s2XGU97oF0nPzk5CwD254G1oOPMd/MbnTRKDUpW7TL/D9QKsO/YktEBpZ3BFZ4zXe8lZFtRzIyfaPE1i2WlZCOpKURs8P/uJkuov45IzTlhv9YtF2Os7tSX3bPt6u/gHvby1DldLBfHBni6w2jtzsPHKSfmOX0tn1J8+XmsWypAr8nNyMrjf0oVeHxuBxc/hEMt3eXUzrmmUYN7gNIsKGfScYPG45GW7DlfWjuOqy8lybOougWY9bY0YOb8Ys+ZCb0l+mWqPLeadPNEFOazW6o8u/o9zMYWQYO8dtpfm9w5fUqnsZk5bvYdq6/WS4rc/YTrZ1DLbPJtTmomJEAOm2QB4+fDN33nwdd11ek9QMN//6djVzNx1iQLNwRqSNImD3AqsLboPrrelSKjQ57/YITQRKXepc6fBuE2twXN+vizuaU3YvscZPlKuT6+7NBxLo/fFiQgMdLHyqK8EB1odraoabO9/4mvDkWJbZW3Bf5zrcd2VtQgLO3ji868hJ+n26lAMnUgkPdPDxgFZWW0o2ny3awcszNvPJwFZULhVM/8+WEh7kpH3tcvyx5TBHT6ZTvUwgv5Z6g9DjWzAZJ/nRcyXjyz3G5OEdspJApuQpD2PfNhvH3dOxR51aCzsuMY2YXccoFeykXFggSWkZjF+8m5nrD+D2GG5rVZX/3dYsq3Tjcnt457etjF24g1AHjKkfQ5P4+YTGrUEwxDe7h9K93j6nP0EmTQRK+YP4vVa1TvapvkuALQcTEYH6FU6vkpqz8SALt8XxQJe6VC59bg3uu4+eZPS87QzrXJsGFXNWdWW4Pdw4+k9OpGSQnO4iItjJt0PbU61sCB6PYfE/R3lq8lpKn9zJdOfTpJgAbpb3mPjwDVQtE5LzhsZYnQMK2Isp9ngyf247wi0tquRIKgA74pJ4cdomFm61JiSM5ARd7atp2rwtd91x+zn9LjJpIlBKqTMs23GUPmOXUr1sCN8MbZfjA/5oUhqPfrcG+z+/kUIg/xoyOEfJwpeMMazac5yEVBeBDhuBDjuVSwdRqdT59ULTRKCUUrlYvP0I9SqEExUemOt+t8cw7q+dRIYFckuLPKY/LyHySwS6QplSym91qJv/N3y7Tbi3U86R3ZcanWJCKaX8nCYCpZTyc5oIlFLKz2kiUEopP6eJQCml/JwmAqWU8nOaCJRSys9pIlBKKT9X4kYWi0gcsPs8T48EjhRiOCWFPz63Pz4z+Odz++Mzw7k/dw1jTFRuO0pcIrgQIhKT1xDrS5k/Prc/PjP453P74zND4T63Vg0ppZSf00SglFJ+zt8SwdjiDqCY+ONz++Mzg38+tz8+MxTic/tVG4FSSqmc/K1EoJRS6gyaCJRSys/5TSIQkR4iskVEtovI08Udjy+ISDUR+V1ENovIRhF5xLu9rIjMFZFt3v+WKe5YC5uI2EVktYhM9773h2cuLSKTReRv79/8cj957se8/743iMi3IhJ0qT23iHwhIodFZEO2bXk+o4g84/1s2yIi157r/fwiEYiIHfgQuA5oBPQTkUbFG5VPuIAnjDGXAe2BB73P+TQwzxhTD5jnfX+peQTYnO29Pzzze8AsY0xDoDnW81/Szy0iVYCHgdbGmCaAHejLpffc44EeZ2zL9Rm9/4/3BRp7z/nI+5lXYH6RCIC2wHZjzA5jTDowCehZzDEVOmPMAWPMKu/rRKwPhipYzzrBe9gE4JbiidA3RKQqcAP8f3t3E2pVFYZx/P/ELcuPLCPDLFJLJIS62iTSRLJJEVlhFKVINGxikyIM+qBBA4smUYIRllJBaUlESAZGg9QUs7DvjLxlKVSKhWb6NFjLuITXD/R4bO/nB5u79zrr7LNezrnrvXvtc9diUb/ipsd8NjANeAHA9l+2f6fhcVc9wFmSeoDBwE80LG7bHwC//qd4oBhnAq/a3mt7C/ANpc87am1JBKOBrf2O+2pZY0kaA0wC1gAX2N4GJVkAI7vXso54BngAONCvrOkxjwN2AC/WIbFFkobQ8Lht/wgsCjpzNAAAA3RJREFUAH4AtgE7ba+k4XFXA8V43P1bWxKBDlHW2O/NShoKvAHMs72r2+3pJEk3Adttr+92W06yHmAy8JztScAf/P+HQ46ojovPBMYCFwJDJM3ubqu67rj7t7Ykgj7g4n7HF1EuJxtH0umUJLDU9rJa/IukUfXxUcD2brWvA6YAN0v6njLkd52kJTQ7Ziif6T7ba+rx65TE0PS4rwe22N5hex+wDLiG5scNA8d43P1bWxLBOmC8pLGSzqDcWFnR5TadcJJEGTP+3PbT/R5aAcyt+3OBt0522zrF9kO2L7I9hvK+vm97Ng2OGcD2z8BWSRNq0QxgMw2PmzIkdLWkwfXzPoNyL6zpccPAMa4A7pQ0SNJYYDyw9pjObLsVG3Aj8BXwLTC/2+3pUIxTKZeEm4CNdbsROI/yLYOv688R3W5rh+KfDrxd9xsfM9ALfFzf7zeBc1sS92PAF8BnwMvAoKbFDbxCuQeyj/IX/72HixGYX/u2L4EbjvX1MsVERETLtWVoKCIiBpBEEBHRckkEEREtl0QQEdFySQQRES2XRBDRYZKmH5wVNeJUlEQQEdFySQQRlaTZktZK2ihpYV3jYLekpyRtkLRK0vm1bq+kjyRtkrT84Nzwki6T9J6kT+pzLq2nH9pv7YCl9b9ikfSkpM31PAu6FHq0XBJBBCDpcuAOYIrtXmA/cDcwBNhgezKwGnikPuUl4EHbVwCf9itfCjxr+0rKHDjbavkkYB5lPYxxwBRJI4BbgYn1PE90NsqIQ0siiChmAFcB6yRtrMfjKFNbv1brLAGmShoOnGN7dS1fDEyTNAwYbXs5gO09tv+sddba7rN9gDL1xxhgF7AHWCTpNuBg3YiTKokgohCw2HZv3SbYfvQQ9Q43J8uhpgM+aG+//f1Aj+2/KQuIvEFZZOTdY2xzxAmRRBBRrAJmSRoJ/64Pewnld2RWrXMX8KHtncBvkq6t5XOA1S5rP/RJuqWeY5CkwQO9YF03YrjtdyjDRr2dCCziSHq63YCIU4HtzZIeBlZKOo0y6+N9lAVfJkpaD+yk3EeAMg3w87Wj/w64p5bPARZKerye4/bDvOww4C1JZ1KuJu4/wWFFHJXMPhpxGJJ22x7a7XZEdFKGhiIiWi5XBBERLZcrgoiIlksiiIhouSSCiIiWSyKIiGi5JIKIiJb7B8d4zoz46qmNAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# We plot for comparison the MSE during learning in the two cases\n", "plt.plot(history.history['loss'], label='Keras')\n", "plt.plot(res, label='dCGP')\n", "plt.title('dCGP vs Keras')\n", "plt.xlabel('epochs')\n", "plt.legend()\n", "_ = plt.ylabel('Cross Entropy Loss')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.6" } }, "nbformat": 4, "nbformat_minor": 2 }