diff --git a/examples/triplet/convert_mnist_triplet_data.cpp b/examples/triplet/convert_mnist_triplet_data.cpp new file mode 100644 index 00000000000..d1eed30cba6 --- /dev/null +++ b/examples/triplet/convert_mnist_triplet_data.cpp @@ -0,0 +1,127 @@ +// This script converts the MNIST dataset to the leveldb format used +// by caffe to train siamese network. +// Usage: +// convert_mnist_data input_image_file input_label_file output_db_file +// The MNIST dataset could be downloaded at +// http://yann.lecun.com/exdb/mnist/ +#include // NOLINT(readability/streams) +#include + +#include "glog/logging.h" +#include "google/protobuf/text_format.h" +#include "leveldb/db.h" +#include "stdint.h" + +#include "caffe/proto/caffe.pb.h" +#include "caffe/util/math_functions.hpp" + +uint32_t swap_endian(uint32_t val) { + val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF); + return (val << 16) | (val >> 16); +} + +void read_image(std::ifstream* image_file, std::ifstream* label_file, + uint32_t index, uint32_t rows, uint32_t cols, + char* pixels, char* label) { + image_file->seekg(index * rows * cols + 16); + image_file->read(pixels, rows * cols); + label_file->seekg(index + 8); + label_file->read(label, 1); +} + +void convert_dataset(const char* image_filename, const char* label_filename, + const char* db_filename) { + // Open files + std::ifstream image_file(image_filename, std::ios::in | std::ios::binary); + std::ifstream label_file(label_filename, std::ios::in | std::ios::binary); + CHECK(image_file) << "Unable to open file " << image_filename; + CHECK(label_file) << "Unable to open file " << label_filename; + // Read the magic and the meta data + uint32_t magic; + uint32_t num_items; + uint32_t num_labels; + uint32_t rows; + uint32_t cols; + + image_file.read(reinterpret_cast(&magic), 4); + magic = swap_endian(magic); + CHECK_EQ(magic, 2051) << "Incorrect image file magic."; + label_file.read(reinterpret_cast(&magic), 4); + magic = swap_endian(magic); + CHECK_EQ(magic, 2049) << "Incorrect label file magic."; + image_file.read(reinterpret_cast(&num_items), 4); + num_items = swap_endian(num_items); + label_file.read(reinterpret_cast(&num_labels), 4); + num_labels = swap_endian(num_labels); + CHECK_EQ(num_items, num_labels); + image_file.read(reinterpret_cast(&rows), 4); + rows = swap_endian(rows); + image_file.read(reinterpret_cast(&cols), 4); + cols = swap_endian(cols); + + // Open leveldb + + leveldb::DB* db; + leveldb::Options options; + options.create_if_missing = true; + options.error_if_exists = true; + leveldb::Status status = leveldb::DB::Open( + options, db_filename, &db); + CHECK(status.ok()) << "Failed to open leveldb " << db_filename + << ". Is it already existing?"; + + char label_i; + char label_j; + char label_k; + char* pixels = new char[3 * rows * cols]; + const int kMaxKeyLength = 10; + char key[kMaxKeyLength]; + std::string value; + + caffe::Datum datum; + datum.set_channels(3); // one channel for each image in the pair + datum.set_height(rows); + datum.set_width(cols); + LOG(INFO) << "A total of " << num_items << " items."; + LOG(INFO) << "Rows: " << rows << " Cols: " << cols; + for (int itemid = 0; itemid < num_items; ++itemid) { + int i = caffe::caffe_rng_rand() % num_items; // pick a random pair + int j = caffe::caffe_rng_rand() % num_items; + int k = caffe::caffe_rng_rand() % num_items; + read_image(&image_file, &label_file, i, rows, cols, + pixels, &label_i); + read_image(&image_file, &label_file, j, rows, cols, + pixels + (rows * cols), &label_j); + read_image(&image_file, &label_file, k, rows, cols, + pixels + (2 * rows * cols), &label_k); + datum.set_data(pixels, 3*rows*cols); + if (label_i == label_j && label_i == label_k) { + datum.set_label(1); + } else { + datum.set_label(0); + } + datum.SerializeToString(&value); + snprintf(key, kMaxKeyLength, "%08d", itemid); + db->Put(leveldb::WriteOptions(), std::string(key), value); + } + + delete db; + delete pixels; +} + +int main(int argc, char** argv) { + if (argc != 4) { + printf("This script converts the MNIST dataset to the leveldb format used\n" + "by caffe to train a triplet network.\n" + "Usage:\n" + " convert_mnist_data input_image_file input_label_file " + "output_db_file\n" + "The MNIST dataset could be downloaded at\n" + " http://yann.lecun.com/exdb/mnist/\n" + "You should gunzip them after downloading.\n"); + } else { + google::InitGoogleLogging(argv[0]); + convert_dataset(argv[1], argv[2], argv[3]); + } + return 0; +} diff --git a/examples/triplet/create_mnist_triplet.sh b/examples/triplet/create_mnist_triplet.sh new file mode 100755 index 00000000000..f404f2aa255 --- /dev/null +++ b/examples/triplet/create_mnist_triplet.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +# This script converts the mnist data into leveldb format. + +EXAMPLES=./build/examples/triplet +DATA=./data/mnist + +echo "Creating leveldb..." + +rm -rf ./examples/triplet/mnist_siamese_train_leveldb +rm -rf ./examples/triplet/mnist_siamese_test_leveldb + +$EXAMPLES/convert_mnist_triplet_data.bin \ + $DATA/train-images-idx3-ubyte \ + $DATA/train-labels-idx1-ubyte \ + ./examples/triplet/mnist_triplet_train_leveldb +$EXAMPLES/convert_mnist_triplet_data.bin \ + $DATA/t10k-images-idx3-ubyte \ + $DATA/t10k-labels-idx1-ubyte \ + ./examples/triplet/mnist_triplet_test_leveldb + +echo "Done." diff --git a/examples/triplet/mnist_siamese.ipynb b/examples/triplet/mnist_siamese.ipynb new file mode 100644 index 00000000000..8e076663ca6 --- /dev/null +++ b/examples/triplet/mnist_siamese.ipynb @@ -0,0 +1,154 @@ +{ + "metadata": { + "description": "Extracting features and plotting the Siamese network embedding.", + "example_name": "Siamese network embedding", + "include_in_docs": true, + "priority": 6, + "signature": "sha256:845bb18929f96543ba2611eb5eca744fd98939cbef876df6bc319c29f616fc64" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setup\n", + "\n", + "Import Caffe and the usual modules." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "# Make sure that caffe is on the python path:\n", + "caffe_root = '../../' # this file is expected to be in {caffe_root}/examples/siamese\n", + "import sys\n", + "sys.path.insert(0, caffe_root + 'python')\n", + "\n", + "import caffe" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load the trained net\n", + "\n", + "Load the model definition and weights and set to CPU mode TEST phase computation with input scaling." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "MODEL_FILE = 'mnist_siamese.prototxt'\n", + "# decrease if you want to preview during training\n", + "PRETRAINED_FILE = 'mnist_siamese_iter_50000.caffemodel' \n", + "caffe.set_mode_cpu()\n", + "net = caffe.Net(MODEL_FILE, PRETRAINED_FILE, caffe.TEST)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load some MNIST test data" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "TEST_DATA_FILE = '../../data/mnist/t10k-images-idx3-ubyte'\n", + "TEST_LABEL_FILE = '../../data/mnist/t10k-labels-idx1-ubyte'\n", + "n = 10000\n", + "\n", + "with open(TEST_DATA_FILE, 'rb') as f:\n", + " f.read(16) # skip the header\n", + " raw_data = np.fromstring(f.read(n * 28*28), dtype=np.uint8)\n", + "\n", + "with open(TEST_LABEL_FILE, 'rb') as f:\n", + " f.read(8) # skip the header\n", + " labels = np.fromstring(f.read(n), dtype=np.uint8)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate the Siamese features" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "# reshape and preprocess\n", + "caffe_in = raw_data.reshape(n, 1, 28, 28) * 0.00390625 # manually scale data instead of using `caffe.io.Transformer`\n", + "out = net.forward_all(data=caffe_in)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualize the learned Siamese embedding" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "feat = out['feat']\n", + "f = plt.figure(figsize=(16,9))\n", + "c = ['#ff0000', '#ffff00', '#00ff00', '#00ffff', '#0000ff', \n", + " '#ff00ff', '#990000', '#999900', '#009900', '#009999']\n", + "for i in range(10):\n", + " plt.plot(feat[labels==i,0].flatten(), feat[labels==i,1].flatten(), '.', c=c[i])\n", + "plt.legend(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'])\n", + "plt.grid()\n", + "plt.show()" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "display_data", + "png": "iVBORw0KGgoAAAANSUhEUgAAA54AAAIXCAYAAAD0R4FDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXtwXOWZr/usvurWUktqGdmxaawEHEMuthGXITiIyMaJ\nwbEMFmCTDMkkoyqSyTnZMwdqpmYyzEyS2ruKue2ZqSTHO/vYGQbhCxdjwI637ViWMEEEMJhgB4MB\ngSRLsizJkiypuyX1+WP1Wlp971YvSd3y+1S5rF69Lt/6+lOrf/2+v/dVgsEggiAIgiAIgiAIgjBT\nWOZ6AIIgCIIgCIIgCML8RoSnIAiCIAiCIAiCMKOI8BQEQRAEQRAEQRBmFBGegiAIgiAIgiAIwowi\nwlMQBEEQBEEQBEGYUUR4CoIgCIIgCIIgCDNKRsJTUZQ8RVFaFUV5U1GUU4qi/HezBiYIgiAIgiAI\ngiDMD5RM+3gqilIQDAZHFEWxAS8B/08wGHzJlNEJgiAIgiAIgiAIOU/GqbbBYHAk9KMDsAJ9mZ5T\nEARBEARBEARBmD9kLDwVRbEoivIm0A0cDQaDpzIfliAIgiAIgiAIgjBfMCPiORkMBlcAi4EvK4pS\nk/GoBEEQBEEQBEEQhHmDzawTBYPBi4qivAhUA03adkVRMjORCoIgCIIgCIIgCFlNMBhUEj2fkfBU\nFMUDjAeDwQFFUfKBtcDfxxhEJpcRhDC+9a1vsWPHjrkehjCPkDUlmImsJ8FsZE0JZiNrSjAbRUmo\nOYHMI54LgV8pimJBTdt9PBgMHsnwnIIgCIIgCIIgCMI8IiPhGQwG3wZWmTQWQUiJq666aq6HIMwz\nZE0JZiLrSTAbWVOC2ciaEuaCjIsLCcJsU1NTM9dDEOYZsqYEM5H1JJiNrCnBbGRNCXOBCE9BEARB\nEARBEARhRjGtqq0gCIIgCIIgCIIQTSrFd3KF6RaOVWa64qyiKEGpaisIgiAIgiAIwuWKoijzotNH\nvPsIbU+oriXVVhAEQRAEQRAEQZhRRHgKOUdTU9NcD0GYZ8iaEsxE1pNgNrKmBLORNSXMBSI8BUEQ\nBEEQBEEQhBlFPJ6CIAiCIAiCIAgziHg8JeIpCIIgCIIgCIJwWdPX18emTZsoKiriqquu4sknnzT9\nGiI8hZxDfAmC2ciaEsxE1pNgNrKmBLORNSVE8v3vf5+8vDx6enp44okneOihhzh16pSp1xDhKQiC\nIAiCIAiCcJly6dIlnnnmGX784x9TUFDAl770JTZu3Mjjjz9u6nXE4ykIgiAIgiAIgjCDJPV4NjTA\nmTNQUACNjeB2p3eBDI4/ceIEt956K5cuXdK3/fM//zNNTU3s27cvpfsQj6cgCIIgCIIgCEK2c+YM\nHDsGBw6oInIWjx8eHqa4uDhsm8vlYmhoKP1xJECEp5BziC9BMBtZU4KZyHoSzEbWlGA2sqaykIIC\n9f/qati2bVaPLyoqYnBwMGzbxYsXcblc6Y8jASI8BUEQBEEQBEEQ5pLGRqivh0OH0k+zzfD4a665\nhvHxcd5//31921tvvcXnPve59MeRAPF4CoIgCIIgCIIgzCDZ3sdzy5YtKIrCL3/5S9544w3uuusu\nfvvb37J8+fKw/cTjKQiCIAiCIAiCIEyLn/3sZ4yOjrJgwQK+8Y1v8Itf/CJKdGaKCE8h5xBfgmA2\nsqYEM5H1JJiNrCnBbGRNCZGUlpby7LPPMjw8zEcffcT9999v+jVEeAqCIAiCIAiCIAgzing8BUEQ\nBEEQBEEQZpBs93iming8BUEQBEEQBEEQhKxFhKeQc4gvQTAbWVOCmch6EsxG1pRgNrKmhLlAhKcg\nCIIgCIIgCIIwo4jHUxAEQRAEQRAEYQYRj6dEPAVBEARBEARBEIQZRoSnkHOIL0EwG1lTgpnIehLM\nRtaUYDaypoS5QISnIAiCIAiCIAiCMKOIx1MQBEEQBEEQBGEGyWaP53/8x3+wY8cOfv/737Nlyxa2\nb98ed99MPJ62zIcqCIIgCIIgCIIg5CKf+tSn+NGPfsTBgwcZHR2dsetIqq2Qc4gvQTAbWVOCmch6\nEsxG1pRgNrKmBCObNm1i48aNlJeXz+h1RHgKgiAIgiAIgiDMKQ1ADbAeGJiD45nxVGDxeAqCIAiC\nIAiCIMwgyT2eNcCx0M/1wO40r5Dp8fCjH/2I9vZ28XgKgiAIgiBoNDc3MDBwBputgNraRpxO91wP\nSRAEIQMKQv9XA9vm4PiZj3hKqq2Qc4gvQTAbWVOCmch6mh0GBs7Q1XWM9vYDtLQ0zPVwZhRZU4LZ\nyJrKRhpRI5WHgOl8kZbp8WrUciaRiKcgCIIgCDmHzaZ+u+/xVLN69fS+3RcEQcge3EwnPdaM4ycm\nJggEAoyPjzMxMYHP58Nms2G1WjMYTzTi8RQEQRAEIefw+QZoaWlg9eptkmYrCELWk819PP/u7/6O\nf/iHf4ja9rd/+7dR+2bi8RThKQiCIAiCIAiCMINks/BMh0yEp3g8hZxDfAmC2ciaEswkcj01Nzew\nb18N+/evx+ebXon7TMmGMQjTR96jBLORNSXMBSI8BUEQBGEGyYYiONkwBkEQBOHyRlJtBUEQBGEG\n2b9/Pe3tB/B4qrnzzkNz4kfMhjEIgiBczkiqrQhPQRAEQZhRsqEITjaMQRAE4XJGhKek2go5iPgS\nBLORNSWYSeR6cjrdrFmze04FnxljEJ/o3CHvUYLZyJoS5gLp4ykIgiAIOUZzcwMDA2ew2QqorW2c\nFVGr+UQBWloaWLMmk35zU8zFvQiCIAizj6TaCoIgCEKOsW9fjS4Cq6rqTROBiUjHJ5qOmJyLexEE\nQZhtJNVWUm0FQRAEIeew2QoA8HiqWb16W9hzM5USW1vbSFVVfUrFidKpopvoXgRBEIT5gwhPIecQ\nX4JgNrKmBDOZjfWUSATOVOuUdHyi6YjJdATt5Yq8RwlmI2tKmAtEeAqCIAhCDmCMZAJxRWA2RBDz\n8ytwOj0pCclsKL4kCIJwOeP3+/nOd77DVVddRXFxMStXruTXv/616dcRj6cgCIJw2ZMLBW5S9UJm\nQ+sU8W0KgiCEk80ez5GRER577DG+/e1vc+WVV/Liiy+yZcsW3n77bbxeb9i+mXg8paqtIAiCcNkz\nUxVbzSTVSKYWQYyFWQI72XmyIeqqkQtfKgiCIMwlBQUFPProo/rjO++8k6VLl/LGG29ECc9MkFRb\nIecQX4JgNrKmBDOF0kytJzO8kGb5P5OdJ5t8mzPleZ1N5D1KMBtZU9lHAw3UUMN61jNA+oXhMj3e\nSHd3N2fOnOG6667L6DyRSMRTEARBuOyprW2c8/TUZLS2PsLISA9HjmxNO3KnRf36+98BUhPYiSKF\nyYR6oqhrJuza9VlGRrqwWOzcffdruFzJv4nPpuirIAhCPM5whmOomTcNNLCb9N5DMz1eIxAI8MAD\nD/Ctb32La665ZlrniId4PAVBEAQhy4gl+tLxTUYef/BgnX5sYeFiNm9+O6lwTXS9ufKRbt/uJhC4\nCKj38cADnyQ9Jhs8r4IgCMk8nutZzwEOUE01hziEm/TerzI9HmBycpKtW7cyPDzMc889h9VqTfk+\nxOMpCIIgCFlMvKhiLM9prMhdqscbj001/TVRpHCmIprJsFjsAFitBXz96y/F3U+bl6GhsxQWenE4\nimdriIIgCNOikUYaaGAb26YlGjM9PhgM8p3vfIfz58+zf//+mKIzU8TjKeQc4ksQzEbWlADh7Up8\nvun7Y9JZT/H8h4ODZ0M/WRkd7cHnG4jpm4x3fKRonI7nMpt8mhp33/0ahYWLuffeUwnTbLV5uXSp\nnZ6e4znt7wR5jxLMR9ZU9uHGzW52T0s0mnH8Qw89xB/+8Af27duH0+mc1jmSIcJTEARBuGxIJC7n\nogiNJjDt9mJuuukxffvkpD/00wTnzh2jpaUhZr/LeFHJSNE4nV6Z2dhf0+Xy8sADnyT1dmrzYreX\nAOLvFARBSERbWxvbtm3jrbfeorKyEpfLhcvl4sknnzT1OuLxFARBEC4bEvkW9+9fT3v7gbTSUdNF\nSwEdHDxLMBhgdLQXmIgaz44dZfj9/YAqnrZu/SjmeDL1L2baaiRbW5Vo83LTTY/R2vqw+DsFQZhz\nsrmPZzpk4vEU4SkIgiBcNiQSl7NRhMYofI1YLA48nhtwOIqprW3k0KF6OjsPY7eXsGTJ1xgZOYfN\nVsDg4HuMjp5PqaprKqIwlhBvbm6gre15JiZ8eDzXs3btnrjzkU7BI0EQhMsZEZ6SaivkIOJLEMxG\n1tTlQyLfolmppYnW01QK6FSxG4fDTVnZyjAv4tq1e6iqqmfr1o84d65JTwEeHPyIQOAiPl8vu3Zd\nE9eP2tzcwNmzu5OmDsdK1R0YOMPoaBd+fz+dnYdpaWngf/0vB9u2KWzbZuHcuZcSHp8uZnlr5zPy\nHiWYjawpYS4Q4SkIgiBcNqQjLmdCEGnCd/Pmk3i9G/F669iy5UPy8soAVcBZrfns3r2c9vbDHDpU\nz8TEmOEM4/pPk5P+uKJyYOCM3nYELHz88UF+9asKhobawu7twoW3cDrLcTiifaMAZWUrWL16G8Fg\nILQlyPPP36afIxAYJD+/krVrn5q2YJ8Lb60gCIIw+0iqrSAIgpDVzJWPcDbTSI8efZCPP96Px7OC\nCxdOMjbWoz+nKHaD8FOxWBxMTvqx20vYvPktPeVWm6uenleYnPShKDYsljwmJoaBqd6XsVJ+tXv0\n+QZoavo2EKSmZgdOp5tt2yyA+rd8w4YWFi681bT5mQ1vrSAIwlwjqbbSx1MQBEHIcmL1tJwNUk0j\nzUQYa8f29Z3E7++no+MwimIP20cVnQqa8AO1yTdAIHCRZ56ppqLiBmprG8PmSj12XK+Qa7UWYLE4\n2L7dzfj4SNg1HI5SrNZ89u2riXkfGzY08/zzt7FhwzEWLrw1rflJRm1t44x7awVBEIS5RyKeQs7R\n1NRETU3NXA9DmEfImspuzI6IpSoUUy02ZIz8uVxL+eCDYlauXER+vprammpxH1BFnN1eyLlz0QWI\nNCyWPK644ibOnTuGzVbE+LgazayqqsfvH6a9/YC+r93uprj40/T3/55Nm15l374vG1JwVRTFxt13\nv87x4/+XPpaioqUUFV2ZcOyx5idbq9zmOvIeJZiNrKnZRyKe4vEUBEEQspxEBYESEc+jmaqnMFU/\nqDHyV1CwiL6+t2hvP8DHHx9IubhPWdkKvN46Skuvpb//dNR+ZWVf4FOfWktBwSKuuqqOyclx8vMr\nqai4PnSeIsbG+lm9+udhxwUCA1y48DqTkz5eeGGNIdJpwWp1AWpU9MUX12G1qpFWRbExPNyWdOxO\npxuHw83Bg3X6HItfUxAEQYiHRDwFQRCEeYkxmpifX8m9957G6XSbHkE1Rv6OHNmqn9vhcNPZeRib\nrYgrrriZNWvUtiS7dn2WkZEuLBY7d911hBMnfqJHDR9/fCGjo11h53c4SrHZ8pmcHGdyMkAgMGzw\nfNqwWCx6Oq3XW0db296oMRojo/HwejfS1fUyPt/5sGtv2fJBVERzaOgsExMBxsbC+5BqEdd05lai\npIIgXA5IxFOEpyAIgjBP0QSmhrF4jlmeQqNoys+v4OLFs1y48DplZV/Ebndx4cIJfL4LAOTlVeJw\nuBgcfE8/vrBwMUuWfE0/R0/Pb/H7p19B1+vdSFvb88Ckvq2gYBFu97V0dh4O29fpLMfvv0gwOE5Z\n2Qo2bDiqC2dQ27zcc8+bnDjxU318vb2vhxU+0tAEKpD23EovUEEQLgdEeIrwFHIQ8SUIZiNran7i\n8w2we/dyRke7oiJwZkXZjKLJ6fTg8/Xy7ruwbJn6fH5+ZVgEU1FsBINTLVEWLryNiYlxenqOT/Mu\nw1Er4E6iRSGt1gIqK2/hy1/+Jbt3f5aJiTHsdheVlas5f/41XUTabIWhCrg2PJ5V2GyF9Pf/ntHR\n84yPXzKMObzIkXbNpUvvYWTk3LTmczaq2uZ6VFXeowSzkTU1+2S78PzGN77BkSNHuHTpEh6Ph+98\n5zv89V//ddR+4vEUBEEQhAicTjf33ns6pj/ULC+i0d/p8awIe87hKGXhwhq9Sm1Z2QocjpKwfc6d\nO0ZPz8sJrpDen2k1BVcVnYpiZ2JihI6Ow7S2Poz2OSEQGAJgcnKqRcv4+BiBwEV8vgt0dBzmllv+\nldHR8wQCF8OEcqTotNuLuf/+9xgZOTft+dQ8vKWl14b5Rc1EvKeCIAiJ+au/+is+/PBDBgcHOXDg\nAP/+7//Or3/9a1OvIcJTyDnkGzrBbGRNzV/iFQjKtBWIVrgoGAxQVOTFanUCasXZZcvUyOaGDU2M\njJzT/ZiXLn1CWdnnKChYRH5+ZehMViLF3BQKxpRZu70ktC05ijLVLc1ud3HTTY8xOTmmb+vtfQOP\nRy1MVFa2ImJ+guzbdysWiyqYrdYCnE4PAOXlK1myZD1LlqynqMhLWdnnaWl5SN83cj7jFXgyor1G\nQ0PJCxpNF7Nav8wV8h4lmI2sKSGS6667jry8PP2xzWZjwYIFpl5DhKcgCIJw2THdSrkaWgSto+Mw\ngcAluruP09FxGLu9EFArxf6f/1NHX99J/Rif7wLnzh2jouIGios/Hdo6keAqU4JUUewsWfI1Uv2z\nHQyO64I3EBjihRduD3ve41nF2rV7qKqqZ8OGo9x99+toolZRrIyP+5icDKAoDurqfsv9979HUdFS\nrNYC+vpO4vNdxO8fpLv7OO3tB7DbizKOLM+kOMz09RYEQZhpGoAaYD0wnZyPTI8H+N73vkdhYSHX\nXXcdf/M3f8OqVaumeabYiMdTyDnElyCYjawpIV2MvkSn001Hx2G9ku3Ro4f50peqsVqddHdHezeX\nLFnPhQsnGRlpT+laDoebxYu/yocfPm2oZps6dnsJZWWf08eiKHYqKm7k/PlXCAaDKIqVu+9+DYej\nhH37bqWg4FOcP9+qH+90VlBRUY3fPxjTi5rIm5mOfzPdok/NzQ20tT3PxIQPj+d61q7dM29FpbxH\nCWYja2r2SebxrAG0Ds71QLpl1jI9XiMYDHLs2DE2b97M/v37ufHGG8Oez8TjaUv0pCAIgiBc7kRW\nrh0aasNqteP1bqSmZgcwVckV4N1367jzzr0cObI15vk6OtQWK6mgKDbKy79IZ+dvpiU6QSEYnKSn\n51VA9Z0WF18TJiCDwUmefnolDz54gQce+IT9+9eHHe/znae9/QB5eZVEUlCwKGHRptraRp55ZhVW\nq5MjR7YmLOyjpdymysDAGb1wU2fnYVpaGnA43DldREgQhMuXgtD/1cB0cj4yPV5DURRqamqor6/n\nySefjBKemSART0EQBEEwoImnwcGzuFxe+vtP4ff3A1rVWFUAxmr9YRReq1f/nGefvZmxsa6oa4Rj\nBSawWBx6P04zyMtbwPj4GOPjg/p1Fiy4md7e15ic9EWPwppPefkqbLZ8rFYH3d2t+P1qKxiHo5R7\n7jnBM8/coPf5dDjcbNnyYZi4i9UaxbitqGgpRUVXRgnDVKrORu5jbP2itYM5eLBOWrMIgpCVJIt4\nDqCmy24DpvOVWabHR/Ld736XyspKfvKTn4Rtl6q2giAIgmASmi9xZKSd7u7juugEdNEZz4do9DS2\ntj7Mffed1gvzxGPDhiZcrqVYrXkJ90uXsbEeg+i0UF6+gp6e4zFFJ8DExCg9Pcfp7DxMd/crKIr6\n+cFicVBScg0tLQ9RXv5FQBOib6ZUtMm4raBgUUzPZype0Mh9amsb8Xrr8Ho3smHDUZxOd84XERLm\nK2a474T5jhs1PXa6ojGT48+fP8/OnTu5dOkSExMTHDx4kD179rBx48ZpjiY2IjyFnKOpqWmuhyDM\nM2RNXd5EVl7VxIvdXhy1b1nZCrzeurh+RZutgHffnRI+TqebioobEl7/nXf+jcLCKwkEBhPulxmT\n9PefSnlvv78Pn68Xi8VJeflKzp9v1YsIuVxLKS29lpaWh6Iq1cYq4mPc5nCocxopDCMFo/aaPPHE\nEvbuvZX9+9dHVc51Ot2sW/cs69btjXmt+ZRmK+9Ruc4ZVPfdAVQROvfImhKMKIrCL37xCxYvXkx5\neTk/+tGPePzxx7nhhsR/v9JFPJ6CIAhCzpFKamaqaJE0QI+ktbQ0cNNNj7F793ImJkax211UVq7m\nK195Iupakem1p08/SGmpl4MH6/Rte/fezOhoFzZbEePjw/qximJncPBjhobOTnv8qRLej1NDTfMF\nsNkKGR+/hN3uMvT69DE01AaA0+lhdPQ8Pt8AQ0MfAvDMM6soLAxPnXU43Pq9a55YTVhqcxtZQChy\nu/E1uXRJLcLk9dZRVLQUiyW+XzRdn6ggzA5mue8EYWbweDyz8mVERh5PRVGWAP8JLECt+74tGAz+\nW8Q+4vEUBEEQTCWWl3C6JKq8unfvrXohHqfTQ0XFDdTWNtLa+oguNgOBQb1irOZh7Os7qafoVlXV\nY7Xm8/HHBwgEBuOmuloszrjPzSRWax6LF3+VW275V1pbH+ammx4LE8oVFdczOPghPt+AIXUXbLYi\nJiZ8evqx11vHunXPhr02TmeF7gnNy6vkvvtOh81dvC8NtNfEbi8hELiovzbi4RRyE7Pdd0Iukszj\nmSvMZVXbAPDfgsHgm4qiFAGvK4pyKBgMns7wvIIgCIIQFzO9fPn5FTidnpgCSEsNtdmK8Pl6dX/h\nyEiPLoDy8yv1sVgsTn07qGJ1eLiTgYFTYV7RWMy86JyKbhqpqLiJmprtYdHCe+89zeOPVzI+Psy5\nc8dwOsvDRCcQFrkF9MJIxtfG4XDT2XkYgLGxrqi5i1eJ1hh1bm19GKs1n927lzM21guoKc/i4RRy\nB819l000AM8DPuB6YA8iioWZJiOPZzAY7AoGg2+Gfh4GTgOLzBiYIMRDfAmC2ciayj3M9PINDbXh\n8/XS0XE4qrCNdp0rrrgZmBK6mrjS2qIoioOLF9/j4sU/8O67YLUWkpe3AL9/iJ6e44aquOr3vQ6H\nO2nRIQ2bzWV4pKAKyPQxFt8x0tV1TL9vzVt55MhWrNb8qasqqV/T+NqsXbtHb8MSOXfaY2PRoJ07\nr9HbuaxZsxuXy8uaNbsZGmpjdLRLTxd2ua6aVx7OZMh7lGA2TU2vAl1AP3CYbPGeCvMb0zyeiqJc\nBawEWhPvKQiCIAiZYaaXL1H0VLuOzzcQ5kHMz69AUWyMjw/rkb9AQI34KYqViYlLTExcirpWMDiO\nzVaA232d7pNMRFnZ9fT1nTCegVhRy1Tw+S7E3K4oNnp732T7dneowJGaQuVweEJjWMG6dXtpbX2Y\njo7f4PNdwGrNZ9Gi22lvP6Sn2p4//zt8voGo1+a++06HzV2kn9Mo4rWeoTt3XkNFRbUeATV6YMvK\nvkBNzfZpzYEgCBpOw88rEe+pMBuY0sczlGbbBPwkGAzujXgu+OCDD3LVVVcB4Ha7WbFiBTU1NcDU\nt3jyWB7LY3ksj+XxXDz+oz9aQUtLA5OTf4zDURT1vMXSyMDAGVpb3yE/v5JVq5YQCAzS3Kz6Opct\nA4B331X//+IXFzA21kNX1zWMjHRSVTWsP2+x5HH11T4gqO8febz2+IMPigkEBuM+b9bjlSsXMTLS\nzbvvToQ9399/EzZbHn/2Z2rV2KamJkZGuujvf5ivf/0lXn/9Q158cS1XXz2un2/Rotv4i79omtb8\n/+53Z+ntfYNlyyzAZNj59u69lZYWdb7vuGMj69btzZr1I4/ny+O7gHZqahYBjTQ1vZll44v1+B+p\nqRkGCmhq+h4Q/f4V//ELwP+gpqYC2J4j95vbj2+//fZ54/E8evQob775JgMDamXzjz76iF/96ldJ\nPZ4ZC09FUezAC8CBYDD4rzGel+JCgiAIQk7S3NzA2bO7CQQuhm3Pz69kdLQLh6OU0tJr9eJCpaWf\nZ3z8UiiaacHjqaav74Tuf0wVq1Ut6mP0i84csb2fDkcpixevY2TkXNxCQGqU9GJofzdbtnw47RRY\nn2+AnTuv0YsRORylbNnyAU6nO2EBqHQwsxqyMF9oQG13chI17RSgnuzzZMaiBrVNC+TOmC9fpLhQ\nhh5PRe0u/b+BU7FEpyDMBNq3SIJgFrKmhHgMDJzRhZXmz/R4qqmre4Wqqnq2bPmAr371BbzeOgoL\nr2R0tIfXXvsALSW2t7cVu10VN1N9QRP+XQZgYmKYrq7mNEdr0ceYHrFTd/3+fj74YLfuv2xq+nbU\nPgsWqD3eHA4399zzZlwhF9krNRZq2q2W/qdQXFzFkSNb8fkGTPP0Gv2kkX7ebEbeo2YSrcemJjpz\nqeXJ9Nu0TG9NNaCK3fWolXoFIT0yEp7Al4BvALcrinIi9O+rJoxLEARBEOYEo0gaHHwPUEXnXXcd\n1cWPVvTG6XTjdLpZt+5ZJif9jI11R51vwYJqqqrq2bz5JFVV9TgcpWHPxxeL6XwzbmHLlg/Iz1+Q\nxjGxiBTFxsfh42lubiAQGAWsTE5O8Mwz1XrPz0hSFXyFhV79Wr29r+v7a77RTCOUZlZDFuYLmnhb\nAdQBh8id6q6NqJHO2RqzJtIPIMWIhOlgiscz4QUk1VYQBEGYYcxMoQzvQ1luKMqjsHDhl7njjr0x\nz//LXzqjUmq1VNzh4TYKC704HMV0d78clbprJP1+nqrodLm8YX1H06WsbAWjo12MjnZFPWe3u9i8\n+W1OnPgpbW3P4/P1MTk5QWS0tLBwMQ888EnU8ammyk7171T9rZmm1kYSWSRKEOa+x6aW6luAKiTN\nHoOZ51+PKjqryS2Bnh1Iqm3mEU9BEARBmBOMkcm+vlOmpVAao2Iez0rDM0HOnTsW9/zGViWKYiMv\nbwElJdfQ3X2cS5fa6ek5Tnv7AaxWZ8zjVSw4naWk2jJFUWxcdVUdR48+yP7967HZ8pMfFPGn32Yr\nZMmS9ZSXf0Hvk2l8DiAQGKK19WEGBs4wOtoVEtjhotNqLeDrX38p5hXVXqkVOByJP6hqKbVadDhS\ndKaSspsIsyKnwnxC67E5V2vCGEW8mvTTWJOlv5oZpZztCKswF7z33nvk5eXxzW9+0/Rzi/AUcg7x\nughmI2u1e9m8AAAgAElEQVQqNzGmbw4Oqu02pptCaRQ0q1f/XBc9a9bsQVEc+n6lpZ9n9eptMQVQ\nRUU1AO+/n0cwOM7YWA+9vW8AasQQ1JYhbvdyLJZ44nMyFHFM1jJFFbb33/8+Y2MX9Hm4cOFE2Hhj\noSjhf/rHxy/R0XGE999v1Ptkqvs59HtSW530Y7Xaw46120vYsKGFwsLF3HvvKVwuL7FQe6Wep7Mz\nvFdq5DxqwtCYymwkVz2amZJb71HiA0wP7QurIqCX9AWiUViuInruY/tAp7em5lqkC7PB97//fW68\n8UbUUj7mYlofT0EQBEGYTYyRybVrn6K19eGUUygjU3M1QQPQ2vpwWB/KpUvv5sMPn8HhKOarX30e\np9Mdtn9LSwMOh5tAYJS8vEqCwT79WK3HpcXixOFw4PerIlEtCK+hkJ6fE0AVti+//H9H9MGM3avT\niCoup65ptRYwMTESYz8/Docbp9ODz9dLR8dhFMXOpz61FovFjsVip6ZmB06nmwce+CRhunM8b2Xk\nPCbrzTpdj6ZUs51NNCEEqoCajUqrM52uaiaRY20MbesHDqMKxHxUAXkW8ALFxL8vTVh6gAuA1h94\nOXDacP65SiUWcomdO3dSWlrKtddey/vvv2/6+SXiKeQcWk8kQTCLy3VNZZq2ONcYK52eOPFTRkZ6\n9Cqoye4tMnKWSNCMjJwjGPTj8/XS2vowEC2ABgbO0NNznLGxLq65ZjLqej5fLxbLlNjUBGnoERZL\n4ihlOMY/3Qr5+RVYLE7Gx0dTOtpmK+aee97EYsnH4SiLm57rcJRSU7ODioobwsbd3/8OX/vai+Tn\nL+DgwTp9jo1zunPnNWFzH68qbbpCcrrVbXM9Uppb71HTr7Q6fXKp6M3zTI3128AjQE/oOa24UVto\nn3bgOInvS0t/tQCDhu1doWNiRymj15REqueahgaoqYH162FgGi9BpscPDg7y6KOP8i//8i8z5kUV\n4SkIwmVBQ3MzNfv2sX7/fgZ86RRumb9k84fxVNtvaOmYkfeiPf7v7eXcuPP/jXrdjYLHas3H7x/E\nas1DUay6eI21ryaOIgWQcR+HowQARbGiJRaVl69k06ZXyM+vDJ3VmMJkxWo1ir/o9Can04PDUUpe\n3gKuuOKPQtvK6e5+mfffbwwVI0qWnhu6mtVJUdGVLFhwI35/X8woqcNRyj33nMDpdFNb2xj2XHn5\nCiC+eFcjr+fD1lWkt1J7fScnA3i9dSkLyel6NKWa7WwyFz7AuRC706EBVRBq+JkSzYcBO+qcafej\nfVlVAjwWca7PAg6gAlW49kc8n+5c5JJ4n5+cOQPHjsGBA6qInO3jf/SjH/Hd736XRYsWzUiaLYjw\nFHKQ3PK6CNnCmYEBjnV1caC9nYaWlrDnLtc1lc0fxtMVxUbRMzY25UXss32ak75S/XXXBE8wGMDr\n3ciddx5iaKiNnp7jTEyMcf58a9Q1a2sbcbmWYrU6dVEaKYCMQrSi4j8oLFxMeXk1oHomh4ba2Lfv\n1lC7kMjU2gm9yq0qQKMLC/l8vfj9/YyN9dDd/dvQtgHGxnrCfJnxcDrLDec6T1PTt8KKIRlRFDsV\nFdfrAtrpdLNw4W2A6nHNy/Owb18N/f3vAFPrR5uDBQtu1rdbrfkxv0DQXt/OzsNYrfYZT301qw/o\nXJFb71Fz4QPMlaI3ZyIe24ktmrX7WRV6fBG4nfCIZBcQQH2POUb4e8oiEs9FA01NK0jFCyrMHgWh\nl6C6GrZN4yXI5Pg333yTI0eO8MMf/hBAIp6CIAiZUGBTI0/VHg/bVq+e49FkB9n8YTxSFCeLgKq+\nvQrGx4fp7DyMzVZIVVU9i69QP7hpr7smeDo6DmO1OsKilXZ7cdg1NZxON+Pjo3R3q1Vpn3zy01Hj\nMArRgoJKHnjgE/LyykL3UoTf38elS+309rYS6ee0Wl36ddUiRPGFpM1WBGipvKlFODdsaGFiIhCx\nVdFf/8g+osFggI6O8CJAd9yxl6qqejyelXz00XN0dR3D5+ulsHCxvn60OVi7do++roaG2vQvEHbv\nXq7P2Wx/6SHVbLMZM1I8s7nojfH+jN7uzwE7mBKZF4Ey1C+mqlAjnGWhfatRxaQxImk81xeAL4V+\nXgGsQU3bjTWnDaHrvhU617LQPrki3ucvjY1QXw+HDoF7Gi9BJscfO3aMjz76iCuvvJKFCxfyT//0\nTzz99NNUV1enP5AESB9PQRBynobmZs4MDFBgs9FYW4vbGV0xdMDno6GlhW2rV8d8XsguIvstGntr\nVlXVxyxCE6tXZOTrHmsf7Vo33fRYVIEirShNd/fxqMhiVVU9Doc7btEa7bw+X79emEf1dlpRRaON\ngoIrmJjw4/cPUFl5C11dL0f4P9V9S0quxe2+hkBAFdbxsNmKGB8fDtvmci1laOjDsG2LFq1h7do9\nUXOrEa9/ZuS+DkcZ99zzBi6XV5+roaGzFBZ6GR5uY3x8GL9/6oOv9tql0k9TCgLlItMp8lPDVDGi\nemanGNFsUsPU/S1AFZG/B5agFg2qQPV0vkT4l0mLgHdQo56LgNcAX+iYk6F9bkEVmk+EHmtFhOoM\n16xELTKkvRbG8WjMx3nPPrK5j+fo6ChDQ0OAGu38x3/8Rz766CN+8YtfUF5eHrZvJn08RXgKgpDz\n1Ozbx7Eu1TdTX1XF7jVr5nhEgtnEEoyRpCJmUtnHSKTQsttdBAJD+jgOHqxLKIibmxvo6zvF4OBZ\nCgs/xYULrwNgsThYuPDLnDt3nMnJ5EWBvN6NrFu3F59vgJ07r8bn68VudxMIDGH8sLpkyXo++WR/\nxNFWYkVHi4qupKhoKRaLnXPnjhEMBrBa81m06Ha+8pUnosS3zVbA5GQgSvhaLE6++c2usLmIRaLX\nLhapfNkQOb54AtW4T35+BUNDbSJoZ4Qa0heR61Ejb9XkVrQtmcjWnn8FVTBqeFCLAPlDj50Rz2tY\nUFusjBCdBVEJ3IEqWGNdX5tTDeNrEfncCuBojPELZpPNwjOSv//7v+fs2bP853/+Z9RzmQhPSbUV\nco7c8roIs0GmabSyprKfVNKCW1sfCatsG4t0Uy61lNCyshV4vXVs3vx23KJCWsqocT0Zq90ODJwK\nbbUyOemno+NwSqLTYrHT3/8O27e72bnzaiorvxxKK76EUVDa7S5uvfVnRBcnMorOqeeGhz/WfZYO\nRwlWawF2exHd3b/l8OF6fQ6Nflu7vYiqqvowz+jkpI9du5brVXvt9pLQ/2rqcnn5St1Pm2zejSnV\n2vmSpeOm4gc27vPxxweytqhWPHLnPWo6PsFcTfE0FuOpRE2LXctUaqtWvdYoKq2ovTr9hm2xfz/V\nlPpBYqfedwFPEr8YUGNoTBD9WjQCG2lqugnYiIhOIRaPPvpoTNGZKSI8BUHIeRpra6mvquLQnXdK\nGu08JRXBmG5Boli+0chtmuAtL/8CPl8/LS0PhUVLkwliTZg6nR6Dl3IixjYj4cWFNm16jdHR8wQC\nF/H5emlrexaf73xESi4EAkO0tj4c8oFGoyhqlDUWPl8vExOjjI2dx+/vD/N4GsV1Tc121qzZzd13\nv47FMvW7NjbWpYvSzZvfCv1/kqqqeu666zesW7c3JbEfKXIjizrFIhW/qHGf8vIvJt1fmC7TEZHZ\n7M+MJJZf04YqLrU+nNp7T6woZnSrpaltVuBOwr2bidB+/2NVvHUDrtDYPkT1jxqf2wv8D9TU30Re\n0Jo4zwnC9JBUW0EQBGFekEo6LkylXfb1ncTvV1sQaKmc8dI7jdudzgoqKqqTpmka02xdLi/nz7cC\n4HCoVWLHxnrp7j6u719evpKioisZH79ER8dhLBY7mza9Rnn5F/jVryrw+XrDzq+l/Uam/2qpuEYs\nlnzuu+80DkcJO3deg893Puz5SG+oxeLA47kBh6OY1at/zvPP305BwSIcjmL9vn2+AXbtWs7YWJd+\n7dbWRzLyZUa+hslSmSH9FGsgrXRr4XLls6iRxTHUdNQy4HWmem6WoqbJjjDVP9OKWn12D6oAj+/H\nVlkGdAJDMZ6zMyUujdiIjoIuCe3rA64PXf8qpgTnYuCTGOeqIX5qdKLnhOmQS6m2iRCPpyAIOUMq\nhYCE7CJXiryk6t+M9G0aRdMHHzyF399PeflKyso+r3sBNW+jUaAl8h1GXic/v5LR0S69P6bL5dVF\nliY4a2p2hBU7Mt7H0FAbu3YtY3LSp+9/yy3/kxdeuJ28vAUMDbWxadMruFxehobaeO65W1AUK4HA\nCH5/H1dccQvFxZ9maKiN/v538Pl6sVjs3Hnnb3jnnX9jbKxf927a7SWUlHw2VIFXjcwGgxNRIj3W\nnCfzZSZbS5HnS/XLhFjkyroVspEG4P8jtcrRsTzUTtRIZjD0L955FFRBG91LN1pg5qOmMmuCNNYx\nGvWoKbS9ofFVAx2AF7U4keYJTeSvzVXvbfYiwlNSbYUcJHe8LkIsEvXTnCuycU01NDdTs28f6/fv\nZ8AXK2Vr9kg3hXWuSNW/Genb1CvgDpzRxVVR0ZVhrUD6+k7i9W7Ue1Rq/UJjpX9q68mY3llX9wpV\nVfVs2fIBLpcXmErTjUxFjXUfJ078FIejBEWx43AU43CUcPTog/h8A5w/38rYWBetrQ8D4HJ5+cY3\nOnC5qvD7LwBBuruP09b2ot4GxWJxct9977Fw4a16CxSvdyNebx1bt36kt4IBi95DVLuXyFYzxrEm\nS3uNtZaM6c1A3P6o6QrHXFm3qZCN71HzD2Nq6SkSi04tHd5D7PRZH1M9NhOdJxjneIiOavpRxWYX\niUXnClRP52uokc5qoBVoB46jeULVNZUoNTpXvbdCNiPCUxCEWUX6aaZGNgn02e65ONNoYmbDhqOs\nW/dsTNFUU7NDfwwwNtaD1eoItSCZ6heaSNDU1jbici3l0qVPePrplfh8/WHPp1PoaGDgDGNjPQSD\nAc6dO8b77z9JV9exuIJQTfM9GXYOi2XKOzo56dOFqjaWdev2sm7ds7S2PkIgMIii2NE+FCuKjSVL\n1idNYfb7B8nLq2Tt2qcSel6N400kEDPpvznf1q0w0xiLBZ1Nsm9F6N9FIvvypk9/jG2xgkbJoq/F\nqKL5C6i+zYeAt5nqBVoS+t9YbCiRvzaXvLdCriCptoIgzCrSTzM11u/fz4H2dqo9njkvmpRuC5Jc\nITIVE6a8f62tj9DXd4re3t8xOekPS/VMJ/0zMq03WXpuvPRQ7ZqRlJWtwO/vZ3x8lMnJABUV11NQ\nsIiPPnqOQGCqoEh5+UruuONZdu1azuTkKIpix+NZhdNZFtVeJF5bFK2lS7wxx/LMRhJrLaXrzU01\ndXa+rlshVRK1O9GeO8tU+mkA1ZPpCe0T7pMOJ57/ci7ZiFo0qIZwb+Y21Pt9DHg49Fh+H+YCSbUV\n4SkIgpCViECfeeL5EZubGzh7drcu3AoLF7N589u6eElH0BgFY3n5Su666zcpC9W8vEruu++07vts\navo23d0vMzbWE+YLjRSKTqdHLy5kt5ewaNHt1NRsp7X1Ec6e3UUgMBh2TaezQi825HRWAMGo4kQA\nXm8d69Y9m3DMkH6/zul4c5MJeCPi9cxmkvXCzIQawgWY23CtQdS0UyN1qIKyM8ZzELuoT7bgQo1u\n/hR4CjWKWgTcjFpoSNZ8NiDCU1JthRxEvC5CLDLxRJq5pszyZrqdTnavWSOic4YwpqKWla0IS8Uc\nGDiji06HozRMdELy9E9tPWmpp07nApYsWZ9UdAIR6b1d7Nz5Gd37uG7ds9x337u4XEux2QqYmPBH\nHVNevhKPZ4Vh7G/p6cTqfamiU2vjYrMVhQlRn++87gEFtXKudt6amu0Jx2zs19na+khUq5p4pOvN\nTTd1dj54Pefv3z1jeqtZr43m1Xwn9FhLLTVe69XQcy7DPsWoFWtfi3FOK9kpOrXWK0Oo0cwzTKXu\nDhPe3iWc+bumhGxGhKcgCPOCbPFEZss4hMQYCwmNjHSGPacJHK0C7XQjZAMDZ+jpOY7P14PdXpjS\neWprG0PeShWf7wLt7QfYufMaXYAWFl5Jd/dxfXswGGDJkvV4vXXcdddvWLNmj17IaP/+dWzf7uZX\nv6pACX0P7XCUcvfdr+N0ehgfH2ZyMvwLEoejlPvuezfUi/NtvQBSvPFrntmyss/j8w1w5MhW+vtP\nZdxTNd510i00NJ+8nqnMU26hfWli9B1miiYwe1GL62jFcbRrFTGVJrsaNRp6LfBc6LhYXximUt12\nJlmAGnE14gZuC/2szZ92j8UR2wUhO5BUW0EQ5gXZ4onMlnEIiYn0TBYVLaWo6EpstgJWr/45ra0P\nZ+wNnG4rkBdfXEtHx2G9P2dkCxe/f5j29gMptXbZvt2tR28VxU5eXjl1da+EtXMxoig27r//fb3y\nbjoYU2Gt1nwmJkax24vZvPlk0vNNN402FeaT13Mm52luGECNyJnpO4zXBkS7Vj9qJND4fA1TabnZ\nSDmq8OwOPbYCbwBXEj5/2j2KnzMbyfZU25qaGlpbW7GFikAuXryY06dPR+0nHk9BEC57ssUTmS3j\nEBLj8w2wa9dyxsa68HiqsVic9PSovi6zPtD7fAM888wqCgoWYbcXR/kLY3kPm5sb6O8/RW/vG5SU\nXMvISAelpcs4d+6YLmBBLYKk9d50Oj1YLFYmJvxUVFzPmjV7aG19hIGBM3R3v0wwGECtkhkMuz/j\nHIDqB928+a24IjFyvNo1tMdHjmzVhbaiWDl/vjXl+cykX+flhMyTRiJvaDIxG/l8A1O+yGxm6ndY\nZSmq8JwJf6wwE2S78Lz99tv55je/yZ/8yZ8k3E88nsJlhfgShFhk4onMZE1FejrFm5kbOJ1u7rvv\ntJ666XCoqWlmpGNq68npdIelxUamnMbyHqpi8TgTE6P09b3O2FgXfX2/Jz+/kuLiz3DwYB1Hjmxl\n9eptrF2rptS63csYHe3G7++no0Nt8aKdOxgMYLXmsXDhl6PuT5sDr3cjRUVeyso+R0vLQ3FTOCPH\nG/k4PBW2LK35zKRfpxGzUlGzqY8uTK0ps+YpOzH20Uz22kV6Q43HQuI2IMY2IQ2hn7NddFoJF502\n1F6eifyxiedTPksJsZhpYRyZMC4IgnDZ8Y9vvcXfDQ5SYLPRWFsbJRobmps5MzAQ8/nn29roGh0F\n4NtNTTy7bt2sjj1byYVKolpRG1A/0E8nHVO7z8HBs7hcXuz2Ymy27+nPJ/IXDg6qvQLt9mJuuumx\niN6bFmASq7UQn09tFt/RcUSvPtvS0oDD4ebcuRbGxqYq0JaXr2T16m0cObJVv64xShp5f1r/TmMK\nZ0tLQ8wIZeS97Nnz+bDxZzKfxmMj5zadNaSJ4UT3kQqaVxugoaWF3WvWTOs8ZhNrnsxhJqvLpoom\nJrXxJLrPSG9oXZJjY7VPaQSeR+3Fma0UovbfXAK0GrYXMSUmS4nt40xnPoWsINNfQxN+jf/qr/6K\nv/zLv2TZsmX89Kc/5bbbbkt+UBpIqq0gCFlJPLGXSATGe17bdnZwEK/LRbHdHnZszb59+ofM+qoq\ndq9ZE3aewUCA492qt6YyP5/T996rH1u2Ywf9frW66JVFRfgnJvBNTHC9x8OetWsv28inUcg4nRVU\nVFRnrQBNF6Mg+tfea/loLIgDP9/lf1PAaFhqqeYvtFrzw3plOp1unnvuVrq7p9J7R0Z6ovpn5uUt\nYGysB4+nGofDTWfnYTyeakpLr43q1VlQsIj6+nf09iuJhF+kqDOmyRqjacb9Ir2vkeM3QxAZr+f3\nD6ad/qylojqdHkpKluFwRKc4p8L892pHfkI1Crd65kakxPNmQvR4tW1aumyyY3cQ3XfTgxo1zObP\nqG7gj4C3UNu8aCxArcBbCpxAFdORaHPiAZYxJbZz/z04V0maaltDZr+GGR7/6quvct111+FwOHjy\nySf5sz/7M958802qqqrC9pNUW0EQ5h3xqsMat6965pmodLhYx2nb2kdGON7dHXXOgpCRvtrjYdvq\n1VHnOTs41W6ia3Q07NjrPWqz8UKrlUG/n67RUfr9fg53dl7WVW216JjNVoTPd35GWllkklaZybHG\nFNOPfXbeYxnv8Hn+i29ERTa1CNXQUFtUWq3dHp7eq82Z3V6ib9+06VU9tVJLrb3zzkMMDbWFic7y\n8pW66DReN57oPHt2d4I02aljjPfa2vpw2Dkjx28GxutpEeF0zq/dR0nJMnp6Yqc4p0JjbS31VVXz\nVHRCdKpq8uqyxt+Zo0cfnIHquo2on5YjhWOs8WrpsjeHfn4VVWjFEp27iRadCmrV27kUnZH3GOvz\n+gDqPY8ZtpWg3m898AGxRSdMzacVtS/pAeBb0x+uMPNkWuQ5w+NvvPFGCgsLsdvt/PEf/zFf+tKX\n2L9//zQGEh8RnkLOIb6E+UmkpyqWGIRwkbiooCBKZGrPF9ls9Pt8YefSKHU4ws75PZst6kOm8Tqv\n1NVRmZ8fczx71q7F43RyaWKCgVDkE2BFWVnYfmbMyVwxnXFoAmDBgpuBmWllkUl/xkyONaacXll5\nAwCryor5G+8wd955iN/+9s2Ex2jzECn2tMebN79FX1U9P7/zEPe5vFSHxJ5RTE61fHGn3CPUeO/G\nPqVaBDOWUE2UKjwTfkPj9TZteiXt82v3kalnd7a92sm+CDH/717kJ9REok8l7AuXj/fPQG/UR1Cj\neFuZSiON15NTows1VfYCcDLG2OOl0mZDlDPydS6JeKwJ0ULDz6XA14AHUft0JkIT537DtilxK5+l\nspDkv4Yze/wsIMJTEISsIDJSGS/iYNxebFf7HRrFYGNtLR6nk+HxcQ53dOjnyrdaAbApCk0bNoSd\ns8jhiPqQabyO1+Xi9L33xhyP2+nkhooKAFaWl7OksJBypxNPSKiaNSfLd+9OKPpmUqROpzepJgCM\nUTqz02wz6c+YybFGwbX7jjupr6riyIZN1K2Ln9IZS6RFij3tscvl5ddrdnPY6Y5bNkQ735YtH/K1\nr704rb6Wxj6l8YSPcdw/aD0Ztsa08ba2PmJa9Cs/vwKnswKHw43DURI3apuMXCvCk8kXIdMj8hOq\nseBObIy/Mx7PCv1n875QioxqGrdF9uTU0HreFgAvhY5bCJQBawmPFEZ+5E2YETgHXIp4HDRsv4B6\n/xtQ5ydRUaFIrg/9vxLYnvkwhZkj+a/hjB1/8eJFDh48yNjYGOPj4zzxxBO0tLTw1a9+dZqDiY14\nPAVByAqm46mK17ok1rlu3buX4z09wJSPM1WS+UoHfD5WPf00iwoKODUwoHs+66uqcDsc+rEV+fm0\nDQ3FPU+8OdGIHHeYD9Xvj7q/ZONOlWz1u2XSn9Hs3o6ZFlOKPH5TSHTGcqxlSqx7T6U/ZCwvdKrH\npsr861OZGpm1SZmdwkDGdQOxi1VlRiyfZiLvJkAbcCuq6Pwp6qduY4RT80LagMnQv2zAguq97Elx\n/xLgI8K9uA7gBuJ7N7V1YUctRrQ9xj7CbJLN7VR6e3tZv349f/jDH7BarSxfvpwf//jH1NbWRu2b\nicdTqtoKgjCrxBNDjbW1afW/NJ4nkljnKnY4gOhU2VTGF6/CpXHfRQUFuvAzXqfu4EH9WKfFgm9S\n/eCTSgXcxtpalu/eTdfoaMxxG8dVmZcXdX9mVeZM97WZLTKp8JnKsemIyUyrqUYe/98cbm4bOMNy\nWwH5tY2Q5MN9OmONde+pRIDjpb9nEj2OxMxz5RLTraqsMjvVSyPXjXlfChgFUh3hAqmR+D05teM+\njyrMzhAuOhXgfOjncZPGahZfBl5JY/8bUe9fS5EuBa5B9W5C7NfduC48qCnMUlxIiI3H4+HVV1+d\n8etIqq2Qc4gvIbeJl7aZyFMVK4001nm0/bYeORIlkhIVCzGuqcjzNjQ3c7KvD4Ayh4NjnZ2U7djB\n2hde4FR/f1QBohVlZdR5vfp1jB/W8w0iOZXvPB9pbeXTxcVU5ufzVIwKuWE+1E2b4vpUPU4nncPD\n007DNb422eI7nQ1STX80tkEpL1/J5OQfJ9w3VlpqpOAaHThDadcxulJMvcw0VTOV1NR4v0NmprXm\nWoqsWSQqBgXJ/u5lWpFkrtEE0mFU8Wmcg8jcQWNvylPELpCkESQ7vJyxOA7Ee/+0od5fsWGbdm9a\nivQHqOnEMPW6R/bt1I4pQk1VDk/Nlc9SwlwgEU9BEGaMWNHDWFGTRC1QItuZLN+1i9P33ZewEi3A\noscfZ1VFhd465ZHWVnpGRth65EjCtNMwsXbpEofb2/XUWUVR6BlTPUOHOzv1gkMepxOvywWKwt51\n69SfQxijhfWHDnG4s5MVZWXsqKlJOn/GHqE/fPnlqAhpZCQyMhJrt1rZ6PXSOzqqR2Mz7Uk4nSiq\nWSm/s02q0beBgTP4/WoD+qKiK3E4ihLuGysyGhnxSjfyl2mkMJUIsHGNpXusmeMQIkkUFcwFEgln\nYxpxBfAcU1HNyhjHLUctOFQNvBbjWjayI/oZWWW3HNXHaWyPshZVjK9AbQcDU0Icol/3yMi39nx/\n6Dy5+sWEMJ8Qj6cgCDNGLE9YLF9mrP2M2yrz83UBBqrQW+HxUGizsaOmRj9PpCdSo76qip6RkZj+\ntEi08XVeuqSLXVAFrtvh4HCn2kttRVkZe9et4/bnn+fC2BiD4+Nxz20UgpFjbmhu5vm2NrX3Z0UF\newxRX2OP0I1eL3sTpOYm8nsO+/2meTSn4/eM5w1Mh0w9lNMhVR9oOv68VPdN14Nqhmc1kzmei9fH\nbF/t5RRhnXuMgvLnwMPEFs41TImpCqZSZzWBZjzus8A51IJCdwH7yA6RCWqCoZUpwWlFFYKtwBdQ\nxxo5BwOk94WC5octQm0zsyd0XLrnEWaKbPZ4poP08RQEISuJFZWMlVIba7+odiYhD2ORzUavz8fh\njg4cVmtUOq22n1bxVotcvtPfrx+vtVmJhTY+7fhyp5NyhwO3w8Evb7uNOq+XjV4vRzds4KcnTtDn\n8ySAV+QAACAASURBVOmiM7JNi4YWJTSOWUtZfeqDD6Z6f3Z0cPXOnXoaq9YjNJUIaay+o9p8mtmT\ncDrniucNTIfZr/qZPP1RI5300FT3TfXaxv0dDjcHD9ZNu7rsXLWnmS7Ga+7ceU3a9z0XY85+ItM1\nZwpjBduHiV+K0xgN/WLoZ2NU0HhcFzCI2j7kWbJHdK5FjWYaP48XAi6migVF3gukX6K0EdXLOYwa\n4dTWdKalUgXBPCTVVsg5mpqaqEkhTVGYe2IVpdEic2eHhvAWFlLscFDicFDhdOIOFQAyHptvtfLg\n0aN8rqyMm+12vU1KpIjRzvu58nJustn4n7fcwsOtrWGRSwvox3/xqadY6nJRYLPxPZuNu+64I+bY\nO4eHOd7Tw+HOTh5ubQ1Ldz0zMMDFwFTK1BNf+UpMMZYsLVijMCSqNX/pnrVrUy7qY7zGU2vX8nBr\nK9tWr+aR1ta4RZimQ7x0y0SYUZwom4vORKaHJnqPmslU0kwLHM1Vexoj6UQhtWvabEX4fOd1AZnq\nfWfzmorErL97yed3dgoVpe5LNaaTamPKR43o+VBbhWiRPWNrlQDR6azTwYIqwl+I87xWNTeSItQ2\nKM2oVXcJjVvrqTmIKg4row+dNm7UKrdaFeDEa1o+SwlzgaTaCjmHvFnmNsa0Sw2P00lvKAIZmYoZ\nmaa5bfXqmCImXjqnMTX0/YsXGQgJxXKnkwuha942NETTX/wFn921i66REcYmJii22xkPBrEoChd8\nPpwWC5PBIEHgS5WV7L3jDrYeORKW2msFbq2sjPIyxkov1sa1oqyMRYWFOCyWMFGdbnQyXmsZM9Jc\ns4FEqaTZljI5V+9RqabxxpuveHOcyvymkuqbynnSaaeiXXNsrJ/OzsNptyIxu6XOTGLWmko+v8na\nl5hFvPTPVFrD1DAljkEttrObqdYqt4Yem9U6xYNanCcSK3AWeDBiPEa0scGUZ9MFDMXZJ5J0W+Wk\nnlYrn6VmH0m1FeEpCMIsowmuErudi4GA6p10OuMKLm3/IpuNm6+4gkUFBTF7YUbut2fNGh5pbeVU\nXx9nBwd5ZdMmvtvczOGODlaWl9M9MkLn6CjFdjsnN2/G63Lh3r49LIKpoQBWRWHc8F5W5/VS7HDw\nn++9p2+zMPVRR+vhqfs3PR72rF3LzXv30jUygs1i4aYFC8KipPHEoxnznaqYzcVCQJdr78dIjELq\nB0533I+r6c5XuvvHE5jJztPc3MAHHzyF399PeflK7rrrN7Pmb71cSP7lhFl+wOn2Fq1hSsTFE2Sa\nOAa1sutypnpZPkJ0L89MsKPeQ+T5lNC1J1E9mu8DHRH7lKJWn53ybDawijMsoIATNOLHnVTg15B8\nPoRcQYSneDwFQZhlNI/gW5s3617BPWvWxPUNNtbW4nE69Wjgk++/H7MdS0V+PjZF0fdraGnhzMAA\nx3t66Bob4+HWVv06v7nrLpYWq6XqBwMBlu3aRdmOHYyMx/YEBSFMdAI0nzvHzrNnw7ZpolNLqT0z\nMDDl3+zspKGlha6RES4GAlwI+VS3Hjmi+01j+V8zbV+SriczXrubbCaXUiZnEqMv1Oigi3Qvpjtf\nQ0PqOrfbS7jppseS7h/PO5nsupHVgdPxt6bjh72cSe4xTtS+JB3PZ6IVqBHr3Mkq3NagptCuBzai\nOsaOh67zbcJ7edpRo4vTQUvbDRAtOhcBt6D6NvtR7zNWRHQQ+AxqJBbAzRmu5BitHMBPA4tJHlU2\no1XObPl2BSE5IjyFnEN6T+U2mrjyuly6yErUw9PtdHJDRYX+OBASgJq404TZ821tujjUivyE9dC0\nWqk7eJDhUJXYtiE11ckK+E6fpt/vJxAM4rRYuL68POl99Pn9+CfDU7lcdjvrlyzh2tJS6g4e1Asa\ngVogaNvq1dgt6tuuBfBPTnKgvZ2rd+5kyX/9F7c+91yUwMxUCCaa21iYUQhotsm23o+pvEfN9EfB\nRB9X052vwkIvAIHARVpbH066fzyBmey6xuNqanYkvc7lhFl/96JFerKVmIqAjEUqginWubU+lbEE\nmbHf5+9Q/ZLGL+N+DbwV+tkOrCLc95kMLVCzErgt9LM1Yp+VwDuE99gsI7yQkXbNCVRxugxtbgtC\n46jGwza8wFYSvwMkmg8jiV7H2K+hfJYS5gIRnoIgZDUNzc0MBgI4QoJtZXk5VxYW4rRY2HrkCPva\n2jjW1aW3HSl1ODhxzz24nU4q8vPxhITt2YsXdQG36umnGQztPxFxva8tWcKCUH/OVLEp6geWodA4\n24aGONbVRa/Px6KCAr0Krtvp5LW772ZxYSGrFy7Uj+31+WgfGeF4d/eUEH3iCW7du1cXr5p4ziT6\nmQpmVsCdLcyKeM1mXGC6H+dTJdHH1XTny+FQP2SnGiGNJzCTXTfbvkC4PEi2EqcbcYu3Ao2/ZZpA\nM547UQVWbSxFqG1VDgDG96gxpn5zi1FblfQBi1EjlLEwCkstq+VK1KimhymB+TnUCrS/CY2tMfR4\nI2qqr/aXpIQp0arhQ5vbRhqpp55DLMOtR2qvJv67TqoVaRO9jmZETYXLhZ07d7J8+XKKior4zGc+\nw0svvWTq+cXjKQhCVhHpMVy+e7few3NRQQHv1NdTd/CgXjDHrih6FNSCWjRoPBjkeo+H0YkJvaJt\nZV4eXWNjVHs8OC0Wvc+l2+HQxd/K8nJ+c9ddACzftYuusTF9XEU2G8MxUnEVpj6uACxwOvmCx8Ph\njg48TifLSkoodjioyM8P86YCNLS0cKi9nQG/Xz++0Grl0kS4HF5cWMjbmzeH3XesQkHZVmQn16hh\n9txUs1XCxQw0D6XVms/QUFvY+pI1l+skW4lm94CsYeq3rA5VfCY7dwPwPDCKKjSXh85RDTyFWuG2\nK3SuAKqYLEZNg60GrkUtAvQuagQyiCrGalArzx4L7T/I1DxobU5AFa5vJxijNodam5cS4AHgCKro\njDW3xp6bw6FtmbzrJHodpY9ntpDtHs9Dhw7xp3/6p+zevZsbb7yRc+fOEQwGWbRoUdh+4vEUBGHe\nEJla6jOIsPFQaqvWp7LYbufGBQv05yeB8z6f7qk09rOsWbRI7ek5MsKr59Um5FbgKwsXsqykhDyr\nFYfFwuefeoq7Dhzgc+XlrF+yhCWh6Gq8N8vIt94en49jnZ2sX7IEi6JwvKeHA+3tvBCKzGr3paXA\nVhvSiAHyQqmuJaE+otUeD29v3ozb6UyaBit9CTNjNuMCqSbQZQNapHJoqC1qfcmay3WSrcRMekAm\n83BuT/HcZ1CF5UXU1iWnUYXhIdT+l6dD97AqtP8EqujUPJRtqD7QXuBroe2ngBeBvaFjTxI+D8Zx\nJhKdMDWHH4TG4w6du4v4c6sdc7PhOpm86yR6HaWPp5Aajz76KI8++ig33ngjAAsXLowSnZkiwlPI\nOcSXML+JFFfXG4RZz9gYDS0teF1qwYjBQID3Ll5kQV4eoApRjRVlZbxSV6enjZ4bGaHX56NzZESP\nkE4A+z7+mOMtLYxNTNB6/jztly6pfTs7Oii026lyufBNTjIYp/BQLALBIEc6OsI8oFq01ON00jk8\nrKfL7omIWg4FAtR5vWHFl7SUV2Ma7COtrVFpt5pHbo/zh/zDpftZv38/Dx49OuPpufMFs8RgKu9R\nufhRMJZ383Iq7NTc3MC+fTXs378eny+zZOx0zzVzf/fMWomxRKYx/XMVU4WBNpLeb1mkP7MHNbr5\nCLAQqEIVlu8a9lnJlGDUisAVAz8DPkEViDB1/17C5yGdd4N4c5hobrXn9qRxnemMIT7yWSr7aG5o\nYF9NDfvXr8c3kP57TCbHT0xM8Prrr9PT08PVV1/NkiVL+MEPfsCYIfPLDER4CoKQVUR6DPesWUNl\nyHOpidG24WF9/56xMW654grqq6o4uXkzdV6v7qnUChjdvHcvL4VSVItCwhbUiGdktVoNh8XCsc5O\nPVU35j5K/IwS3+QkYwax6p+cxGGxMD45qUdBtchnucFL6Z+cxG618tMTJ+gZGYlb9dYYGb5m507W\n79/P9at3UFVVj8+9mpbuXg60t3Pg449zrkrtXHGyuYFv7KvhZROERTLMFDGxMd+xGsuDORu+zJmf\nq9QwRnd37rw6o/HMv0hxLI+hMWqopbQeRjUopLNWGlHFqpbdokUH/ws1qtgPdDK1ziuZ8mLClMgc\nBD6L2ucz2e/FbH01lItfQQkzxcCZM3QdO0b7gQO0NKT/vpDJ8d3d3QQCAZ5++mleeukl3nzzTU6c\nOMFPfvKTtMeRCBGeQs4hDY/nB/HahERWYXU7nZy+994wMeotLNT3L3U42F5Tg9vh4Oa9ezl27hyX\nDL04G5qbef/iRb30Q5HdrotTlxYhXbYsanzjk5P0jI3pwjRSYlqBP6qsxB5HfHqcTr2Crba/f3KS\ngdDYtCq3AK/ffTfO0L42ReHQJ5+w++xZXTBeHRKWxnnSIsNFNhvnfT4OtLfzg9aTrFmzmyK7GgGu\n9nj4YqhC70xUqc201Uu2YZYYSOU9amDgDI91LeGH7ctZvftnMzB/5pcvilUcaDZamWSLSNOiuzZb\nET5fb0bjSTdSnP1/92IlqmtRw2tRi/xopOtxc6OmxL5LeHQwuueyyirChZyxAu0YpFTUZ/6T/Wvq\n8sNWEHpfqK5m9bb0M0gyOT4/9AX/D37wA6644grKy8v58z//c/bv35/2OBIhwlMQhDkhnTYhRjHa\n0NzMqVAKiRX4QkhYRfbM/OLTT1Ozbx9PffCBLjqtwCt1dTy7bh17162jwJCaG8lkxOMg6OLQqihM\nAMfOnWPt4sUsLiykZcMGFhcWcrfXi8fp5ILPx1Ao4mlTFL0qr8ZVLpcurr0uF13f/CaeUGGkgUCA\niwbx3BsSlvcePqxvq8jPp8Lp1M9rFJbGqPGetWtnrEptLvb8TMRspo3abAV0s4D3WMbvRj0zMH/Z\nVckyk6il8XWxWvPnLPqpRXevuOJmfTzTXSfzr4JvrNRULZrXxlS7kRJgxzSvERkd1ASlhfB+nY4Y\nY6s0XB/Uoj69qAJ0OdHiM52MAemTKZhDbWMjVfX13HnoEE53+u8LmRxfWlrK4sWL075mukhVWyHn\naGpqkm/q5gHr9+/nQHs71R5PWqKoZt8+vbKrxtL/n723D2/ivNNGb1lf/rZsy8QhBgU3hKYfCU7c\n0ha81tZOKSbUboKSJu1F0rO1djdtt/tuN+w53bNnu233fa/T9Lq63Z7Tbjh9NyRN/YKTNIEU3BQT\n/FGSOk1DIF+NuyTQGjDGIGHjD9mY3/lj5hk9Gs1IM9JIlsxzc+nCmo9nnueZkTT33L/f/SstRXhu\nTimpAsS7zTptNmxZuRI9IyOYW1iAx+3G9SUl+N2FC8A772iqnjxYmZaHhoYQmp1F7+nTcX0PDgyg\n+/jxGNIIALVFRTg/O6vkljptNtx7ww0xLrcetxsrfvYzjExNAZBupdTk111QgNkvfSluHpjrrVli\nuXv3+zE9PYqCAifuvPMVlJX5lHVqd+FjQ1+Ncy9N9RzmKph7a1PTjrTIgJHvqEgkjKbuH+G3M15l\n/rTmOHXklpPl3r1+jI5KLqH19QG0thp37+TPy/PPd6TcjlWw6joxg/z+3VO7vvoSb24YJyGFzf4a\nwJcghfE2IDbMFpA+B29ByvV8HsB3IIXn9nLbqB1l/TDucW1m29xBfl9T+Ylcd7X953/+Z/T09GDf\nvn1wOBz4zGc+g09+8pP4l3/5l5jt0nG1dSRaKSAgIGA1GKFx2u1o9/mw0+83RViKOdfXi/Pzkro4\nOxtn/sN/JfJlWGZkl9zzkQguGAxvdNhs6ONyRsORCIKDg9jR1KSosMPhMI5duBBHOovtdvymowM3\n7NqlLFtWVIQ9J04o2960ezfevuce+EpKFOKpJp0A8PJnPxs3D+mQvunpUczPXwQA7N27AZ///J+U\ndUzNBKSyL1+YHlZu+AcHg2ht7UZXS0vMPKihJq+5TkxZ2Gi2jvWru78eM38spBSIznHqkNShXDkH\n6ajJ/HnJBTMjq66TIIIYxjCKUYwudMGTAw8IMoMuZOYhiA+SURAgmfSojxGEFHJ+DBLRBCTS2Q3p\nwcxNkHJEtaICzEQM5FZ0gYBAqvinf/onjI+P48Ybb0RhYSHuuece/OM//qOlxxCKp4CAQFbBK3Va\ntSjVUN84f+3FF9Hzxz/iA5WVqHS7cW52VjEAsgMokOt68ornNYWFIADnZmeVZWpFlEeVy4Wpy5cR\n4Vxp230+PLtxY1x/tg8NYec77yhqJoPDZsNlIqnkS02NUlP05qoqjE5PY0zlFBeor8dLZ88qxPOD\nHg9WV1Tg6zffjNv378dQR4cSVgwgjvxqzZWaZKjX79lVh0hkHHZ7Me6++60YxVOtZr548LMYGemB\n19toODzQ7LnOZaRD4NjtbzGkW3C9mdu/v830HCdDrpwDq1TCxVAbMwU//OiXlbIAAujOE6Usf+BH\nVIkE4mtcakUFsE+rE0AJpLDgZNdZutEFRr8hjG4nkKvIdcXTKITiKSAgkJPQullPVotSDbXyNjY9\njXORCPpHRxGor1dKqNhtNiwQYYEIdSUluMjlWJ7VsAP3ut04F4koyimP8NxcnOI4cOYM2vbvx8Tc\nHA6PjSn9+cXJk3GkE4i65U7Mz6P39GksKyyEr7QUZQ4H3lIprWwu7ujpUYjnDRUVeGbjRgDAzF/8\nRcz2/Lzy749duKCEGwcHB+NIhnou/+edr2Dv3g04cM0j+Omhoyh2vKmcJ7Wa2dLSZfqG3+y5zmWo\n584MgWM2P4B066hHL1KZ42TIlXNglUqYTVU60yiWlbJGNGLHoiplmSY0VrQf28bAwHYDYelMiVwL\n4HpIdUP57VjOKA/+0xow2FetdszA6DeE0e0EBHIXwlxIIO8gak/lD7TMZ9TlUpJBfeN8fGICgFSz\n8+F165T2/vzaa5XtXt+6NaaGpvqLrqG6Gr+9804E6utxdOtWOP7wB2WdQ8elNjQ3h56REYV0Mlfa\nuYUFze3VGJudxclLl3B4bEwhpaUOB9pWrFDmotzlUsZQ4XLpOsby83pTdzfeunAB/aOjCumskOdG\nDfXclZX58PnP/wknpi/HnSe1u3Aq7qVmz3UuwwyBU39HGQ3Ey4RD7FI6B0sNXehCAAEcwIGkYbaZ\n/d1L7ICcfjkbKxyWY9sw5nTMDI8OAXgGxkjkYoTNGj2mke2MGx2JeymBxYAgngICAhmD1s26mtAk\nA3Nv9cikzFcmuRdOzM/joaEhpb0nb78dq0pL4S4owH0HD+JDVVVKG7x6aYNEwD6xZw/6T5/Gjbt3\n4zJHUi8TaeZXqnHswgXUPP44ktHOKpfaYVEKCQaAS5cvo/fUKVyUCWNXSwtWlZXBbbfjuZMnFTJ4\n689/HkNCi7lapKMzMwqhLLFLLV+U50YN9dwx6JEqM+VStLY1e65zGekQODOl6K1GqueAkY3v7m/D\n7ZFw3vh15krNTyPwwINudOdAbmdiQpN+ORsryFxsG8ZyfVOpkbkYn1ajxzSynfVllAQErITI8RQQ\nEMgYtPIQjUIrfDRQX49LsvKoZarD57PVFhVhdGYG5U4nJubnUepw4JLKgCgTYLmdDO0+H4bOnsWo\nHO5rB1DucsU48BYVFOCjy5bh+MQEJubnMcGF/jZUV6PY4VDyWGvcbtzi9eLY+fMYm51Fo9eLp26/\nXXLbjUTQe+oUGr1eXJybw9jMDJwFBXjlzjvhKyvTdaHVO09m8gNzJZdQwBrwLrSv1Afw/7V254Vf\nZzruuVcvEucopp97bIXDcmwbkcj9GBzsQVPTLXC7n0yj3aUG5iCszmcVyAWIHE+heAoICGQQ6She\nLJyUEbRGrxdFdjsm5uZQW1iIp26/Pa5dXrn7TUcHAvX1OLZ1KwL19fjYsmUx25Y6MpPi/ufLl6NI\nVh6dNhtGp6bwoepqtK1YAXdBARaAGNIJQKoJOjqKkenpGNIJSPMwJIf3ljgcOCeTy49fc42i8G7Y\nswcDZ87gt2NjWFZYiBvKy/HuxAQuzs9jPBLBhr17AeirdnrnyUx4KdvW63bj9NSUIZVUIHfBFKWQ\ntxFPNO3IG7/OXHC9XSykrvYmVgbN1xxVh3smat9oaGhsG273SbS2noPb3Quh7PFYzPgKAYHkEIqn\nQN5B1J66OsDUuYbqalyIRFBXUoK3QiGFtBXZ7bjV60W506kY4oQjEdz6859jeXExyp1O1BQVKbUy\nf9zUhBt37cKc/H10+3XXoffUKcnZ1kAdTyP4QEUFGpctw7PvvRdX3qXa7cbFubkYNRSQyKmWOREg\nhelqGR1Vu914v8cTMx88vG43xmXSZwdw/N57lTBbI1BK3hQUoNTpxKMGSt4w1fT01JSizl6tyudS\n+I5i7rE3N+3AV9weU1rVwEDQwnqk5pCK620+lDUxck3ljtrrR+ZrYAplL10she+pfINQPIWrrYCA\nQI6Cd1XteP55JYyTgZUnAYA1u3fjnXvuwfahIVyYncV7k5MApLDUczIB+8jPfx5D4EocDlQ4nQir\nFMZUUVtUhMMdHVj+xBNKrVAe5zn1rwBS3qmroADuggLM64QAT12+rJtvysYOQAknBqTQ3OrCQvSe\nOgWnzaaE2ZoB7+AaqK/H9qGhpKVEmGratn8/gFiV1EgpksUkKwLx4N1jzdIXa+uRmkMqrrfDGFbK\nmgQRzNuyJrmj9maqBibvbPtjAA/B+tqgySBKmggIpAMRaiuQdxBP6K4O8OGfLIyz2u1WnpbZuW3H\nZmcRHBzEcydPKqVRKpxO3CLXvSyVQ1SZ2thQXY1ylytKOtNUO20A3r77bmwfGtIknWowMrlw5YpS\n8oX1k2FtVRWucE8UPXLZGK/bjQV5+YcrK9Hu8+HY1q1o9/nQ4fPhhTvuwJOtrQjU12Ns27aY2p9G\noQ6x1XIn1oNWOK+R/dM3MEkPVprSsO+ofDK6sRKJCJBxz03rEEQQfvjRhjaENY6aO2VN9GHkd898\nSGymYCbc08y2vHHOhwD8CsBqACfT6axJLB3zHnEvJbAYEMRTQEAg58HIjMNmA6NpFTIRA4AyhwMP\nr1uHCEf67HJZlA6fT8nvLLHbsaywEM986lM4KauiPHgyW66TA1pss6FtxYqYZZUuF+7o6cFT775r\neEwFQIwj7u3XXYc3AgF0+Hxo9/lwaMsWlHB9KHO54HW7cZkIYTm8tnj6GIILP8C3XvkNwpGIMv7t\nQ0MYm57GfQcPKnmWZhxq1eTRTK6nVr6okf0XW62xjPgGg4DfD7S1ITz+VrTNbwWzy7YWEYkI0GLc\ntjNFswc9CGoc1UxZk1xGJsrxpAYzbrJmtuXV0QIAFwGMA9iA7D3SyGa5lcV4TCMgkFmIHE+BvIPI\nS8gv8GGWfM6lXshlonb+8/e/V8ia02ZDicOhqJZetxsEKaSVd7AN1NdjR1MTan/6U0TksikdPh9e\nOXcOI9PTUmOqHM+bystxXWkpek+fjutHAYCm2lq8eu4cJg3W8DSClSUlmLtyBZGFBdxWU4PlxcXY\ne/IkwnNzuLmqCmUOh1JDFAAKMYfr8Qd4cBG/wcch6a5A24oVmJqfj3OY5V1na9xuNNbUxJyDROGw\n6bgTG90/ldw8K5G+c6cMvx99/f3wA9j/r7UY8Y7Ce74Rm799AO4Zj7k0tiWIxcjMa0MbetCDRjTm\nLbm8un739MJZeWfb1QDGMTBgRzjcCIdjGC0tIUhfL5n8kFnh0GsUfqSW/2oMV9c1lRvI5RzP0tJS\n2Lg65jMzM3jwwQfx7//+73HbihxPAQGBnMVzJ09idGYGAFDtcuG8rNYFBwcV4xkjOYDD4XCMQjhP\nhGmZXJY6HIqZTl1JCd5fUYHe06cVh9X7Dh7EHFers+/MmYR9/v3EBMpdLk3jnytAXL6pHZJ6yXI3\nU8HU5ctKHmjvqVOocbsVZfOtCxdQLtcDXVtVhT9NTeF8BPg9PogyzICRTgB47fx53CLXMOUVRqY6\nsrBjFvbKzgGf18kvB6IqZqowsn8quXlWoqWlyxriWywrIo2NaPnSUxg89hCa9u2QSGe+WMNaAD3q\n0IXs3bazPjixB+3oxE78W16SzqsPTBcHpLPIvhc83N+vANiAcPg6jI5KNYkHB4HW1kx/yPg+ZBrZ\nVFcFrnZcunRJ+Xtqagq1tbW4++67LT+OUDwFBAQsg5pAbh8ailEplxUWKrUn+RxAIzUgmcstj2q3\nGxtqa/Hbc+dwenoa5U4njm3digqXC7c+/TTOz87GucuqcXNVFd4OhXSdZY2gyuXCBQ13WQZXQUEM\n8dXcxmZTHHdtALR6U1dSgte3bsV9Bw+iZ2QEN7ou4rrq1Th0RlJCCwsK8Pt77kGFy6UojMwYyGm3\no8ThwNT8PHpPn445B+/fvRv/dfEiFgB8yOPBYHt7QmXTyIOCqxbhsBRuu2MH4JFJTjZFEg0shnGT\nH/FaTbZtWbT6IKCP3DH4Mq6LRyMVGrB580q43TsTbp8vGAgGER5+C47i42jp+g3cHt9id0nAAuSy\n4snjsccew7e//W3813/9l+Z6oXgKCAjkBNSq2dj0tEI6PS4XXv7sZ/HQ0FBcyKVWDqCa3DCXW75c\nx/lIBIdHRxXToIn5edy4ezeG77kHK0tL8Z78BM9hs8WVMWGoKynBssJCzbBaI3DYbLipshKHz55F\nid2OKY3wW5fNBp6Waimpc9x7rZ6udk3hmyVP4sWDP0OV629Q43ZjZfVN+ElzMx789a/x2vnzeLG9\nXXGw1VIyA/X12On3x4W9jnLn6cLcXFIiqT7PHpdLEFEGjwfoVlGcbIokGlgMl1ktrUZPx7IKauJU\nLBMnoRcZw2/Dz6FM/lwfHPwi2lqfWaSeGNfFLYtUyDGEh4cx2n8YADAYfAit6u8UgSWJdB/+WPXw\n6LHHHsO2bdtS2jcZhLmQQN6hr69vsbsgoAM1gWTvK10uvHbXXfCVlcUZzwCxZjbbh4bg37sXMEr3\nVgAAIABJREFUT737ruKEeuOuXbjv4EHsaGrCLz79aRTZJRsgO4DxSEQJSQWAuStXsGHv3phjr7/m\nGmV9md0ec2xXQQEK/vCHpGMrQNRZlsdlIrw6Pg4boEk6AeCSarndFv9AkPWKN00qtNlwXXExql0u\nFNFFjI0dxv8YqcYz7x3HuUgEvadP46GhIezbtAmnvvCFmLIpzEzozVAIQPScaJn/zMr9KwDQs2lT\n0rlIx/X2akCufUcthnGTlldppgMH1QZR6j4EB4Lw7/WjbX8bwnnmMpyNa2rCIYX6v+cFnmhaTFXG\nuOHQ4hoqZc78xyGH7HsbG9G0IzOf2Vz7nhJI3+TOCpO8kydPYmBgAPfff39K+yeDIJ4CAgKWQe2G\nyt6/e++9CWtJ8mSIkZiQTCbVOYketxu3yiVCGJ1rqK6GUyZzxXY7fv2ZzyjHri4sxJHxcYk4ulyw\nq4hnKBLBb8+di+vT8uJiLCssVN5fAXRrfi5cuaKpUlbJeZnqZc6C+K/eBUjq69GtW9G2YgWK7Hbc\n4vVi+vJlnJ+bw7H55XgCX8AFx/swTRI5rXS5dF1i2TyORyKoKymJIfVqZ9u1ck7oFQDfOXJEsz0g\nSmbnr1xBh8+XkuutQPZRVFQDt9ub1ZtzLepgpnBGKlATbHUfhsPD6B/tR89ID4KLULJnMZCslAyP\n11puwyv1wMDmtfiRe2fGerR0nFqfQ9Sj+QFLW27p6kJ9IIDNBw7A7Vk6Sq5AYqT7kNCKh4w//elP\n0dTUBJ8vM+HdIsdTQEBgUcBCaY9PTsJXUoKTly7BV1aGd8JhjEciWFtVhevLynBJzklkxPHVu+7C\nXw8OomdkBCV2O0qcTrz82c8CADbs3YsN11yDM9PTStjn9V1dSm1PPahDcddWVeHQli0AgJt278bo\n7KyyzobYUigFAApU+zttNthtNly+cgV8hul1xcWYnJuLyTtlDrylDgc+ds01eFIm4HzeKwDcVl2J\n/7P0GXzl3Cacmp6Bw2bD7+68U7dOJ8uJ5XM59XJptbbVgt7+6breZgq5k7O2uNi716+E2tbXBxbV\nxCmTSOaM3La/DT0jPWj0NuLA5gPwXAXXgx9+9MsBzgEE0J0gwDmMMIIIYgd2ZNCEyY/sZ95mKru4\nCkBI/rsDwGKFJgvkC5LleKbr7m6FO/yNN96Ib3zjG3jggQd0t0knx1MQTwEBgZQQDALDw5KJZ1dX\n1Ecl6X4y4Tx24YKiaqrBTHQ8bjfCkQhqHn9cIXZs3epduxQnW54EqcnRk6ramg3V1Tg1NYUxjkzW\nFBbiHPfeV1qK60tLcXxyEtcVFWFofFxZV2a3JyyjoudsawdQ6nQqJNhhs+FTdXX40YYNuGX347h4\nRVIx7/TV4emNbQoZbKiuxsrSUuz0++Fxu7Fhzx4lx1XPiAnQJoN6BJPflpkRaeVrGiWouQKjhGup\nE1TLSsXkOcKRMIKDQexo2nFVkE4gF0vJpFtQJxUS6UdmyO7tAHoBNAB4wWBfBK5m5Lq50IsvvohP\nfepTOHv2LEpKSnS3E+ZCAlcVRO2p3MDwMNAv/5YHg/F+Knrgy6sAUk7jxfl5lDudmJifR6nDgfd7\nPPjaiy/iVyMjiCwsKMVCWBitx+3GR2pqFBLEh3eysE+v243TnD04Q3VhIZ751Kfw0WeewdjsLNZW\nVeHs0aPAihUAJHLI18EcmZqK2Z+RTlZChY2hwGZDaG5Ot5zKAqCQzgqnE0e3blXCj9/nGMOrc9fB\nh/cwef4U/Hsv4w8XL6La7Ua1262QTgAol3NA2bjVynG5y6UQRjUpZQZNjIxqudMmKqui3j/XYTTs\nyGrzHfYdlcj9NxnZtZIMLyUDlnS0K4/bg+48VXuN/e7Fz04XurKgYppBugV1UrGoylR28ZNYVLvq\nNCHupQTUePzxx3HXXXclJJ3pQuR4CggIpASuXCHMeB9EOLXQXVCAgc98BoH6ehzbuhVetxuXLl9G\n76lT6PnjHzE6M4PQ3BzmiVBot+Otu+/Gvx45ouQZ+kpL4S4owH0HDyo5i10tLVhVWoq5hQUcHhuL\nO37vqVN4aGgI79xzDwL19Ti0ZQvGOCJ8aX4eFxOURgGAQrsdf37ddQAkc6L3V1QoNUWdNptiFGTX\n2f/Ply+PyXn9B+9ruA2v4NtVAzi2UI/+0VGcnpnBedlAqObxx1H4k5/gY888g3kitHP5lYwojkxN\n4fDYWEKDH4/bDY/LhY7nn0fb/v1468KFOFOgRPmaamMilvOpzhnNFbS0dKG+PpBU5cuU+U4i06Vk\nJhBWmEQwLK4Bi7VgtKMH0i1/MqgzCpdShmE84mfHAw+60Z0C6czUTBk3DtJGKiQyU9nF6Y5FQCC3\n8B//8R947LHHMnoMoXgK5B3EE7rcQFdXfLlCPfDKz83V1eg/cwYAELlyBd85ckRR1XgV0+NyKSVO\nGqqr8cIdd8Qpcl63WyGXK372M9htNjgLCvC+8vJoKRVIamOFy4Xw3BwavV4U2e3oeP55hWTZ1qwB\n5LARp82GIocD8/PzUt1LjTqgQx0dWFlaiuDgIPpPn44JxeXLpGgF5DZ6vXhUdQ0/X/IVzLtfxRNF\ndyBy6ULcPpeJcJkIQ7IJUm1RkbKOjYEpx8kMfvj5q5XNk7xuNwZOn0bVzp24uboa7T5fjMpqpC2m\njuZSjU9GuJLBakWQfUclIvHJyO5iONFaiUxl1ZmlHWp9bAyZLemSKah/97QVcSuVvUwXv0n1CklF\nMV3kekY5CnEvJbAYEIqngMASwGIoT6xcoZHcTl758bhcCnFS35DzrrhP3n472n0+dPh8CukEYm/m\nXbJDrdNmw/Tly7g4P4/xSASvnT8PQFIjFyDVxQzPzaG2qAgHNm/GycnJGCXKKxMwO4Cbq6owkcSM\naMsvf4mburvROzKCC6r5ZuVQtNTO5cXFmrmRxydncCxSiV+dGoVLdry9uaoK7T4fHBqlV0ZnZhQF\njc3Z0a1bYxyF9a6J4xMTAKSQ3ec3b0agvh5rKipwdnYWobk59J85A5fdbogwahGrfCytkilFUO3y\nzCOZGmtUreWRS+VCzCqTRmFWu1JTsePy+woAD1vYr+xBUiLD4ac0FHErlb3MFb+RnHZ3og39CJt2\nhBUqo4BAPkMQT4G8g6g9FY9cv9nnCcpOvx9v33235g05H8rpcbvx7MaNeGbjxpht+Jv5VXK46jyR\nkltpB/DyZz+LQH09PlJTE1PmpMBmiyn/wfJAJ994A4CkUL4qk1YboKl2AsDpqSklDJiZHhXb7VhW\nWKiEDm+49lrpmNx+H6mp0SxpwvpT6nCg4Gwlqv/kw7KfbsHOdRtjapDyGJueRjgSwfahIYxNT+Ov\nVbmXz508qVwTD/T1KUT0kkyqJ+bn0fqLX+DS3ByKHNHgl4bqasMlUbSIlSitEv2O0qqZypCM7KZC\nhnOpXEimaItZ2qGmYj4AXxgI4i/2+vGz/W2I5Ek9z+jvnkTpHQ7JTTVWEbeSlGWu+M0whtGPee6h\nREIvEoEMQdxLCSwGBPEUEEiAYBDw+4G2NiCcw/cnuX6zryYoiW7Ik4HflxntMNgAvHrXXfh/3nwT\nY9PTeIc7aYUFBXixvT2mP2sqKnB4bCyGYJLqfy3wdJQplNMLCxibncV3jhzBsfPnldqh7Eu21OHA\nDz7xCc2HBF0tLUp+62jFGZw/a0fvXjeCQeDZjRtjQmsZ+kdHERwc1H3owCuxg2fO4Kl330X/6KhS\ni5Svj1rqdGqqy8mgdR4TqXwCEjIVoVAsh+c2ehuxY5HDczNds9Mo1FSsHMCy8DDWjPbDa0H+rFUY\nGAhi714/9uuQ4e/he3I9zjcRBtDSshb19R0ZdCnOnLJYLD+WkB5KfBjAo5YfwzyWdvavgECuQJRT\nERBIAL8/6twaCBh3bs02crWOYqYRjkRwU3c3RmdmUOly4chdd8FXVhZTUqW2qAgFNhtebG+PMfQB\nouVB1lZV4Y1QKKYWp91mw4LGdxdzs61wOrG+tha/PnNGqcvptNlw7w034Gd/+INmfqdDru8ZuXIF\ndgAbamvx7MaN2D40hKfefRehuTmUhasx+c074P3LIaxpCqO80IF3Ll7Eu5OTMW1VOJ04cd99uO/g\nQc0SJ5WPPqqQTB7q+qj5UhplKUGvHmq6SKVciFamXabyM5P3JYhhDKMYxehCV0ZcWMMAfrS/Dd4c\nKy+TrPRPbD3OOnTjdeRruKlUL/SL2AGCBzuRG+PwI/v1RQWuNuR6ORWjEHU8BQQyhLY2oKdHcm49\ncMB4rUqB5EhmQmPUpMZMvcpE+wZ6e9F76hQKIJHOPRs34rO/+hUiV2ILpNx+3XXwuN3K8Woeewzj\nkQhsAG71evHuxIRufVItOGSCy74l3QUFKL1Qg4WaEMLzc8o2PCkuANB07bV49lOfkuZK46HD7b/4\nBXpPn0a5w4GJy5fj6oHqPawIDgzg5ZO/hX1hAh77AubLb0Wps9BSo6BUa8DmMsyYKuVSPVQ/4m+3\ntZZlpy88uQqgO0NHNlNkPVskPFmt1dyrx7nUkG590Vhk4yGKQP5BEE9BPAXyENmsPRUOG3duFTCH\nZKrPtT/9qVLvs8PnwzMbNxpum5GqIrsdJycnY8gATxBqiopwcnISM2+9he4vfxk37NqlEDxXQQH+\n7Npr4SoowMFTpxC5cgVlTide5+pvAsDJyUls2LsX1xUXK66zDOu83hjHW0AijXq1PrXQ6PXivclJ\nnJdDMvn6oYnUMjYHD69bh4eGhgyr4fx5KcUELqE86bHMIt1IArWj51eHji26ky4/b82Tk+j7+td1\nt82lCAWt221rb8GNIYggnsJTCCGEBjTgBbyQlZv1ZMTSj+yQ8GRk+Bd9v8Dj/sdzqB7nUgMrtmNN\nTc5sPURJB6KOZ/YhiKfI8RQQSAgzzq0C2tDLZ0uWl8rX+zT7Nc1yD9XutUCsEVPPH/+I/tFRvHzu\nHB4aGoKdc5Cdu3IFvadOocTpRGNNDQBgcn4eDw0NxRxr4/79mJybw6sywWyorka1ywUAGBofh1Nu\n0wbA43Jpkk69L2Lmgvu7O+9Esd2OMrtdIZ0Omw0Pr1uXdA58ZWWm8mkVoyNM4gqkcVS5XDg9NZVW\nTiJ/HTgrpDbM1oBlUNe4zAVzLf56/vsPfzjhtunkOFsNrVxMftn2LDlmD2MYIUiGOSuxMmvkKpn7\nrtUmSXqZhMnMpEpRmmI9TgFjsDanNZrH2ogdFrsCCwjkM9Imnjab7T9tNttZm832uhUdEhBIBvGE\nLreQzChFjxQkM6G5zesFIOUkVrhcMccwas6iRW75ZbdUV0t/r1+PHU1NWCu/Z/C4XNjR1KSYGGmR\n5NHpaVycn8c8EWwAqt1uNMh997rduLm6Gu6CArx21134+LJlAKIlVz7k8WB5cTE6fL64osoN1dV4\nMxCAx+2Gr6wMH6mpwSRHxi8TxZFgK9DV0oI7fXXwuRYwDanMzNTlyzh89ix6RkbwRZNOiOxcMXOj\nnpERlP7lIAKB1MPX1TUu2Tm9wXEeW2d/uChOpfz1fIccAp0LSGaZonW7zS/LFqnnb9R3YmfGjhN/\nXMjHjRJLfs5+DGtNklItM5Pq755UusSPNrQhLExzsoYudCGAQE6HRYt7KYHFgBWK56MAPm1BOwIC\nAnmIZDemespmMtXnydtvR6C+Hoe2bIlRLmsefxyPvvOO8n71rl26BFSL3Kprha4qLYW7oAAffuop\n/F5lXfyJa66R8jiLilDjdsMjK5k8nLKrbQEkZbb39GmUOp0I1NejwGbD78bHEblyBf/8yitKO2u9\nXrT7fBhsb8epL3wB5yMRxSm3wunUdJdl88jqelrlYKwm8R63G09vbMPKZR9SjsOXW0mmPqvbY9cH\ny3ttqK7Goy1NSiRBKg6v6hqXXS0tWO8+iS9f/jbCp/cuilNpLqmYPNKtp5kJx2yteqOJbtQz6Teq\nVnyDkEg3m7NGAJcsOA4bw5vye+urY2pDKl3Sjx70IGhpRVUjiD1zi0WCkzkGZwIeeIRCLZB3GBkZ\nwZYtW1BdXY1rr70WX/3qV7GwoGWVmDrSJp5ENAjI8TECAlmAqD2VW0h2Y5pqeQ3+Rp4dowCS0sfy\nMO0AxuWSIFpKnBYZUNcKXVlaisODgxiZmsJFzgV2bVUVfvbJTwKQ8jjPRSLoPX06jly/cuedqCsp\nwTK55Em504lCux1j09MxJU2Ia6f/zBm47Pa42peVLhc2rViBUCSC+w4ejCFibB7/63Ofs7RcCV/v\nc83u3cox+fPWyKnPO5M8JecfRGzZ/d/htseuX1laGtNvvQcXiQipOizR43bjGzVHUIwZVV3DxUEu\nfUelGyqayuc3GVHUqjea6EY9XfKcCEzd3S73+SkAF+V1dgDjFh2XjWEcQB3iFdRkc5bqNbW4IZ+x\nZy4bJFiL3KpD8wUk5NL3lEBu4G/+5m/g9Xpx5swZvPbaa+jv78ePfvQjS48hcjwFBATSQrIbUyuU\nIHaMSlUbBVxOJq/E6ZEWreWM9LHw17VVVejw+XBoy5Y4YqhFrn1lZfjT5z+P95VLJjwT8/PoPXUK\n/aOjCkEusdsxdfmyoo6q22Hje/fee3FmelohYjdxRNBszqZ6rHpzwufSjs3OKuSPHW/70BBmLl9G\nbWEhnt24Melx2Vz58B7umn0Yf+3YhdrCQmXcjLiy/rwZCmnOidkQT7UKKiAh3XqaqXx+k+ZNquqN\napEutmwFgKPysrXInErI+syeolcCqJH/rgDwcJrt8w8AtAqhZIpcL27IZ+xjj2yQYC1yqw7NF+Ah\n6pcKRPHmm2/innvugcvlwjXXXINPf/rTePPNN5PvaALqtKKM4IEHHsD1118PAPB4PFi7dq0SW86e\nuIj34r2Z9wy50p9cfX/HD36AkUuXsLyhAV0tLXjtpZcycjzmdmpm/+DAAF4eHITbbsfz/+2/weN2\nJ9ze43Jh2cmTOH/xIrBmDRq9Xhz/7W+l2pcf/CB+8IlPKNsPT0xIDqPvvIOOt99WHEZfHhzE0QsX\ngDVrEBwcxIMOBxbeeQc1N98Me0EB6J13UHD+PB79u7+L6U9XSwuCg4O4+Prr8H/ve3Hz2VVQgLdC\nIeCdd/C+8nKsamxE76lTKHv3XUxdvoypG29E76lTWB8Oo9lux7P33x833u7WVvT19WHmrbeAqioA\nwOjRo+g4d07pv5n5HQ6H0S9bxwZdLoxNT8e8Z8dbdeYMQnJu6w1nzmCb/F3N2nv58GEclc2V7t+x\nA9+87TZ0FRRgOBzGzFtv4Z9uvVXJaezr68ODDgcmC0/hrtkfYGJ0BYqvvw9v33M7goOD2HblCl57\n6aW4/tXdeisObN4cc30WOxzAO+/gxooK7Lj//qTjdbs9cDgexEsvvZYznz+t998DcMnvRzGAB/v6\nUJqF43dnuP0uvx/DAGb6+vBPAIrl9Tf29WGbtEPs9i1dCA4Gse3KNrz20msY9vsl/8++PnQA6JPb\n62ff9/L+JX19+IKB+VP35w4D4ymWj38DgA/6/dgJoKmvD6MALvr9eEg+Xqrz1QWgo68Pfw/Ak+D4\nNwLYobHe7/encf67TffXmvcPApiG3/8sAA8e7HsQ05jGs/5n4YEnI8efwQzgl8jttr5t6EMfWlq6\nMDgYxJUr23L++yGb76VlL8PvPyr/3QHgmznTv6X6PhGCA0EMh4dR7ChGV0uX4XrMVu2/ceNGdHV1\nobm5GRcuXEBPTw++853vaG7b19eH1157DWE5RenEiROGjmFJORWbzXY9gOeIKM7KT5RTERBYPGSq\nUL0VMNs3fvu6khK8vnUr7vjlL3H47Nm4NvTqJGot59tl0OsPv+2q0lKsLC1FscOBifl5pR9Omw2f\nuOYaVLrdODczg8NjYwCkMNp37703qXIUjkRw0+7dGJ2djet/OrUi7zt4UHNOwpEIHujrgw3Ao35/\nXJusHa97Cmsq9qPc5cTE/Jdw+Oy47lwZqZOYrJalVsmR4MAAnjt5EpGFBdxWU4MnczCnMhn8yE55\njiCsqz/Jt1UD4KSqXT+iY6oF8BsADwEoUm27XadPiUq6MNgB/DmAGQCH5WXq+WP9PIaocml0jrWK\naWSzrIy1xTz0YOVVkZsIy7mkouyMUSxG8aSrF8nKqfj3+tE/KpfhqQ+gu9XcL0S6+1+4cAGtra14\n/fXXsbCwgAceeAD/+Z//GbedKKcicFXByFMjAQks7NE75cXp/7sJbW1SbdJcgFnTEn7717duhcft\n1nWbVYf/srDO+StX0OHzxRAdpqwlcq5lOD45CUAKy11WVKSEgh6fmFC2mSdC/+gonHY7jly4oCzf\nu3Ejtg8NJTXS8bjdePueezTDl82En6rnQC8k2uN249mNG/GMThgt229NxX4cHutFz0gPjk/8Xneu\n3r97N67pegb3ntqM0Tl7XHt6/dOaB3WI53A4jNGZGYTm5tB76tSilU5JhkTfUVaX59CDlaGbfFv/\nS6PdYm7bUUiksxsS6eS3fQJh5f1qXFEC+7qgXdKllmt3AUAvgOPyey+A04gNEFSHy5bDeIislruv\nVr9SgRFTnWTFPKz53ctktmxuQJj6GId0TVl1lQtYAXUaQjb3JyJs3LgRgUAA09PTGB8fx4ULF/AP\n//APpvuRCGkTT5vN9r8AvAjgRpvN9iebzfbF9LslICBgBRTSsH8zDve60dMDBDN0vxEMShFxK74x\ngA0/T+5S2tXSojjKqo109LZP5FDLq2I3dXejd2QEgQMHEI5EFAOd3tOnQUAMmelqaUHz8uVoW7Ei\nzrlWnRfpKykBAFycn8fL584BkBTO64qL4SqIfp2W2O0IRSKwc08E733hhRgjnwcS3Ejq5dWZIese\neSw3dXejaudOBA4cUNRDM06yrC/lLkbMG/Gbji/pkkZWXmY8EsGGvXtNjzERijl33YbqastcVrOJ\nbN3mpUtwg5CUzDYATm45s98qhUTwwogliV5I1KYKUi4j34c57pZjHAW4Sd5fi3RtB/A+1bFdkAio\nTT72YWgTYHaUCfnYibLX+HGqt7GqsuPiOsvyyNxjD2scaxOdDYHMwNr6pQLpoaulC4H6AA5sPmA6\nTDbd/cfHx/G73/0OX/nKV+B0OlFVVYUHHngA+/fvN92PRLAk1DbhAUSorYDAoqOtDejpARobU6+d\nmAx+P9DfD+Dv9gJrjIXQZiIUWB06G6ivR+/IiFLOo8PnwzMbNxrqi3rZpbk5qQ6lw4FLly/HtbG8\nuBiRhQWcl8mcDZLpUbHdjrfuvhsNTz+dtB+AfkitVvip2bnwuFzoPn5ccfA1Ou/hSBjBwSB2NO1I\n+INW89hjGI9ElDH7ysqStm0UycKC9TAwEEQ4PAyHoxgtLV15bT5kNFgy1dBNrXDVlQDOQCKdHkjl\nRdjVz0JZ2fFOIxoKC0gOrsxMx4tXcR63xhyvBhINqgHwKwARALchNqQWkEinG8Ckqr+VAKoBnIMU\njgsALM7ADomo8v1Uw4/Mhz63oQ096EEjGhe5rmPmAnr98KNfnskAAuhOaSb9yE4guoDA4iBZqO1i\ngohQV1eHr33ta/j617+OyclJfPGLX0RJSQmeeOKJmG1FqK2AgEBCdHUBgUB6pDOZSnb8EwPA3+2F\nfaW2S6kWMlEjkFfF1lZVYUdTE26TzXEaqqvxKGeskKwv6mVMYf3YNdco+5VxIbpvBgL4aE2Nso4g\nfcm+1NEBX1mZoX4A+iG1TMXseP75mPOgd2605mI4HFZIZ6XLZXjePW4Pulu7kz5FZeVlrCadUh8S\nhwXrYSmVUzAaLJmqjqEOV22E5CzLlE47oqSzElHdjB3vJNfWhxDr4Po7vA/FGIND9qAuhUQYewDs\nhxSmG0JsSG0DgHa5DTXp9AA4IrdxERLhnOL6toEbA+snr6ndD4lgA8Ycc1PV46xwlrWmFmXm1C1r\nHGuzFYguICCghs1mw89//nM899xz8Hq9WL16NdxuN77//e9behxBPAXyDiLH0zw8HqC7Oz2lM1l+\noa8xDKwZxUJRBHUlJYbq/tUUFcWFt6aLrpYWdPh8aOdKojzZ2opAfT1euOMOzT7d8YMfYGJ+HrVF\nRXjq9tt1Q3m3f9WNse+0Ajta0bbchw6fD5tXrIBXrgnK9mHlQwDgCoDvHDkCAEn7wZCIkGudB71z\nozUXfM3QI3fdZbk5DysvYzXpTAfZLqeQye+oTN+aHx8YAPbuRdn+/VgRicAN4B3umA3y35WQSJ+6\nFuUE9/4GROtjtgGoQAXKsQyXIT0QvyRv9yFIxI+hDMAnIKmg1QB2Ikp8AWAZAB8kFbQBwLS8vBjA\ny5C0sncBPItoWDNfp5PPV2UE+3qNsfghke4Ncv/fQmoZklbkHQ4Ovqz78MSaMNf0YE3ZFpFvmE2I\neykBNdatW4fBwUGEQiGcO3cOu3btQg33MN0KZKWcioCAQP4jmTpZXhhdb7TY/MnJSZyLRNB7+jSC\ng4OWhNp63O64EFaWT8iDD2f90+Qk3pBdaR8aGlK2Ve83PCyHE8ONVbcUYWVjGMcuXIgxu+lubcXb\n99wT40zL5otXLFkY7fahIQyHwzg+MQFfWRnKnU78uKkJDw0NaYbUJlJmSx0OhCIRhCMR6Vgac8FK\nw+iF65pxzs0XsHIKiVx28wVdsDZYUh266wuHMTI6ikkAhYODOCxf/3UAPgBJiWTOtT5VO92IEs9K\nAI8C6EA0ePJWSOqkGhcADEIiquchKZuD8ra98n6MpJZCIpf3c+0yPA/gZkQDNIMAxgDcB+B38t88\nGJllobor5DGVy+Ngob4j8v8sjzUR6TcSCp2Kt6zdLn0OtR6esBxSqe1gimGuZnoZv46R6/TAFFkB\nAYGlCpHjKSAgYAjJ8gvN5h8CyUtqWA1Gqo5PTmIiEsGEnKdZW1SE0ZmZpP3gc2Xd/8deHB6P5k/y\n+wYHBvBWKITjExP4jRxmy6DOGx2bnjZczgXQnudwJILVu3ZhXA6zTSdfVi/vNhiUiHfnNT18AAAg\nAElEQVRxsRS6nYk8YYHsw4/YrLpL3GfSs3kzet1updDCTZDCYQGJUD4D7ZxQJ4A/QCJxrFhDKaLk\nkUcxJCXxXwE8BmBOXs7yM9cCKEFsvqcbUiQBr4Kytj4CiRz75HZZn1i+NSAppXcPBLEsPIw5RzEe\na+nCpOqBRK081gpIYbyNkNTSh5CY9PuRPEvRyDZqJCpRlJkc0kS9TLRucRBEEMMYRjGK0YUu4Wor\nkJPI5RxPMxA5ngICArow42Cqub/sVnvfZ93Y0ajvQpqKS6le2ZNU+5oMLCR1ZGpKIZ2VLhd+09GR\nsLSHUo7lr/aj/d4IDhyIKrxrq6riSrQMh8M4fPYsRmdm8NDQUExbasWSvTdSzgXQnmeP242PyOEw\n6ebL6inbTO3NpDPyUkC++XKqQ3f5z+STbjcCkJROnnQCURKnzgmtBHAXJEWyDcCPIRFFLdIJSGZC\nHwOwG1HSCURNgV5HfGhWBPGks0DuYz8khfIw16dSxN7stAKoCw9jzWg/PjzSg8/JoavMnKgBkqIb\nAHAU0eBPH5JnSBoJhU4lXNrt9qC1tRtDQ9vjcj2tCXM108vcy8XMHedgAQGBRBDEUyDvIPISzMFM\n7UfN/TNIONQkKt2+JgMjVRUyybMDcNvtuOMHP8CluTnd/Vi/ekdH4PrfBuHxRG/QD23ZgmdUNTr/\nINf1rHA68fC6dTFtsf0+UFmJjuefxzwRfKWluMnjicsxNQOj5WkSkfvgwIBmrisgKZ2ApPbuyI17\nzZzEMID+vr6crJSoJsUsJ7MWkprnAbDd7cZYayvuk889MwziSacTwDhiS600QHK//QCkkFeWC7kG\nElHUw4Lc9kSC9YMAEj5Ch6SAHtFYboNEehmRXQvgZwA+Luf9vudtxM+bdsDBbbMSUZJphGzyMJKl\nmEomI/vd0zLKykztykS9zJ1cTJbf+ibeBKBtbpQLObC5CHEvJbAYEMRTQGCJI13n2GwSjuOTkm+l\nFmEzAz1yxUjf0a1b4XW7pZvemRm8EQolJLtac5iINEcWpFvYi/PzcYon2+/k5KREZk+dwtT8PIbO\nndNUSOPGJivQbW1AmLuH8rjdWFlaisNjYwnHwvdz9a5dMXOkp9QGg8DEBFBbCzz1lAiz5aG+1qys\nn2n1LbLaEXcYkjI4CimEVGsbIKpvARLN8CBaQ9MFYBWAU4iWUuHDW62IW7iCqMKqhwpVP/n+AlF3\n3EOQjIZ+2NKFN+oD+H83H8B5t0dx6m2U2/IjtXNgxDc2HW/Z7BllJepl7tR+ZErnOMZRhzpN1Tdf\n1VBBmAWWIgTxFMg7+BOUoRCIhzqc1fT+cimWD/z3AXQMZC4MFgB8JSUAtAmbGTz32yi5+uLB2HIk\n3a2t8JWVKaGpFU4nsGZNQmJuZA55ctpQXa38rdcmv/1arzfp9gyJFGi+zaKnmzQJKm9ENB6JxJDU\nRGG2hw8Do6PAQw9BgINape8CEPD7U9aCjJZLSQVqUqx+H0S0vEgVgAH5/2lI5kLV8rbjsvMt9u/H\n85EILkAy7lFXtk01k6k0hX0uIlpKhUcIQCEkYjwA4IOQwnp73R78sLVbye30QCKmByApvJk6B6mC\n/e61tHShvj6AzZsP5L1RFo9USRZfxuV1vK6p+lpT6iX9vppFpgmzuJcSWAwIcyEBgTxHtkxf9Exn\nrIQRsyEjrqtV/7wfoetGgPe8aD+5Gc926ZshPbxuna6DrBnwpj8Akhotmd2egTc4Utdl5dvs2OiW\nHXilBwfd3bHbhCIR9J46FTPXauMiNtdvHnVg/HgRSq+fxMdudeDJjUvD7dYKWG2Qxcx4mKGPlR9n\nFl7LzHHY+yJIZIs3CHIglkjy5jzYu1d6CgEA9fWAxd8FyxDvQJsMMf0zCVYahrn0ZvIcLDYGBoII\nh4fhcBSjpaUrZ8irH37FmTeAgGGH3DDCCCKIHdihG2psZJts9NUsMmMaJbCYEOZCgngK5CH6+vrE\nkzoOfj80CYbVyIYDrRFnXCME+Pb2CHqvGcTaN5twaJ87KRnPp2sqHJYeNuzYkfghQyKCCsTPtRah\n5+caBCXRrsPniyvTcrVC65pN53pSk0OjYKGzxyGZ9MwDuA3AckikMlHpDj/iS5PwKIAU7qpg/35g\nZATweoHNmwELvwtskGp4HtZZH9eXNFEHycCInxc1ITdT9sQKaBUyseo7au9eP0ZHpbNdXx9Aa+vi\nO9IC2SFZVjnfZosQahFmK9178+l3b6lAEE8RaisgkPfIVg5muiG7RmDEGddIzuqTj7kRCLcaIp35\nBo9HeriQbFwsRFqLdALGjJ3YXAOIcXex+mcz027GmUQqbs4J20Nq2XMsRHcEkloYglQDczeiYaMP\naOzHh9d+GJIDrRpxRK+lRVI6LSadgHRt6ZFOzb4YAH+jUw5JUQWksU4AWA2JYDKwc7BYIbeZDLfO\nXo6oOWTGmTcWRkJXjYTRZqOvgLZplJnwW5EjKpCLEIqngECew6gClktIJzw4lXqhAsmhpWiHIxHc\n1N2N0ZkZlDmdmJyfx9qqKhzassXSuc9GGPdSBVPH3oTkNFuOqENsFaTcR+bWympv8vAjqnZ2ILY0\nihFUAbhgttMyqgGcT3HfVOCEVOrlT5CU4SkAk/K6Onk5j8UKuc3kcRPVAzUKI6pbLtbVNKJUZiuM\nNlWYUVtzfSxXI3Jd8Xz77bfx5S9/Ga+++ipqamrw8MMPo6OjI247EWorICCQV8hUeLCR/M+rCWbm\nQ4/QW50Lq4VMhnFrhS0m3SdPrqMgJGXuovy+DsCvAfwtJOVwHFH10APgPcSPX01yApCUUjuihFUP\ntZAUSLP5mJmGOj+VwQ7JuIjNlwtSWHIxgLcQzfFk14wTQAmAnchunmeq4dbZghFCs5ikR4/0Gsn1\nzPW8SjP5qrk+lqsRuUw8L1++jA984AN48MEH8bWvfQ19fX3YsmULjhw5gtWrV8dsK4inwFWFxcxL\nyJcb0lSQ7tjM7J8s/zBVpKKcBYPAyy/3YflyvyXmTGbmIdG26ZyP4MAAnjt5EudmZhTykMtKohbp\nteqz5kdU0QsAhm5/01Vgs/Ud5Ud0bJUA3kUsUWGkUm2ew4PPZ/wVogpkCRKXErHJLyvzLdOBHUAZ\nJDJ5AMBnEBs+q0YFgF8AuBcSWefnxg/9a4aRmuP4B/hwO8rhQBekEi1mH3CYQS7l4xkhNItJetIh\nvVYbES0G2DXqhBMlKMFO7NQcSy5dU1cLcpl4vvHGG/j4xz+OyclJZdnGjRuxbt06fOtb34rZVuR4\nCghkCVp5cEsF/Nhu/f6gZikOo/snm5tk+YepIpWapcPDwNGj2uVJUsFzJ08q83DL008nzF1MNGda\n64zmQg6HwxjlSGely5VSDddsQStP0qrPWip1NdOtfZstsLExYqn+KHVBIk7vAvhXaNem5PMZRyGZ\nEs0jef1KQmZIZ8I7FhXs3N8FkPo8BuD/AnB9kvYvAvh3AJsAfAxSyPB1ADYA+I28TTmAh1X7sxy7\nERThMBwxNVHVeZmsJusKud1M1GZdDBjJccxkHuTAQBB79/qxf38bIpH4GU2nfIpWXmW+gV2jveiF\nCy7dsXwP3xM5oDmGdP0OrPZLuHLlCt5444202+HhSL6JgEBuIZNP6JLlHubLDalR8ON1/lV0bO4n\nm5RQ2GDQWCismblhBjnpgil7kYUF3Ob14ifNzYbCQXk1zVnRAsBvmTlTZCEaoDg1P68oZ8HBwTjl\nLNGcaa1jZEyvPfW+AOBxuXDkrrvyTp3XmxuzSmgXzIctdrW0pJVHbPV3lDpcmKlrTki1J3dCe2yM\nVAJRYgQAtwJYKbdXA4l0vmlpj1PH5weCWBYexpyjGP+zpQszCfIQ+VBgngTbECXlgERK/wySey1T\ndB2QSOXHIBFuQMptPc3tNyGvfxvR+WWkphxOTCD6QOM+eT3/gIOf8xH5fxYebRaZ+N1LJQwdiJKz\ndLdJFeHwsOLMOzgYjHPm7UJXTqmWeqG/mcqDNUq8L/kvKcpwEEGRA5oDMPobn4n916xZg2XLluHh\nhx/G3/7t3+LQoUMYGBjAJz/5SVN9SAaheAoIcBgelnIP9dQvI86uwSBMqYVmt7cS/HhLdkXHVu6U\nxmaUjAWDwMT3W1B7qh5Pbcic660aTNkLzc2h9/RpPDQ0ZMhhlFfTSv9y0FL19baaGgBAQ3U1Gqqr\nAeiT8a6WFqwqK4Pbbsd9Bw/GPKFUX2vBIHDsdxIZa6hMTO67WlrQ7vOhw+fDe/feC19ZWfoDyzL0\nPmtmldBUXGKtdqpNB0FIxJJ3pmWEphdSaKne2Jji1gbgD/Iy5urK2ntc/nscUrhtJcypjkk7zzpg\n8LttWXgYa0b78eGRHnxhsNPwodgcVAM4B0m1vQ4SOW+CZKr0UW77ywAeAsBrAuxxTTm3bBSxzrJM\nyTuGDyGAqPkPU5d5MyBGfp1cu2oFdTFh1j03V1xSkznz5ppqqedEa8ah1gyMqs3pKMMCmUG64kY6\n+zudTjz77LPYt28frr32Wnz/+9/H3Xffjbq6OtP9SAgiyuhLOoSAgHU4dOhQxtretIkIIGpsJAqF\nUmujuVlqAyAKBKzf3gqsWUNUUUHkdMaOt7NT6k9rK1FHh/E5WIwxEBFt2reP8MgjhEceobVPPkmh\n2VlT+zU+/TSFZmctvaZCs7MUOHCAQrOzMX/roXnPHmUMgQMH9LdrJkLRLKHzALXfa2yci4nO/n5q\n3rOHNu3bZ/i8GIH63OUirLyemin2R7WDiDbJfzcSEfuIdsrbbuKW8fs6ub9dlPzHu8DANklfzdHv\nBQSM7fOVfZvokUdA//vTjVQ0GzJ8LJc8xnKd9W3yvNSq5q6Vm5/b5PVHNbZLBSEiChDROq4fqX49\nGr2mtK4DPWhdR4nQTM0E+V8g5ZGkhk7qpFqqpUqqpE2zzbTvQAfNzsb3upM6qZmaaRNtolDKZ858\n3xIdcxNtIhCokRpj1ustzxaeO/QcBSiwKMe+WpGMExm5Z8jk/mp8/OMfpx07dsQt1xuHvDwxL0y2\nQbovQTwFzIARn02b9ElPJolnKCQRp1RJJ5F58mp2eyNzlAwVFdEbwsLCaDupEkgrCHsqCM3OUscv\nf0ntv/xl0i9angidmJiI+XLe/G//lhGSpHX8uieeoPXPPKMcyyiRWqw5ThXJCHWqxNTqH9ZMwMrv\nKEYOQEQfJokgMELDXwbN3HYBjX3NEMsGIjpBREXcstVEVJVgH82XfM2ikQghY/sUzYao80DAFOks\n0VhWqnrvlOfjBDd3nUS0jOIJK1uvnmMz6O/vpD17mmnfvk30GXks6ZBYo9dUMxknuUbG2N/ZSXua\nm2nfpk30mVBr1oiSmszxpDcR8V0McpzsmCEKaRI8veXZQibvpQS0keuc6NixYzQzM0NTU1P08MMP\nU319Pc3NzcVtlw7xFK62AjmFTJXZyCbM1tU0u70Vc1RTA4yPS7mdb70F+GRLx1TdZtOtJZpOXU+j\nSORUmo06kvwxGAL19djR1GQonzDf6rUmK5EiancaQxjAFyGZ+eyEflitVu3HMICbIIWLNgA4Bcl8\npxGSMc+QTltVANZBqs/JtukA8AqiuYqGO5+huiCrAcwCmIY0N6yWaDGkkik3IZpfyWMVovmtE4iW\nm2EwUzszUY7k3r1+JQ+xrj6Ana3dWSmPYnUN0L1+P0blH5y6QAd2djtjciczlaeodqa9hEvoQQ8A\noAENeAEv5IybrihbImAUuexqCwDbt2/HT37yE8zPz+PP/uzP8MMf/hD19fVx26XjaivMhQRyCsVy\nUoxVRi9WwwhBSmaco9WGGfJoxRy98gqwYQPw619HSScg9UeL3AwEgwgPD8NRXIyWri645ZV682GW\nSLJcU7ZvJh44mDXyydTxK5xOXJyfV47F8gmTwSpDJjNIp6RJMoOepWbUlQqMmLt4IOUnJgMzUSqC\nRBKZcdD75PXPQCohwnggb4YzD+Ao19YFSOSlVn7vALBvIIgr4WHAUQy0dAEq058KROtjxnQ+Q9fs\nJIA1iCeX0/LrEwC8kHJXSwFcgjRWN7dPLbffhwHUQyL3Rkuj6Bk2dSE2D7GlaQfazA8xJfOfVMy0\nEsEh/+B4GxvRsuNRtKlaZXmKUn9TM6jRIq9a+YcP4AHYYMOjeDSG3PH7/xg/xkN4KGvGQgMDQXwp\nPIENjlp8qeUpeFSfi1SJeaL9MkX2BQS++93v4rvf/W5mD5JMEk33hRyXlQVyC0ZCXQ3nuqQZkqre\nv7MzNkQ11VxGo+Gsev1PNkdWhOKqsae5mR4B6BGADnCd5seyalX0uOvXm5unbISRbnvhBfLu3Emt\nv/hFXJjmc88/n/HwTRYiqg7zzWUYzT9NBfkQMpsqMhEWaRR8m16N9jtJCqG1ycvXUzTPkX9VkhSW\nWsOW7WkmPALpdSCQ2RsHAy87Nwb1y8uNq4Niw2v5vMYT8vp2Sh62rEanPEcgKTR5vWqf2dkQHTgQ\n0MxD5NtoJv18TL4fzYsUFjkbCtGBQIBmdb6YrchT1ApVNROGyu9fS7VZDV3ds6eZHnkE9MgjoP9x\nYFVcrmeyMFy9/NBE+1kVTixCbbOPpcKJ9MYBA6G2QvEUyClYqeqkq6Kp9x8bAy7Kj/QrK1NXG5Mp\nlkwtPHYMCIXi+59sjsyM26iixT/1buI6zY/F7Y4et7Y28RjV0FNarcTJyUmMRyLoPXUqzma81OVC\nd4YLafPKZr6ElWZSlTSq9OYjvnf0KL45MZH0c5WsxqhRxYvfjjmoNsrb96rafwLADLfvYQBc0IOC\nmyGpmI2Q1E/ICh68jYCGk2i2saCxzAWgFZLyykJonQBehhRiex+AH0Nys2WKoJaabKT26zCk8iuA\npHTOqfZxuz1xZT602mCKabTMSvRsFmMPACcaAfx9wpYyB7fHg9YEPyJWlC7RUjfNlGMpVs4YMIpR\n3IpbsRIrs6II8sr2k03uOPVXzzmWqZYv4SXMyVfPA3gAz+LZmDFpOc7mixutUGYFNJGMmab7whJh\n9wL5h3RVNPX+7H1lJdGJE6n3S0ux5FVKXi1Mpf9mxm1U0dJ76s2PhT/uiRPpmzRZjXxwQ801LGVV\nMpMw+rlKZu7STMYUUX67Dq5NdfudFP8jXUX6TrBs33YiqpwNSUqnCdMfvZfX5PZOInJTcqfdFfJc\nuIkI/Z2SSrtvU0yfjehDRkx31I6wqZgR8W1sI6Z+vkQhqiAiUIjuT8vgKF9gVN3UUgc7qZPW03py\nkUtRXtfTeksUQSPglW0t9VdvbGqzJBCogzqU9Wy/bbQtbsyLbUpkFEaU2cVwIV5MLBVOpDcOCFdb\ngaWIRKGk/Lp0yY+aIOqFuFoR2sqHrNbWSv83NBC1t5tv04wzr5VkzApH4EziaiNRmQi5FjAGs58r\nPYdfo+Uu1NutIaIKkgge/4ysmWJ/oGsonnR6KEoO11M0DJQd40OUfqmVExRbYiTRS8uxFiSF2fLr\nKik23JUPDXbIocGJ5tFMGRKi9F1v1W00c30P0C6d3prtZXaRaRKhRWT4ZXVURyHSJoCZ6iPf3gk6\noUsI1cdlfSyjMgKBGqhBcz9+fF7y5hVBMxKGfbWR06XCiQTxFFhSSHbDfMsth3TzB9X5k9m4+bai\nhuViqYWMjG37q1nNedKbv6VGapZirsti1Va1ApmqAZotJMoZ1qy3qaOQbiOJALZSYpqhJkGSXia9\n3BRV09RKo149z+UUS+K88rIquS/JSKNe7iXkNowS1zqK5p8yglxBUk1Ovn9OksgsI8d2IknpfARU\n8XQjHZ0NJSWJzVx7i/FxiT48mKcQ3U/q3krfUc20uL1MjEyVMmHEw0veOCJjRmXMRB+NtqfejvUx\nEVnlx1dKpaYJWjLCls7vnhEyaESZtYqc5guWCicSxFNgSSHZDfNHP3pIN5RUHWaq15aVxKmuTmq/\noiL1ENxU1EIrSaHePJldnq/IBvHMNplKJ9Q8o301INpk0tQoG0h0PTVTPHXQU0i1ttWCekrVBJN/\nz0hhMRHVkvYPdztFiZC6HiYS7Gflq5QkMrmN/oZq6AVqpm3UQRHlkmH9YyZIJI9dMUOaDdHyAwEK\nceY+iS49I+pyJvXGZAqqdE0Z1cAlZFspStVoKFk/tVRNBrNhp6yPXvLSelqf9twYHXOqc8PG10rJ\na6iqCRr/3k1uqqRKaqVWZX/+e8rstVJLtUrbfIiwWVhFTvMFS4UTCeIpsGTQ2SnlULJQU60b5lBI\nclBdvz654yu7+fZ6Y7dXh7amQz7NOrjyY02H/FpJCvVIitnlVoLNT12d9rnOZWidW6vJVDJyyH8W\nzBLJjBK/ZkrKpqwKAefHrafqZxta1EEvDNwozeCJYDtJRMxNRI90Er3STHRwE1FFSFILB0lSEk+Q\nKjRVfn2AYnMW1Y635RR1ia3Q2N/Kl5eIKuklAlUoN9OM/LUSkY+IlpFEPpkqzBNmtVLczLXNu/yy\n9tqTzLPW/gzZCYI1F+CbbaUo1dxDvX4mUjrT7WO6eaCsb63USh3Uodsvre06qZNqqTaOCKr30cvr\n1COJalLN5o1XS/XGq3UOEpHRSqpUtm+ndtPzZwb5ktNqBJDKDi+Jl974SBBPgXwCT5raE3yXGSVX\n7OZbTQ4ZcbJCtUuVhKWrGlpJCvUUV7PL04GarPHzk+ghQS6G/WqdW6vNjcyQQ7NEku/rthdesEb9\nZHfmTH5LwKasysflx13zjQM5odKboQ5Gt2VlPUCkaA8hInqjObpiVyCeMG2i+B/t5Rp94NcvI+lU\nGlU97RQlgmZuHopj3u9SSAc3pDhll82VVhkZfrz8pdess60WEj0I4NtZRbmRiWm1UpQpBVWvn4mU\nTjN91iJ56c5NqiG26mVa+ydrW289I2jLaJmyfjktV9RSEGgtrY0ZbyJyn6gfrM0SKtEkzwJXJwTx\nFMg7GCFNhw4dMk2u1NuHQlETn3RVu1RJGOtTaSlRa6t1JkK5bvKjBzVZY/NTXp74IYEVYb9Wh9pq\nXZ9Wmht19vdT5aOPEh55hNY++WTSNs2SXr6vlqmfzRT9ZaijrNyR8+Nu/cxsxlV6hmznDC8naVqd\nJOVfKoRHZkpHGiXFk6mVRBJ5XE8SkXRSlOydoHj1jpFHkJRf2UzxP/YOjWXLKaqOnlCts9EsNdM2\naqMILdPYl4XMNtA8tdP9tI1m455b8GosPzY9gqhF5M0Er4ZIIpW86ZJWO+qanlYglWvKaqUoEwoq\nc6WtpVo6EWOFZX2NUL7f6c5Nsr5pETrmUMuW8USQJ/VaYbXJ1vNQq5HbaBtVUzUto2Uxc3zo0KGE\n5D7RGEMUihnHKoqvYZoqlpKZ0NUGQTwF8g4x4YE6StahQ4dMkyut7a3Mq0wFoZAUAsyTJr79bdsy\nq+Rl+lhac5Vo/rQeDgQCUt5soocEVoT9Wk0UMkn+OzuJKr4VJYMdv/xl8v6kQXotU2rNpafpwkzY\nMD/ubD6QySTx1Arp1AqZDRApTGtjKJ4INXPbekgKzT2qsS7AvS8hiey1EsWUKymYDSnmP4ykrqX4\n0xxLTs8RaB+10/0UIkmpdXLr2yiWJPJ9Ys8tQnK/2XIWJJOuqpzoxpfvRy23H9+O3qXe2d9JzXua\nadO+TTG5p0aQCwZomVBQK+Qwai0yawVxZn0G6TvHpoJkfdMidPyy5bSc2qldU11cSSuphmpilER+\nfTu1Jzw2I6aM1Oo9MDh06FBScmnE+MjqEjZLyUzoaoMgngJ5DSuULKthdZ/UxkR8+2pSajX4Yzkc\n1h9La64SzV8iYpDquqWI5mYifEUig5XfzXxNUsuUWjNsIAHy3XgoHXRSbF6lYk4kv2fr1IRHTYQ6\nKRqey5ckYURKnSd5gmKdaJ1EMeVKquRyJSBJqewg7dMcDc+9rGzfQRFlPVMwtUirHpnTCjNOF4lu\nfNXhyTz5ZNC71Gu5Oes4kCM/aiaQSQW1kipTbncNraEKqiA3uWkdrYvLjWyn9pg8zGwoalqETs/Y\nqJM6FZWygRpiSFwN1VAd1ZGHPIbIs5aCbIRcbqNtVERFZCc7VVN1nPqsFbLMXw9WPpRYSmZCVxsE\n8RTIa2TDwMYsrO6TXu5pY6MUfssfy+pcRj7Ul/WhslK77TVrJHLs9Rp37tWaq1w8p/mGTZuIUDRL\nldsP0Imz+VdqJF1YnSubVZhwoNEsu0LRH9dKbjkLAV1HEhFSf0TVRIjPz1TnSbL9a7hlAYoNtwWR\nUq6k8ulGWj4bilmnNu5hY1Arsw00HzMNfD/V++qRueX0JoGICmiKmmnO8G2qun3+fSttTXCjHp/f\napRCVspzhqcbqd2k4rkUwQhGJVXS5+hzKZNBXjXlCZtWW2qVdRWtSmj0YwTJzIDYNowQrqN1MQ82\n1GqmVgkVfr3WMdl7PszWTFixOiS5juoSrs+EOp2JtgSyC0E8BfIaekqWkZAjK0iaVhtWq2t64aXq\nv4ni1dB0CShzB/Z4KEZ15cHmgFdE6+o0m9Ns34rwZiuRKHx7sZAwdFSDfSz2HC42rMyVzRQObT6k\nTTCbyTBb0dpUq4SI2aY7KT5nU4tIsWN5SSKMcbU5Z0PkPBCgdbMh8nDLeUKs7lOd/HeZfNxEl7DR\n8Syj/QSKxGxrhN+r2+ffd1Ak4Y0vTz55FTnZMVtnQ4QDAVo7a/6WOhdCba0AT5j4GpbphFeyXMMC\nKogjbImMeyqpMkZdNHJsLZKpV1qE35Y/Dtte7T7LHnSoS6gwoszWa4Uoq4lhIzXSalpNFVRBXvIq\nCibfp+cOPaf0lQ9JLqIi3XxbPoTX7DwZhcjxzF8I4imwqMiE22hnJ9EttxxK2GZnp0Si1CGd6v4k\n6x/LKwSIOkzGcBkdu1ESwZeZKSmJH1uq4Mms1hj59QBRcXHqtUpzAXqhvmkV0k6z5mXC0NFmMi+r\n5BhSmZ9s1zxNB1p9PXTLIe3zZiLPVbPsCulHKxttupmiXfNQVE1UEyl2LK380ZGHiMoAACAASURB\nVOUk5VOq19nl9tnx1X0yYrpjwvyYiIgq6Sg3ngUKkf7HJlbVjG3fbAqy+lzoHTPRPmawqA/HEhAB\nsyRBTTD1XFXNtHuCTlAd1dFROhpD2LQUa15lPUEnEuaA8n1gxkBaiqLazIft5yKXsryQChUSyfrJ\nk1E3uWPIHq+Qsu218j0d5IgZRwM1KLmjPDllCmYM8T4UDW8OUYjaqI2W0/I40snWd1AHraSVhuqf\npvMgQeR45i8E8RRYVGQiR9NIm+rcRUaU1PtqtcUTRp68Jirtkmo/9aBZA5Jrb9kySilcVasupjqc\nVw2myN58M9Hy5eZIZy6WOclEqG+6OYcJQ0ctMuRZTKQyP2b3yU4NRW1o9tWMraoOkm1qNBRVDS3V\nlDncaoXpMpWSKZ5a7rFriaiaoj/8Xnk/deivVq4pPwaiWALnovhanGq00pxCOvWOw8Aru2rzonRI\nYaJj5iPU5yURETBLEtT5e1omPKm0yyNRqCaf09hMUn3NNmrTrMXJ96GGapS/Wf9ZG1VURUwJ3Ebb\nNEN/WY4mPx6e9IKiYb8ucpGNbMpyJzljyGAxFcfsx8ZaSqVUTuVKrquTnAQCFVOxEsrMO9GCQD7y\npfXggEj74YNenqaRBwp8qLEo1ZJfEMRTwDDSJQla+6dyk5+sH0ba1KvRqS5fokW6tAje2rXax0rk\nCsv306xjrBZp5dv73OeIamrMl2BRq5eMUCdSXNMJ68zEg4d0kYkw1XRzDhOGjqZ7N5wDSGV+zO7T\nTNEfHe1LLQE1TZO1avY1C+etmZKNWRtaXUvUlpbi2UHxZJU3JFJvz9pMpBJqGRsZGZ/WeEJEVEpn\nqZyOkZdephMUJqJYIyKTzxKTIh8/qnqXfjPFzn0isxezRjBqUqi3v5F2UwnJTJQLqQbfB94p1kc+\nWk/rY9oopuK4ZexfBVVoqrAhCilht2pnWPU/plh2UqcSUsz+HZX9qBnR5P8xJZUnjDypraZq5W+9\nkijJSrlokVE98q+neKvzY3mCLFTP/IEgngKGkS5J0NrfTBgpI2Zqsx01QiGi5uZDScNXtcpvhEJE\nbne0/ba2+NItzEm2sVFS99T95/vKiClAVF0d22+WP7l+fTRE1ujcataA5OaSn+tVq4yTWtYuU3L1\nyLtVSmU+GQmlE8aWDzmHhpEB6TCV+TG7T3K1qZl0aUyCVUawbXaWag4coNbZWeXY/PVk1ZSq27FS\nYePb2qY6DlM8+Vc7xU8bI16tqm35nE+94yZqJ9XxVdARpd06OkxEiV1zGbKlnps9jjWhtrFHbSbt\nS199bRlREFNVpfT2N2uIo2cmxKBFOJMRW74PvFKqVjS1SCMjdw5y0FE6SttoG3nJG6fgrabV5CAH\nVVN1TK6o+l8zNccpxOyfi1xxKij710ZtRBRLolkb7zv0vhgiqVcShe9XG7XFnRczDx8SKd78MYWz\nbX5CEE8Bw0iXJKSzP0+kEtVrZND7AeYJkxZpJIolgcuWRUknH1ZbV6d/bC3VUC/8Vb0tv05PLd22\nTSKrtbXaYa18rmdDQzxRT0Qa+bqYeg8E9PJj9eY5lfzVXAzBXSrGHWmjmdIiYYuF5GpTApqWJoNr\npvgp468nrfWpQN1OojEnIzVsfR1JqmUrSWQypHEcXvF8pJPot81Ec5uItoa0yeoJig1p1TJCYghR\nfG4pwzaSnHWThdrqwUsvE4iomN5QFE8jqmQzZecjYPY41nxHxR7VgojwjCKZoqnl/qqnjqkJG58L\naaYP6vzKNmqjEIWojuoIBCqjMmqjthjn2lW0Ks4MiLVrJ7uynFcs+fxQfj92HPZPrX6q/7Gc0/W0\nngqpkNbROmqlVmqndnru0HMxhFhLzeykTnKQI649fk7MPHwwqngLZ9v8hCCeAoaRbghiOvvzpJWR\nIrPhqUTGVFsWXquX66lXTkTdV74EidNJtG5dPFlk27rdRHa7pIqy9bxxEa+WJqvdyfe1vT2e8GvN\ngVZupxFizeZCTRQzoY4L5AgskNE6+zupeU8zbdq3iUI5UzIiwa10mnfZyaYs1SlNR+FspsSkhl+v\n3k59HPa+gYhe53aMBKLTpj5eiIgc3LI6jfEw6E1/sjEkwwkKUx0dVkinUWjNs7UqqNTaJpkYZzcn\nNHZ0uUIw9ZAsz1Pt/qquj8mDN99hobJGQnTVfVDnZTrIQV7y0m10WwyBZCGsaiXRTnZqpVbNsFpG\nQhuoQXH8VZNBfj8b2RQFlyew7F8Jlegei82nlprJclfVbrwVVJFQpUwFIQrRKlpF62k91VGd4fMi\nkJsQxFPAFLKlRKmJUGur5Kj6/7P39tFtnfed55cEQIgvIgG+GaYp03QiK87YLhmxcRLGBVpT9ZB2\nQ9QTbhRvDtOzO+DO+GS3ezqxN+2cnHZ3JzOd05w5090507VmWuXNTCNbtWVFVhwqAWlVSezaieg0\nTc02Cd3IDi1LASVLFqm33/7x4Ln3dx889w24AEHpfnFwSAD3Pm/3Eryf+3vjffqBE9VNtrvbHrCm\npwUoSoshj8eMREQ7dq61vMRJX5/YJxol2rlTP1a5bXu7+bksRcItrxxg5batraIPdR5yrHKOY2MC\nQCWoc1dhO8ur05rK9pNJfVKmwUFz7F5iX8uN0w21QarkSrR4dZ7+XFrUKnwMNDV37d9ZcFsyr0u6\ng4g6SCTmWSZ/Fk5VXmG4XbOd2o/ltabhHJmxk8PFt3NkgmezzXwqnYNfGMxRjlL0DCVpkcYc6n3q\n1pmPfZD+xndcoVWitQJ10BQdq/Hldb2jplVO7pa6six2rqJEVgsaB6cttMUWdnKUM8Cui7polEap\nj/oMCyCPlYxTvATu+qiPClSgVmot+ayXeg04VD/roi4jk67MbCuTC6ngK/tZpMWShETy92ZqJgnJ\ncj0lXPJ9pFsuXx8JprzWqpxrO7VrM+C6ycmKHBTQhtoYheAZypdqZYnSgZBal9IJTqTLkQQcDnES\nZnXzUN1IJyfFe6OjJoyqlkL+Pi83wvuQ2wwNEW3fLl5HoybEShjkpUhUy6vbU42b5fGl2ax1TVVX\nYTXZkpNVV2e55seCz7urSw+XbueRrg+nRE1uCuKGSehqWyrfJU3SRASi8U+NEx4DjewfKbF4Vlom\npT6tqaUq53zqIPMfZz85g5cbdLnhxTQJwE2TSBTk2bKnaTjNxj2peW+i+F6l5UqsylGaxXB6+Xcl\nLm7zYp+FHHX7OI94wqMOepFQdJss7+K4PBt4kN9R1aiTWI02ndwtdfGdTmVUuHTwJsFMxmCqtTJ5\nEh75kG6ujdRIR+loSYxmL/XSNE1r3WFvpBstLqx8DLo+4xSnJmqidmov2UeCqlyTIRqiPuoz4JBb\nY2Xm4DSlCXlrO5PFv2AO/Ha1Vvm4kpT0lH3WLrGT7E/OLcxmu3kVgmcoX1KtadWyfKpJbrjbqkyW\no7OCSt1/f74EODmkcndYbjXk0NTQIPrnYOlmKZQxoXwO0aj5+cSEFWwleO3eTdTUZLWmTk+XwmUk\nYl0Xaf3UwTefu87lVo13la693JLpVRwUda7GKlw6jcWLi29PjzO4Ou1b7g2TEDxL5VbSpAQii9fT\nhQ8VaOrQlPaCvtLSM+kD6U1hTS3nfOomAUQNB9KUPjROy2sFW/BKk/lPtpv8u4Dy/acc3ucgqiYd\nktJhlO69YG1taRqnQ8U+/sGjFXicUNyn9cCvOZ5HulI1PcQvbv6ygqQn/lciRzn6lfyvBAZ1Xlwl\n/VqUy3W/LBdYdfGdWcqWuIryWEVuIdVlgOXj5/ORYMXhaIRGLEAnrZLSkikfavkS+eD9N1ADdVAH\nbaEtFkiVbekAVffopE5KUYp2027LPvL3IRoy1qOf+i3geQfdYXymAr9aa5UDorpuXs8RuYbcqrtI\ni2E2202uEDxDeZIEA+m26ZZZtlKpSW54WRPed0+PsN7dcIMAJlk+RAXCoSErpHIrI39K+JKApz77\n+pwthdzKJ8eeSJifxWJWILzrLrJk2AWIBgZKrbT8GY8TLS7qkwBxF2UJpXfcYXUB1kFzLCZeT06W\ntuX35oLsx67+p7Qg83hXL2DIYdWttqjTvqHrbnByK2lSApEerqcrLT0zfsjemrrZtUxETQyse+am\ntBf93LW1lfQA6SY7m5v6fpq1H7PpS3fYvaCVDmy8w8540VX1m1TwGMNZoAJN0icpS+s05nIepal0\nrmas6yWapE/W1BrjDRQFHHiJk/OSMTRNzueWCozlZiH1A6y8z920m3qox6ih6VbeQ3Ufla9VeBqm\nYct8JHgu0iIN0iDdTXcbkKmrwzlKo0ZioBEasWSb3UpbCSRcX50y2Eq42027tS68bg872OU1O3ny\nI7kOHdRB3dRNy7RsWWvuwtxCLcYa8DWV6+YkHmcrEzvZxdCGVs/NqRA8Q3mSCga1uJC3y0Db3+8M\nh3x8w8MmTKkgpSsdooIuf27daoISB/GJiVKrKAcoO5fZjg4TlDlk8kRCdk/pdjw9LQBOQjd3r5XP\naNRaz1ONd9WNWXfM/Wp6WvTR12e9MaC7aeHlfOLg7DdRVTVqc4Yimv72v6KeL/wBjR38iPbivByI\nrLT0TGGtQFNzemvqtSAJ1m37RwhrBeIX/RLKeC3KXtIDpJq11mtCH/m+tG52s77k064vvwCZZm3K\nOaYWcoQDacKhcZp0PMaV2U/dzqPqW2z9yRsomhfwbiBX6sJaesRM9+K/ozH6qOF+qoMR2ZbqFuvF\nmsnnprNU2s2Rw5WsoamDYNmmCmO91EtZytIyLVOWsrSNtlEXddEYjRlWOL59IzVa3FwlfPI6nFto\ni+Xz7bS95Jg0UAMdpaOONTtBoAQlSkq/RClqicm0e/BtZNKhDuqwwCa3uLZRm8XS2kiNlmRFco7d\n1G1Zg17qpQmaoCxlPQGi7hxRz+0CFaiHelzPYT83Wpz2DxMZBasQPEN5kgoGlV7I+3Wt5ODDwYW7\nmwIi4c7YGNFXv5ovGZ/anlPpkELB6iKrjkNtSyYS4k/pNlsoEDU22kMkB92hIevvW7aY20nQ5i6s\n3OVUQqZTP+rYec1SmUjJ7pjbaccOAdHd3VYXXdXqLJ929VPrHQyr4mpbq4KAVZKbW+umqF/q9RgE\nfKzKPZ8kEI0VoZODT5pKAXCZ3DPCqnDnRXz/ePHnMJklV/hy8XIrU5r97frVwV2SnXPZDXSl3kjI\n1Gmapqkj3+FoAaosTi5N6hET7sXfJh7PqloNdTDsBKc62SX90W3PIcWp/iQvEaJmgVVBTq4Rt0Dq\n4jl1D2nhbKZmCyzJ9VHrfcpHH/UZc0lSsiRuUyYDUh8TNKGN8XR6SIuwtt186fbqGsmkQj3Uo3VP\n5sfJDuacIM8LjOrPWO83Wtz2D116g1MInqE8iYNBEIla/LpW6oCoq0s802lhdeSWwnQ6b2yfywnY\nkVCmZlq1m48EwK1bS8fB4xjHxkSpFCfLJM9qC5ggOjJC9O53C3huahIutHKtp6etUN3dLdyFJeTy\nDLfqUwXdri4zjlXOq61NzHvbNvH52Jg1aY9TLU8utb6pepz4c3jYe7v1pqqAZ5rKu+rfQPG/l7ED\nG+fWWmkSIkNp8nYMvG7nUZWeTzrwkaA2RNaEQDrJbTuoFO68iEOhDm7TZC6Xrg6nl/Q5qnV1nIjS\nRYvv8DXoSl2J0pQ2IMEN4JZpuYw4Of0RUwHALulMyVjJhC83gLCOwhk4OKTw39X9dGNQXWr5Y4AG\ntO9voS22FsYRGimJ53QCOG5BnKAJCxyrbrcS8Pg+7dRO0zTtyeIpH13UZbS1lbZaYlJBKAHPOMUt\na5egBO2m3bYArbrX2sGcX3dqNZOvDlzlMZdj8+viXa5reChnheB5naoSeKzUBZPI2ZrG3Vh1yYMk\nmHHL5+CgADcJXdLaqI4XEJDqZT6yn927hWWRu6uqMaLSisctjq2t5u+qW/CuXWLMo6PWz+Jxsw8e\n98nHp1p8nZ6plFhDvk82ax27zuXW6diq544uIy+RtSxNY6NYQ79Ji+pVgUFPADUx7RTYGBXxv5fJ\nj2+cW2ulSYgMeT0GVTxWfuRkePVjhZPb6qDRi9z6cgPTAhENkrCG2rn7SqXJvGCYvMZdqcuV34tk\n/xfV+iOuWqOcsszq+raDU/tRuLevQkgHdVAjNRourHZjkBZS3cMteY+M2ZSPPurTutHaPSZowoCv\nO+nOEjjWWVjjFLdAZoYyWiufXZIk1TUYhBLA1Y1TxEJP0gANUC/1auuDykeEIkZMqLruXiyYO2iH\nJa6UyD0+d4qmLHC6SIu+zjE/51oo/wrB8zpVJfDoN75TB7mFggleKvx6HZtdCQ91X9XyFo1a3ULt\n5qMrxdLTY45XxprGYsKimUoJi2U2K+JKeQIcaTWV26sJhXTjdsvIq85Hvnay0Kpt8EQ9/Kkrp6Jb\nD+mq3N9fCpU6V9tqluCpVY1ZogChp4r+euWO0WuN1f3dRJecaKHKqjQJkSGHY2CB9zfX6sK3Mk3m\nP896NpJ7ObXTVHpBoJtTnTB/XcvvRfJGXlTbZUStVkZeDkZbaIt2DPI9p0y2TnCluuumKKUtkWIH\ngKq1NUYxo80RGjFKnzg9uHuu20Pn+ttIjTRKozRBE7SNtlGCEhSnuJHwCCQAWMZMqvGlTo9+6rdd\nd50FU4pbUmUbOkhV3+MAnaUshaofheB5naqS5EB+4/HsQFJ9X016Yzc2nUVUhbFbbskbbqNjY9ZY\nRvlsahJWOZ5hlccrqu6zlpIun12g9v/zAOFThwjNayVgytu99VZzv8ZGAae5nD45UiRC9OCDYtzS\ngtvWJqy0HNp57Ke0Yk5Oip/Ly6VQrx4zNVEPh+CODr1lUgVJt5I68pg4lXwJUkFY4p3EXSMDg54q\nynGMDmYzt3WUSaOOJ8gXAQVtga1F/GhgNxg08utqKw9ZNxE9liN6KU10kR+/HST8ZruJyqjXbtuf\n032FSsNeJVC2kzNYVvH+zDUlr+dUPSRN8Rvnqe7j1aJaoILFKigtnnaSVs8EJShWfIBEmREeCykB\nUwU3O/AqcWH18YhQRBu32U3dBlRL2L2b7naF50ZqpJ2009aKa4nVzcOSEMnro4EajLE1U7MFKFUr\nppObrXQJb6EWow27mwb8PQ7Fk0b1YH/nUajqKATP61S1TOYiLYPt7VagUeGXX/D299uPTXdhXChY\nYzwTibzFCiohTn3ybLRqoh4VVmXG2JERotH95gUpZuZKwJRbILn7bSoloFOXBddprKmUdT343HTW\nSb5GsZjVTVin6Wmxfr299u6w8ngNDYmSLzy+VNZWVa3adsmbpIK0UlY70zK/qNsMSXMcx5gmW2h0\nW0d5bh2S+3s0Q1UT4qqlat5g8AueaTIP2Xf4ix4SBNfO3uvXNlF2f3Yo4GUbJ1Xq7hvKKq/nlO5C\nv9YX417jPMsBVBVCOPQN0mDJPHn2U1kGhGd3lRlxJQQ5xYKqjxjFaJEWPVsivTyaqIkiFHF0//UT\n58kfUYoSj4m9LX9bSVkVdXu1LzU77gRZ45tUK6bOgimPSZrS1Ed9JZZQN8kbCLwuqe7c8xLfHIJq\nsArBM1SJOAzwZDPlXsyrsZh2yWu8goPcTrW4qVZPCXdDQ86gJy2R6ns33mju19oqxj0wIPqMf1pc\nkOIP9hsWTwmYXV1m7c7hYaLOTvG7jIHUuaByi6f8XE0cxK1Pcq5NTcIyK9fAa6kU9XjzBEHcasuP\nPb9ZwefQ1GQdqx9rY5BWys2QGddOtXQTJiJH30XtOjLT1keLrtmZIaJ1tww2vMtNYCVWFeQNhkou\nXnhdziEqWjpBRG1k/idtKv5sIcPiWYlF0ot767M5onya6K/HiVY34d/d9SrdhX6tM3h6jfO0SwIk\nS5o4/U3JvzkJjLrstmofur4SlLB8Zgd1W2mr4b56F91l1KEkElmHvbjx3k63G5ZT7iLcRm10E92k\nrdnpFGPpNOZO6ix5f5EWiciE92ma1pZsAYFaqVW7bk41W3OUM/aXllCdBdPufPT6PerkSu43vrnW\nfxvXukLwvMYUdMZZDjBBJBLS1XCU4+Yur06ySy40OmpNMCQ/y2ZNEFSf0WhpLCWHQPW9hobi781r\nhNycxc1WPrnltbdXuNbyGEhdtlf5HB01gXx5WV96ZMcOMwsuz5Y7NWU9dk6lUqRLcTxutcjyOXML\nsHrs5RySSatLMre+ejkXg7JS1hzcAlY5AF7RnP36LqbJ+MZezxLNDfqP79wMVuJqyrx4eYy66W99\nwWCazH+YWSLz+I2RSYeLJCydy/r9/H59ezlFLlXSQagNU7nlKao9Bp34uHbTbouVz62WIweGOMVp\nmZapn/oNWEtT2gJJEgxjFKOdtNN3vOdO2kljNEZt1Ebt1G6bEMfp0U/9xto8SA9aPnMaj5OFM0KR\nkqyzcYqXWDKTlDSATgKeXRxnX/Eh2weZGWydYjb5OqiWULvjbgekTtZrJzD1G98cZrcNViF4XmMK\nwoKkSzwjy4b4gQopbkFRy5DoMs+qQGpnfbUDWgGfeQMsl5f1CXTk0y7Jj/rkrrR2z85OPeRGIgJI\nl5fF2FU3XgNoYU1gpFqfcjnrfrIdCW7crXlx0Yz7VI8Rt3Cq41ePPYdCp/jaZNK+jqfduejXSml3\n3lU7vpOoSuVUiioHwGsxZ0Oq+StNIXD4lLx4aaOXxNLl856Xztb66EKH1UrKIy2pL1Y7608AtVOD\nLpWbW8hR+kCaxg+NVy2zbrnW8Uq+o+o1g6ddDc8kJS11Op0sWxxOucVTwouEJLs4TJ1lz+sjTnEL\n3PJEPeqjkRrpFrrFiH90cnH1+miiJtt27GC1mZrNmNK8eE+1qk7SpGUtpTtyP/Vb4lHVGwJeIY5b\nXPnfAt/fzXodlHWyXv82NqtC8LzGFIQFSU08wy1fEorsLJde2tZZ0uzGzS+u1f14vUtptRwelsCU\nN7aVgCQBk1tDYzEzGQ+3/KnPeNw6ltZWe0up01PWueT7NjSYbXO4jcfFdmNjRNu3C1jkgAoQ3XST\nsEr39YljwqGXx4WqwKZLtgSIJEb82KtQaBdfq8tQXI2YSzvYqnZ8J1F1wbMcN+FazNmQCjjXcppR\nN1Ipk2TkxcsYXSQQ0W35vOfdp0nkDBrz16Utl1YKY2kSh7+jQHSsmsGZsqMKbnAE0IS1vQNpwmMg\nPAaamqvOXZdyL56r+R1ViYKKkZPQIWFqjMYoS9mSNmV/YzRm1NFU64xKoORutNK9VloH26iNeqlX\na62MUMSSTMjJ4ihrcd5MN9PddHdJ6RW7h86t1u8jRjHP/ekerfnWklIuHdRhWctGatTGmyYpabFE\npihFHdRBvdTrOWZT/Vtwqs3Kz5GNtE7KucqbIyGwWhWC5zWmasS5cSul1apoXvT6sYCqF8xObrY6\n66uM7ezvFz85xE1OijZ5TOfNN5tWuslJqyvs6GhpzCJA1NxcCqLqvioE8ufWraVxlg0NZkZbbnHs\n7RXuqlu2mLGSvAao3bOx0d6FmMOZ2t/UlNU9VkKo6o7r5dhJ2QFpENZML/1v5vjOcrWhc/brqltN\nFSlq4fdzdGB/mg4dGqe1MixREsYW0+RMKm6fu6icpauwS9/tuYFpze47BNCR1ya8wvj4oXHCY6CR\n/SNVs3jWw8VzkAoqmYuEDrckQ7y/ARowwG+apmmURqmXektAkceaLtOyxY13kiZLMtruol1G7GiE\nInSUjlIzNTtCHG/TS6mVXbSrpOSJ7M8NNu0+S1DCyFIr20lS0tbau4t2WWC9gzos2WVV4JTQnqSk\nBS5VeFePm90xd/pb8JLddiPkNtfrXSF4hnKVvMBV3VV55lmvbn86yFT3zeXE5zJpjcy0qovt5E8O\nI3KsQ0PW7WWtTZ45trvbhMTOTgGt6bR1XFu3ijHoINzumc2WWhYnJ52TC3mBWgmd/LVdPOrOnVYw\nlzGYHOCcss7anQvqtkFY33TngQqi1yNghnJRmohAdOD30vTYY6DHHgPNlWGJKjZDh9xIxQPJ6CCm\n2ol+gmwvTc5gWrP7DkpH5bi5eh1rmrzBfWGtQFNzU1WDTqL6uHgmKt9SqR4nGVfZTu2eLF1uoCph\nJE7xklhK/rnqjilBUX1Id1g+bxnbKOMWVbfdSZo0YkaXaZlylNOWPOHxjzImsp3ataDHYbid2mmU\nRn0nE2qmZlqkRUsdS/mIUpSWaZluoBuM9/qoT5tAiEPsIi1SlrJGsiR+XkhraDM10wRNWBJF8WzB\nPMZ0iIZKXGjtjjn/W9gs2WX5uSLPn1CmQvAM5VncXXVkxBpzqVoj7axWOriQYDQ0pLc+qjAr4xgl\nnG3daq1zSUR08GC+JK6Uw58OIKUFlYMjt3BKd9JUyjpGaaWMxcz95fqoVtOJCefkQl6fst22Nqul\n1OnJYzCle+wNN5juvF7Knehg0E9iKCc5ldepegxjUU61Ju+/P1+7BEZBB6ZdyypS1KF/O06PPQba\nv3+kLIunhLFMgWjdiVQ8kEyaSiFGfc+PW2TQoOfWXr16UlfTzbVe5+xHQbvaluvyqx4nr2VQpCSo\ncusaV4EKNEiDFqthP/VbXGylCy6HUB4Tyl1sG6mRuqhLmwhI1oAsUMFw2+2kThqlUQsAuSUPmqRJ\nC/DJtviji7pogiboZrpZC7FOjwZqoDjFLVlpB2jAso1M5sMhM0tZ57HnRdvSZTRHOQtETtCEAd9S\nTomJ+qhPC5perPzViN+shgpUoEma1LqBhwrBM5RP2ZXU6OwU4MFdOHWw4AQX2ayAGLWOZTQqXEol\nHOksnmpf/B/w9LR1295eot27rRldOzsFhMnX0u1UQm4kYoW7m282614++KAA7rEx03o4Pa1P4HPz\nzdbsu8mktV272Evd0642qe7Z11cKS2pSJd3xUuFPB4O8nWw2mHNLPVeCKOvjRbpakxK229rytQPh\nNJWSy/WqHFHu0wuU/twBGj9QekNAUtTamwWam5sqCzpZM74uE+wscDqIHj3k7QAAIABJREFUUd+r\n13g8ovrypOby4+bq995Nvc7Zj4I+p8p1+VWPk992dKCqWrs4hEQoQsu0bHlPlvUoUIHaqM2oNxml\nKKUpTYu0WBL72ERNNE7jtkmLnFx9dXU6pWVStsNBbIImaJImSwC0h3psrY8gZzdaPm8islg9pbWT\niCwW06N0VDt2Dp58rmqCJ102WTWBkwRVuQ47aIcxhjvoDuM4qVZ+bjHldVT9nI+bxUp6PSkEz1Bl\nS2c11JX/4HKCC25RtXtyWJTupmqGXFU6CypPVARYLZuAsIoS6SFXzaLL40mlFVdXN7S93bqfdFWW\n1uLhYQGuXuAzEjH3c4sLvfNO/dpwF9xEQr+Nenx0LrUcgCfss6P7lt1NjmpCn67WpHr+1CSZz7Vg\nfglKaaL075XeEKgHqZYdCaJjh8Zpcq1Aa4x+VgtWsAmN2v7lx801TaX3bsKLUH+yc3N0q5+pHiev\n7pK6ups6i+IgDRpwFqUoLdKixT1WhUK1lIgENOn6CrK6uU7SpKOrswS1buo2Mrn2UE8JFMYoRgM0\nYFhH5RyGadhYwz7qc6yLyduyi/lUH73UWwK6EmrHabwkvvVBerAkVvNd9C7L6wQlLC65shyNnAfv\nSwJvK7VSL/XSIi1a1pMfjz7qsz3/dJZYWW7GqzaLlfR6UgieoRzllPBFjf30Gy8o2+AZUe3KfKiA\nq3uqQCLHLuM3pWtuR4cVJoaHhUVQvr7rLnP80uKpWg7V9zmQ2MVwdnaaUBmJiO102WNVS2ZLS2lb\nN91kurcuLpbG4N5xh4BAmWxJJ7l9ImG6yMr4Wul+qx5PXYwlT3DELZ5uyYL8fK4r7eKkcmtc6mpN\nStgeGtKXpqmKrgXzi07l0NY40finijcE9u33Vwe0ynSnWnZKXEHTVEo/RTl8VKvhbwqpoOI5CRCV\n3rtxvwgNV9xOfO14rKTfi3m7Y6C6cKqAYRe3mRWVbUsghceT2sV28mytchsJk17qQKqWPd2Dx2hO\n0IS2NIx8SCBspEZby6Yue+xtdFvJ9iKD9pjxuo3aXMfKH1nKGvsnKFFiUZYPtV+ZpImveYpSlpsV\nMlFTC7U4xvzKY65aTP3oWkvUdS0oBM9QjvJiaao04Qvvo7dXD209Pc5JeWS9Tql8Pm/ZXiYq4hbN\nyUmigQERI8nb2rLFBGHuOsz7UPeRGWbHxqwQK2HXzhoZi5mlUCQkqZlqJyas69LWZh3X4CAZWXud\nYFOFMbdyKV6ti9xia9eWHCMHQbdzS3XD9nOOBWkhlet08GC+soY2mcqFd0elyZ22VBWICh9fo6lD\nc/6gs9z+fEi17JS4gjpYrt+fz9NjOaKX0kQXbTinysPfFFJBJU3e1kR378b9ItRr6/Upe1fbyoHa\nLlYyKBDgbqEJSpS067WMBgcsCbbLtEx91Ee7aJelvAqfx27aTXGKW9xQdVDNb4RIiynfp5VaDRhT\nY0kbqIESlKCx4oNDle7RT/1a0OSPCEVojMYsfycyoQ2PNZT9eQHQERoxMgAn82Z2WhUE+WOYhi3J\nh1RrKwfRbbSNGqiBOqjDsdyIPOYyYZGbpd2pjRA660cheIZyVC1qBaoZVgcG9Flao1EBoNJNVk3c\nE4uZLrf33583XEnV7Vpbze2cYFYHwm1tzlZZoNRtV93faV9dW6OjYrzcNZaXs+HuuzrAk+JuzNKV\nWEq1DutA0k5eMt3yMcpasBLQ29v1SYn8nnuVWEid2pL7u8VPVQXUypRTkiSvqop7cz+Jb/12Io9l\n3CpTNV2WlWv53EKORp8apdQXU7R8tjg5B8v1wXye/jZNjpxT9vADNNw5NVUL11UVMio5pO4XoZvb\nx93+OypNlQB1jnI0SqOUohQt03JFF/N2+3JQSVLSk8VRF3/pBsV8X/67as2TtSpV8e04FHLLX5ay\nRrvc6sgf0vq5TMvaDLQRilCi+ODv30V3GRDHXXNl1lme0Ib/fUqwlmP+Z/TPjBjXrbSVQKA76U4D\nHo155k0An6Zp6qEeSlHKaOcOuoPaqM2SXVhdyxEaMSC9gzrobrrb8rkb4OvcrYN2mw3d8GunEDxD\nOapa5Sv4RbrqzukGg5OTArDuvlufYEdNgmP3bG52r4M5MWGN19QBMf/8rrtM6NHFecZiRNu2lcKw\n7iktofK1Gv8qY0TtAC+Vsh43vladnfbWx74+/y6lbqVP5Bj5vDlI68BGdcN2Go9aq3RyMjgrvFfo\n0u1TVRh1IAJdkiS/qspNp1Fyvv5V51QpQFXgsszhfXptrXQYaSICUe4TOUr/cZqSe5P+M666cE7Z\nwy+OrUzO8NxU0BeCumRNKmRU1wv9WvVxrwyoaxEnJwGNw5TXWo9c8nxRrWNu+6oJdiZoQruPLhFP\nkpKOCYl0YCmz5eYoZ8l2207tJdbCFmqhJCWpl3ot4M8trmlKl8xL5xrLQVW1KHJglWNoozbDKqlr\nL0tZC3T3UZ+xRsM0TDfTzTRKoxYrKV+PIRpyBXw1gZO0yAYJimEsaO0UgmcoV1Vy8Wy3r5P1TS03\nwoFJJhLigMVrXnZ0WEHHCRL5Mx4XsZLc4ifb0sVY2j1lzVEny6bdGPizrc1aN3RkRGTilfvyhEo6\nwNNBkw6ypQVX164fOYEaL7fC4VBak2UJHlnOxo87rq5/Wau0EpUDXbp9qmI1lEqTLRHokiT5VVVu\nOvktIKm+rqE4vPfMzZUOoziX9B+mDeD0mnHV0DQR9RDRGAXLOgEa7pyaCjp+qprlUjazKrfGVAbU\ntYiTU2FKV0rFCQ6cst6q2VgHabBkPXm5FAlDuv4KVLBYOhuowYDBQRrUxocWqGCJJ+XWVBXmeqnX\nYiUdpmHbcjRqjKm6JnbZanlyI905pQNMXvJElnqR5wNfD2n1tLMkt1EbpSltZPX1k2DKzkIdBCgG\ndY6HllN3heAZylWVXDzb7cutXTIhjYTUsTErnMm4Re7CKsGVgyJAdPSoaOs3fzNvAGlbG9GuXaIN\nHhspP+eJfiQ8yJqXuZyZPdfrs6fHCsPlPmVdTHnxr8v4K7Pocuux3E6FSDUL7siINe6Vj1l3nNXE\nQxxInECNnwMSNmUG36kp5/I4XgFQPW6VSgddbq62un2q6qruQAS6JEl1IbfrX+mK20HCFXcDPR/H\n/2MR3v94P33kzbXSYRTnMn5AxHUOPzlMk9+Y9Ayd+Xy+emAdoOHOqamg46f8lEvxKruSN26f1ZO8\nXmRXq0RPLePknGp+6uBAVzNSVxfSLjkR70Odp6wnyhMVEZnJiiIUMepmEjkfJ9l/kpKWtnRw2Eu9\nNEET2lqk3HrL4yZlXCfXNE1b2o1SlCZowhXcLLGcebNfuT67aTf1UI+tJbSbug3wkm0N0ZAFvu3O\nY96WUwbboG+GBHWOh5ZTd4XgGcpVlVw82+2rS0ijA5TOTtMKpsueq0JLf79oK5nMW96XcMsBZedO\n676yFidPzuPkshuJlJY+0Vk6t261utfq3HXtnhzK1f10WXQl+HAglxAnwYjDrNyupcVshx8rDrXq\nWnM4dbKOqTG8dnDGgdgui66dBb1aLuFc5VzUyXGtTVNgMXdm47T5vAJzRJQioiQR9ZFwvR0nyn2z\n6Nb62UNUaF4zQczrHKuQjLQwtkZTuTkqNK/RlRTRJws28LVWoMHHB2n0qVFfAJPP5zd7SGHg8lMu\nxaucrKibxcLq9SK7nmvDepXdXNU4UyldPKEav0lkBQuv62kHqMu0TP3Ub4zDzkrHrV+qO6uUTACk\n1vPk/cnYSgl63FU1RSlLXCcXX5sYxSzrxqF6N+22WOm4C246X+rCy/uXVkv5Hk9eJMcst/Gy7l6P\nTb0mDQqz6LorBM9Qrqrkot5uX/n+9LQ+IYwOLHt7S2MPuWtpczPRrbfau7K2t4u+BgZEuxzOeNZZ\nHhtp57KrezY2Wi2IgJkJ160+KX9yy6N0Q+Zw2d5uhWM5RlnjNBo1LcpuNwuWlwWsLy/rjxXvl89h\naKi03XKhUAfEdqqq62o1lSbzG28zjTsoSTBMkva/QPrfspjU3Jx/EEuT4/qWZdmSUGjXLoPd9P4y\nAaZGNw9yCzlKfSlFyb1JGjs4VtfWvaDlZEWthoWVKyi3u6AusjeDG6DdXDnsyBIqRMxiuADCAVD7\noXYaW9NnSpXz5zDkJAlnOrdfLg54TdSktQS6Wb84vKnQorbDrbsyVlQnbmXdTbspRSkjHpUn+OG1\nQb1Y6Xj/smaodDWWyZB02YW9nMf1CpRetdnHXwuF4HkdqxqJT3Rt2vWjJoTRlcxQwU9CIXfL3bVL\nJMRZXnbPOCthUP4uE+nwGpgc+NRapX7cbrkLL3ct5gApLbtyTCMjJlwNDQkwT6XMz3nNTSk5RhV6\nm5rKi9fkUq2V2awJvepx1UGhn/PB73g2OnOsL13vlq00lX7zbyVjTcY/W3Rr/cx+KrxrrXSN3Cya\n6voq25dl2SqQsM7aHTc2p/HPVRdgvMgJrvn8a2LdCzKrboXusE5WVDcLayV95xZy1HGgg3AIhLX6\ncLvbrG6AOcpZ4gg5bBkxhgfM8xtz+vnp5u8E405uv1x2pVzsXGT7qd82FlRN8qOzpMo42K20tcRa\nyeWUtZdDrt/yOGqmXrk2vA9etiaEsFBcIXhex+Kg4FSGo9w2JXzYWan4+3YJYXSxjWrWWt6macXM\na2GQu8LyupyFgtVS2tVlXYvpafdMtPLz4WFrQqQtW0wglv0nkyJZkEy6s7hoQje3BqsgzRMxqQDH\n3X6bm/Xr41fcWukGml6T61RitayFS62dKnJj24xusQ4yMr7+x0NUGFtzBwwJhkNEtI2IukiAyaTY\nr/CRolvrhzTQSaS3aHK4WSaiQTJcd9XsuY6WLe7+y5P85IrtpEhf+kXOqY2oMF6gqUP+XEQt51MA\noOYE13L+eAw09MRQ9eE4TaXHq9ymNtAdtpK++b7JuWRNLr5131EcrCqpv1lLOSUK0pU56aZuAfiP\ngbAfNLSmz5Sqy4qqxobabe/FSqeurwqSdkl7dHNWt+fxjhyI3ayVunjQLuqiu+luT+Vx8vl8ydjs\nrLN8vexci0OFCsHzOpZdGQ5dVlIvUJrLmZY9HrtpZ6Xi8Za7d9v3weFzZEQAmewnEhFWQGnZW14W\nVsydO/M0OSmArq9PWEWzWSuQybnK+cnkRRxOJZDrLJa6pyxxYrd9U5MA3HTaWiNUTbCki6lU3VtV\ngJP1TQETouWaB2HddgNNr8l1amG1rIY1//7787Wt01lSJ7Ly2pxBjSe9X3GNdQOMaSLqJgF2HApT\nJECrQFZwVNdXZzFOs3bUDLiKpdLRssX34/NQ21dVIJGRVreNB5C0QIJbXzopfTjBdWGtQNlvZH0l\nP6pIQWbVrUbCIY8up5X0LfdN7k/S8tpyTRIZ6cCTw8skTXqGgY10y1Utk3aJeaSWaZn61vpo19wu\nmlyzd6F1sgB2U3eJFVIHZE7r4uZmaUna4wGA5fZ8bNM0bWw7TMMW2JVtcYsqh9Q+6qMsZT1bconE\nOaUeDx5vyy2uEjaDLnUS6tpSCJ7XsXidRGkpdMtK6rWkBbfM2dVjtLPs2dV0lFDD+1EhkqgUOvjr\nrVutcKa2199vjTXVZVyVTw56gABgOUfdGNXEQ9yCqovllJAcjQpwVt2UJdxKIFVhV0Kwn2PoJC+g\n6XTcnN7zIj8wGcR8a9Gmc4dkgZEganP6FV/zSwwYxz9nZnwtNK+5A0ba3NeAQj+gp7MYq3DDXy9r\nti+ZXLHPbtZvJ5nwO6a0r5MdYDnNxU87TlL68J2YpwoJmQwFaOGvSsIhjy6n5SaOkvvycfuxngYF\nfQu5HH0unaRPjYM+VNBbAe3k1y21XDnVyrSzHPppy06yj67iQ8YmSpfZDuowSoNwleOuLMfVR33U\nRV2UprSREEgFYF35EA6KquWSx6vqLKrywbPe+k2Ao27P++HjaaZmGqVRRytyqFAheF7n4hfTjY2i\n3Ih6Ye+3pIUfeFXjPLnLLb/o1SUh4jGN3Bqo9sVfSxfYSERYQ/m4ZfkRnuSmv9+Ev2TSatFdXrbC\nI3evVcu/qJ8DJuzzsfM15GCbNXMplMxxYMBqsQUEYPNYULdj6AXq/ABjNSyOfsCvGlbVmseXKjBi\nV5uzGmstxdf8ZQmMbUSF8TWaOjRHhTfXvAGGCoW62Ek3+FJBSYUbJ9jRQVYzmf+FeologIja2XsD\nZFpp7eZn16dfkCy2Y2T39WLVrtSqmCbPcFxNRt0I+bnwDsrV14/1tBy40elAOk2PAfQYQIemsu47\n8PFq1iiocXHp2iw3QYuf8emgTs5X1qmULq5cXhMO2Y2LAxt3fx6mYduER9zyKQG5kRpL5qrW2eQP\nNS7Wz/qq2/NzQ433tINoVZsh0VWo6igEz+tcdllbufVwdFRY33RQyuW1pIadu2gsZoUl/hmHsMlJ\n0c/u3QK2GhoEaHV3i/cEHOaNUizcmru4aGZx5ePm7XOo0Vk8uSVRth2JmCDc2ioAVk1Y1Nlp/t7R\noc/iyteQWzClRVRCBp8THyOPU/Va7kRda/XGQDlQE3R7RP7ArxqxoAcP5stus6x5K1BjV5uzmpZY\nvuary2R1LZVusl7E56LGTkqqGSOirEObHBQnfE6EW1nl9VeEvddHVhBLUkmcqC95sPhp3SL9WLUr\ntSr6ANc0lb8U9Si7JC66i+CgXH2dLLcq2JdbkkE9pw6Nj9NjAO0fGaE1n19cOjipRqmIctq0O17l\ntCX34eAnrXgt1FIClxxUB2nQm8u24mLLkxDp3J91MZU6SAYJ92PVQrqbdlOMYsY2uhqfXqX7nuLn\nBo/3lMDrBNFSXm4ShHB6baom4AngnwP4ewD/AOD/0Hxek8mGKhWPn5SWR7vkMJVc3NqBgLywjUbJ\nyACrfjYyYnV/dRqbaVXMWyyAHBZ1cotD5TUmZabZZFJAX1+fgHJ1LBMT1qy1sg6nhE439fUR4RML\n1PjoAUrvP0TT/2rNYh2Wc3JbJy/ycmPAz3EPuj2i6sCkH1WSXKhWcOhpbXyYr0rW3K3EiBellf3V\n13aKsu36HLbTzY+XcZGGn67i6xYSACznllReczDjbVdYm1V3PtlZtasiH+B6rSdldroIroarb2n/\n1j+Bci1+6jm1VijQ3NSUb+hUxcuQ2NWM5NtxUHCDh3Lmane8ymlLdxPibrqb4hSnRVos2Z7DrddY\nSdmHjIF0S/LES8d0UqexdqpF0y7mla9PH/VVBG1e/u+p8yvHfVenaljY60HXO1BXHTwBRAD8I4Bb\nAMQAHAdwu7JNjaYbSienOoryolYHparKseo4WRv5Ra9T4hqd+6pfCLODGt3aqMDLE+2o4KnOT+c2\na6fRUSL8nmkB6fmDOQtgy3jS5WUzhnZsrLTWqU7qsXK7MeAXZCttb8cOcc51d3uD9JqpTJ/Darrp\n+gbyNOlBz8vcCmRaD9vI2Q3VTk6xmbwtOZ5+EtZHCZ7NpM8yK/fpoNL5yXjN4WIfO0iUc2kgoqNs\nblNkAuUYGVl3DaVZ29z6202B+KHaWbX9qBoXNZUaV+tdG130vd7B3isA6LarBjx4OV7l/h24jZeD\narnnjRsg8xhJPhavgFeOO3Ct5eUmwUb/XVZL1ypQe1UtwPODAL7BXn8GwGeUbWoy2VD+5QSlqoK2\njnkdm3Q1veMO6ziDiEnUvc8hU8ZSTk9by5lw91jd9p7X5VPCAtL1+f3UceOacROAW1jVONaentK4\n2HITRvEEUEHEEXo9Jqplt26UJj20uWijrbUWVZoQp0DeXW51MMspxqlkCR+PfG5h2+na5vskbfok\nssKpen7xNtR14GsnYbbNYXsnVSlwkl/UDC4MBp5JNcjsrG5tldNXOcBRroUxCOUWcjR6IE2pQ+O0\nvEE1YN1kV4/Si6trNeDBy/Eq9+Lez3irdd5Ii+hW2lrW2vnJWstVb5a4jfy7rKauVaD2qlqA50cB\n/Df2+hMA/l9lm5pMNlT1ZFdKxYt0F+V+QFC3v3QP8dqOHYjp3i8UrPGa3d2lGWUjEWuNUO72K8HQ\nixV28uNrlD00R723rFksqSqs8wRJvB87uNTBvpPF2glUq5HcRlquW1ocQL3GGU/y+Xz9mya8yM58\n5WZ55Ovs1eU2zbbpodJjxT8fJGs9TQl2EhKdQHmw+FpmqG0iors1/UnJ7aSbrZd1ILKunfxdzX7r\n8bzM/0q+PGB1kcUV8MCokRhn8PFgIDTIuppubZXT12axJsiL/OSBZGDrWVGtYQepAODH1XWj4KFa\n1shaqBzXVa5y5647rtU6p65n1cM5tpGqBXj+ixA8rz05gRsvpVKu/ICgTvLL0ms7bjGeaj1MCUZq\niRTVBVeulQRTvr1d+Red1ERDKmzL19y9WP4us/W6lTThayLrl8oxlZOxuBItLwtLp1N913Ktj+WC\ncj6fv7Z9DnmtTTvAky6lu4koVnyvtXQfucYvSsCzswr2F99rJwGK/L9HlES22UVyB2WeCKi/uJ98\nLV3bORAuFrfT3dTwe4zV7dM2c1WUf3++KjcxLK6ALDHO6FOjtoDjx7IYZF1Nt7bK6WuzWBOMi/xD\nCGw97//P91e9VijR5ljjal3clxPHWkvxGpt+3Wx1xzUEz1BBywt4RlGZXgewjb3eBuCEutHv/M7v\n4JZbbgEAJBIJDA0NIZPJAADm5+cBIHxdR69ffBFYXBSvs9l5XLgAABmMjAD/8l/OY37euv3nPw+c\nO5dBSwvw8MPzaGsTn8/MAC++OI94HHjuuQwSCbE9b296WrQ3O5vBK68AwDze9S5gzx7n8c7MwNj+\n3e+2bq+2DwBtbRns2QMcP262Nzsr5vfpTwOJRAZLS8DCgvi8vz+D97wHOHJEtL+6msGpU6X9vfji\nPAoF0d/amv7zxUXx+cyMWB91PoODQKGQwdCQWN/jx4F9+6zz3bcvg9VVc7wf/nAG27cDp07N48gR\n4PbbM/jxj835qfu3tIjXt902j6YmYGHBPL6f/rR+fQDgwgXxemQkg+ZmYGio9Hjqjo/b65//PINM\nxlzvmZkM9u1j2xfHO3/bPDANZOCtfS/r7fj64Xng+Mb8/dn9vQBAZjYDLAHzP5oHUkBmWwaYBeaP\nzwOfBzLnMkBLcfz/n/K6Dci8lgFOAfNH5oEskJkv9l88vpm24ueH54EOIHOp+Pn5eeAIkLktA4wA\n81fmcf+3gP9wJYMLAA5F59HaUDw+I8D89DwwX5zfADB/Yh44C2S+X2wPxf4uZ4CTwPz/Ng/8EZB5\nVJlfKgNkgfn/eR74v1n7fzgPfJydD9+ZB4aAzD9lgEKx/XeAzM9t1vv4PPAwkEl4PD7q9nK9RjLA\nHof9n8sAM8X1CPB8Oj5/HA/jYSQyCczeO4vsf8ni07d8Gv/18n8FANy2chumb5mG1Pz8PF489iIW\nexYBANn/ksUf7fwj2/Yfjj6Md95+B09/8mkk4omKxsvHl4gnfH+uHd/8w3gH7+DpzNNIIIEH/vQB\nnDh3An3DfZi9dxbHv3u8ovUN6nVLpgUA8K7ou5B6O4Wvf/LrFa/nucFzWFhYAADMNM1g39i+qoz/\nYTyMv8/8PeKI4775+/BZfBYPZB6oyno9MP8ATuAE+jJ9mMUsjs97P377YD//2cwslrCEC/MXfI3/\nxfkXsYhFIAPMYAYPzz+MF/EiFjPFv5/5LP4I9n8/1X7Nx/cIHsHD8w973n8Ws8jOZ/FpfBqJjPh7\nk9tsxPHbLK8/j8/jXOYcWtCCh+cfRhva6mp8G/36+PHjWF1dBQAsLy/Dk9zI1OkJIArgJxDJhZoQ\nJheqa3m1BqkWsELBTHDjx1XT7n03i5xM0OPVPVS1wpYbc8fnPT0t5ptKCQtdoSD6UZP76NxgeXkU\nLy7K09PCdVa1XHodr594TjcLp9N+QVs/ZR3V9naNy22Z1sea1+MMUI7rm6bSb+ApzWd2mWSl9bGD\nrJZAnUup6gbbyNrrptJxgIjiZO/Wyi2iTez3rWwfp/mp54Ic3xBZraHlWBj9unRXySpeqWe5U3bW\nIK2YGyl1jXILOer4i47AXFmDVDUscrU8jrVyafbaj1+rY5AxoPVkAa6nsRBtHtf3SnQ9zDFIodqu\ntqIPjAN4FSK77e9rPq/JZEO5y2/SGa+lMry6sjpJt61dn/l83ti+u1sAYn8/0Q03CNDzC3C6eftd\nK7eSME4uyuUCHS+X4we01ONb7g2JSsVrlNrN26/rbLk3HerB5chxfSXE6WIinTLJyiyucj8OdFwc\nqKRbboqs9TBBlrInV9X/BmoiomkSQNpAJmguklnqhO/jND+nscr9hsi5Tqid0uS8LnZyIcWS88ll\ne2MYCwvUfeAAjR86FFjJlWqUDAkyCZFXpUm5v8JiRKN7orR8Vr3zUXtVc10OPnew6qVfpGoFOF77\n8XvxH2QM6DRNUzd10xiNbQjsceguNy7UTpX+36s3EK6Groc5BqmagKdrByF41o0qAQenfe0son4g\nwKmkitpnPp8vyXprF4NZrvyulZ/xV9qXW79+VckNiUrkZd7ViDHVqR7A03F9JWwtU6nFTbXC8ddp\nsn4jDyv75sia9Ee3j58nLz2ia2eSSpMXDZKZ/dYu5tNOfK7lmA3LTSiVJkdgLTmfXLY3jMMHzBJL\nU3NzPgZUW7klBqoGgJXcXylaAIO2eFYy9iCTM6kK4jvKq+WwVglSvPbj9+I/yPFvtMWrmv1Xek5d\nD4l0roc5BqkQPENZVAk4uO0rLW/cVdar7KxaXsar1iJ1c2v1YkHL5axutuXK63pvdDmOoC2ZXq2U\nfo6vtGwHmV3XrxznVY30v5UqRwIo+TdykpivIlktoSBhjZTutPIz1erZSPpv+1b2+6Cmb5CwSr6b\nvW4hyv1POUr/XprGPzVOhY9r1q7Ex5L0gJlm7Xq9PlOh3av8AqvL9nIYY4dEiaWR/fsDs3hWQxL6\n2v68jca+PlYCaNUAsJL7K2sFSn0pZet+Wi5AljN22Vf3F7rr2q2gjvvyAAAgAElEQVR5oyHKTm5A\nvJEX/xtt8apV/0ElUaqnZEyhaq8QPEP5VrnXz2pmVrUdv+U8/MLL7t2irElvrwmLuja8WND8lBfR\n9VGPDGKnHTtEjGVTE9HiYjBtBmml1Fm21ay8VRWDnPudXIMtaYPnaloKRjdWV8ulGguqPmVWWB7/\nKZ8xm33k+2omWv7cWfpe+vfYhf4fTJWuGR+nXQwrkb+SMZXKL7B63L6wtkZTc3PeoXODvmwKawUD\nsvAYqPsL3RbAq1U8opMbMQfI1FzK80Wwl7GrF9e8r/6v9FdtzpVakmsNUV7HW69ATLTxFq9a9R/U\nMajnYxmq+grBM5RvlQsNMsZxaEgfI+k3RtRpe517iG573Xt2JVT4dZuf8iJe+3XSRoIqtxT393vb\nx6l+aipFFI1az4UgxI+Jl9hQv7J1OUqT8W224BRPaxngJT0YBakdJCyS3STKn6TJamGcIhPEtpIV\nDGMkypvE2fYgUfYEJJIB9RHRDcU202zbDjKhspUsMZ8EEvGcaSqFVYfn+P9avND/zAgVmgti7BwW\n1VqadoBpB3dpqv7xUFQz1+1a+aJrxK2eqoXQa1ypDkyCctM1XHH3g7Dm7SI4t5Cj0adHKfWllGPM\nqHpxXQvQzufzWmus03qpgFxriPJqPd5oq6IfXUsWPf49FdQx2EzHMlTwCsEzlG+V63apuk2q7bjF\niPqJj9Rd1Om2172n9qW7bnNyAfUyLz9rmMtZ4a/G145GzdKWFu9uxXZu1Xwty3G5dpJbVt5K4d0W\nFMaJcp9YoPQfHqCx/Ydo8uNrNjGYfICkB6NKxWFTwqSEPf6tK/uVILZc/KnW0lSfSSJKlL6f+0SO\n0v+m6ArbXCiFTd1ThVqQ6aIrf24RPwvNBZrKTYm2ZQxqmu2XJfsYVjdxd2M1vrWKqhl4bmAaZwmX\nY18fc3S7dZIOTPSwkqPcQorSB5I0fshbH4W1AqXmUoQ194tgCW/JvcmyQKkaCZxU5fN5LeA6wZ0K\nyLVOCuUVyDfaquhH1bLobQTQ8u+poI7BZjqWoYJXCJ6hfCuoeEO1Hb/tBrG9nxjCcpMIeenXq5tx\nMul9vkFZSZeXhaXTTywrd6vu7S0Fbrc420qlW/OqGX4KROk/9pnwxa8bpk7c4icz03Lgk7DZQtY4\nzRgJC+E02397cRs7F9lGTbvsaXGFzU25f+tr2qBeInpQmUNv8WcrCVDtJDPBUVDwnmb93czW5Fq5\nJtroAHEqdbv1E9OpAxM9rKQpfcBfIqHcQo5GD4xS6lCKltecv+A4vEnX4dGnRm0BTXdxXQuo0wGu\nE9ypgFzN5Edex7vZVS2LXuiiGupaUAieoTa1JFz191cvsUwtrtvsoIjX+ezo8Ad/G+hhZ7hV6yzF\nulqntZCvGwg+Y/7Giwlf2v79fhr7iI3FM2ilyfwWVWtnthDRUTLjMKUraqvDPk5POyAtPsc/pbjC\nSlj1Yvm0eyaLY9eNc5Lc4d3rMeS1RLk1N7yuC1TluprqwEQPK+M0fgjFPoY99SETD+ExUPYbzu4X\ncvxDTwxR9htZGn1q1Deg1RrqpJzgTgXka6Wm60aqWha90EU11LWgEDxDbapEN6pU100JOUG6sflZ\nn3LX0g6K+PyyWX/tO4FWtY95oVBe/VA3VTJuXzcQ0lQCIE7nVGFtjXr+YI7QvFZ90JdAJYFshEyw\nvItE7KV6g0JC2phmnwDA0+IKK9/XWTXVZ5fN+xI6mRvsFfb5+i4P65Rm7dkdjxyJOFW1/6BdoDVy\n+47aiDqY1ZQKP8HPr0CFtUmamst6bo+7zU5+Y1I7rtxCjlJfSlHHX3RQ7xd7jbhOP4DmlNE2yHUo\n9/+epQ7k2vI1Z4GsZ/lxn90IF9V6KCMW6tpSCJ6hNtQy5iY30JBw1d5uhZxKvyx5v06JatTxlbuW\ndlCkwqOf9p1AqxbHvFJLsVvG4VSqijdKNG6cJeeUYlHzHUrnN5Oq3J4nCOov7jtNRD0koHPUoU1u\nJSyQqItZ9W94h2cXETVr3lsujjdtvn+RbfNCn4f18uKKy9o3ni3kvIYBye07aqOsYxWDkMe7Q3x+\nasbbcsfhd5+xg2OGFdMuHlJ1sfWbHEltU81oG+RxLvf/XujCuXGq97UPwTNU0ArBM9RG5p5wlRsg\nSbhZXg7WHVYFHC8ZbFMp08U0qLV0S8hUrur5mEs5ZRyu+o0SLzGYaTK/xabKAG1lf1cQ5durQKV+\n5tRmjgTsRYioSbOf16edRdOre61drU+QAGIex9lGtFps9++aiVYlmNrNL03Copu1WUsp2b583knW\nMi/VOL883nDYKJdHJxCyAzvL+2PmnbrcZwdtQdAp463bOMoZu3UiRJQmKnykQFOH9PGQ3V/opt4v\n9lJ0T9Roc/hJby68qhxjLFl/o0/bx4zaTiUAi+lmceGsZXKdEst3FfrOUY6SlCQQaJiG63rtQ4UK\nSiF4hqqH3BO26u8nw6LpJ76xUnEwc4JaFYSy2equZbUSO9Wj7DIOB+XCW7G7caXJbdT90+QMPHL7\nISoFKhWgkpo2e0hYSNupgm9r0ma1tTy3FPuy+zzio68kWeB4rYFoldeS1a2Z2zpyFUjEi06QGTda\nrYzDUh7Ht1FJV5wgyQ7sLLGSf9hr/IGm99vHQaoZb9X+dONwgyxP9TUXcpT+v1gGZuUYFNYKNPj4\nILX/ebvF0tn35b6yj4VjjKWmPz+WzyAsppsly2gtrYMllu8q9M3bnKTJQNoMFareFYLnJtBmjsGs\nVOXWY6zUPcQrmFUrlnHTKsCT1e4YBAXNft2NS84pL1ZRJ+uWun8ReH7aTXS/LlGWU38FMl1Wo0S0\nWOy7Eoum3dMtdjPoPtX++LHSQaJXcLQ7Nl6OayUqji9/Wz64Pvy6bTtIgpAuY6sd2FliJQ9OGH+g\nfPvpb09rodEOynTvu0GWE+DpyqGk/lOKCm+WQi1PHITHQJ17Ox2tkeVaHXVjSu5N+or/5Gt88LmD\nnvv2q3qoTVlLy6x6rlej71pbmss5hqGrbaigFYLnJlA9x2BWW+W6hNbyy3IzWA+rpRLO3ICTNeiE\nTka7yj/pss6pNOmBSaci8By+gSgPokMgyjuV98iRcElNkojt5HU7p5S+y4VI3fZbfe7j9PRi/eSu\nu3eQOyR6BUe+Pj1UuxIqxfHlD+aDazNN3s8zL83ZAJ4d2OliJdXty3HhVVWO+7EO7nQxm3x8qS8K\nC27HX3TQxLMTNPq0CaKpL6U8W4LtxiLnqcaPJvcmjeRFXtvla1zN/3v1EItYS8useq5Xo+9aW5rL\nOYYheIYKWl7As0FsVz01NDRQtfvYzJqYAA4fBkZGgLk5IJHY6BHVTqurwMwMsGePOe+ZGWBpCWhp\nAWZn62M96nFMQclpbpkMsLAgfp+aAvadq/3JWjKGfd72051blnaRwQJEw1OYwj54bJhrAsBhACMA\n5gB4WI5XOoG7CuL3q11A4+niB1PF/ZcAtAA4C+CYTSMdAKIATtt8Xom6fLTbDOCC5v1WAE0ACprP\nOgH80qa9LIA8gHMAGgH8VnEsLQBm4Wl9Dclj01ZsDxBrXMZh3nCVcZ7pNPP8DJZWl/Cjwo9wav0U\nRrpHMHf/HBJx5wZX11cxc3QGe+7ZY9lWttcSbcEluoQjrx+xtDmDGSxhCa888woKK+JkmLp1Comm\nhLHf7L2zRpt2/fD+Dr52EL9c/yVaIi24ePUi1q6s4SquGtsMdw3j9fOv4+TaSWMsj77wKJ786ZMo\nXCxgqHMIT9/3NB554RGjn4lnJ3D4xGGjjalbp7BvzDxR5Odu65V5JoOFFfGd0hPvAYFwav0U2qJt\naIm24MXffhEDWwd8t+tX/Ljw9XXSBCZwGIcxghHMYQ6Jck+yUBum8BiGqgc1NDSAiBoctwnBc2Pl\ndoF8valc0Kim6nFMQclpbiU3RVD7kzWQGzMzMIGuCDCB/JNeLba9B95gYAa4+gTQuApcvhOI3gDg\nCARQvBfAAQBnitumAKwo+0cBXGavGwA4fbW2A0gC6AbwsofxyT6uuLQLCDD8VQAv2HweA3CpuN1V\nm224hgF8G2KsV4rv8flxaHwPxNqsARiCgFkVTOWxKcBc4wqgbUPl9zyzEQej/tZ+/PCjP6wIdnh7\nkwOTaIo0Yc89e/DoC49iaXUJr0RfQeHeAvAtACeAtmgbPnDDB3Dh0gUcOynuqkjI8wJLN375Rqxc\nUP8ohKINUdzUehP6W/rRHG1GW6wNezN78egLj2LfT/bhzCXxh5UdyOKp+56y7Lu6vorb992OlQsr\nWgjkQPzoC4/i4GsHsX5lHTt7duKJsSeMbbc9vg0nzp9ABBFcKZ7ETY1NuHj1omWuunaDgk7AelzU\nPu20ilXMYAZ7sCcElk2q8BiGqgd5Ac9orQYTSq9E4toCmUrV0iJ+jowIvtFpfn4emUymrsa0WeU0\nt9lZlTNrf7KWjsGjOGxy6+EMgH3ALGYt/6TLOqcS8GdBWxLQCQDRdwHYC+B9AOIADsKEziSA7wF4\nP4CTEBBHsEKnnbWR63xxv9d8jPGy+yYABEy+aPNZFAI6ATEXaUHdCuBtzfYpCOhMQIDqFQjo7IGY\nfweACIAMxPH8BcQxBUzwLR5XQ/LY6KBNcyMiaAX6HeXzPLODuJao+GMPysLG2/tC5gtGe0urSwb4\n4CjQ2dyJsw1nce7yORx5/QhSW1LGfnvu2VOyz7bHtyHSEEGsMYaXHnzJsBKuX1m39M8BrzXailRz\nygK0ibiwrEroTDYlsTezt2QeiXgCP/4ffoyZozNojjQj+1wWLdEW9DT34LW3X7Os49LqkgG/R14/\ngtSXU2iJtmBn907c1HwTTpw/YYxppHsEiaYEjrxxxDJXqUdfeBQn3zmJh771kCfLpO6c0h1rflzU\nPu2UQKI8r49QdaNyjmGtr6VChQLEv/lQoepGs7PC8lZPbsf1OKag5DQ3eVNkQ+Y8MwNkMkg8NIF9\ne1b9j2EJwAKEi+JPiu+NQAAIzH/SNbkzPAMBTT9i49gL4FEIt9NjMN1SkwB+AGAAwKsQlr5mlAKh\nA3Q+jxk8gwyevTKB9bdXncfW5XkWpeJWUf6fhI/1fRAutJMAfojSW52tAO5gr18CsAXAcQDbi++d\ngbCayeO5pvTJjqtFM8W+zynv83NjRrOf2szzM8g8k8HEsxNYXXdZzzqQhLjDJw5j5qg5wdl7ZzF1\n61Rgbp127UnwaY22one9F9vPbsdlEidFsimJ7/3290r247DUiEacuXQGp9ZPYcfXdhhrvrN7JwCg\nPdaOba3bMNQ1ZPR55tIZHD993GhDAtdP3hZ//A1owK1tt+Khbz2kPYaJeAL7xvbhmye+aazdV//x\nq8bvt++7Havrq8Y4AWHBXb+6jsLFAo68cQSvnRN3eIY6h5AdyGLu/jk8sesJ2zW3O06A93NO10bQ\nxzlUqFChglToahuqItVr/ONGjKte12JTqlL/Zh4X9ySAR1Cxq6Krpczu8wxQDCcF+iEALKG83wRh\nERwG8ITSdg+AU96H+QwyWCk2fCumMOZ0F1x139VJ59LLXWgjxedFzb6TAJ4u/i6tkmcg5neO9Z0C\n8GNY582PYQKmy+zNAJ6CWK8tEJbXAZQqA3N9uauuz5jJclwXN1J+YwdlLGYLWjCL2YpvxqyuryL1\n5RTWrwoLZe+WXpxcO4lkUxL39d+HX7zzC8f4zu1/uR2n1s0TXq453yb7XNa0qgJoibTgu9nv4t/9\n4N/h+KnjOHnhJNaurCHWGMO5y9Y7D3ZxpjPPz2Dvq3sNSFY1desUmiPNOPTaIUQaI7g9eTsWfiHG\noIsddZN6nKSLcku0BWcvncWxN63uyDpJ996OWAcWP7poiSENFSpUqFordLUNVXUtLZl8MDNTP27D\nGzGuel2LcrThEO3Vv9luoLOwulgGcSykpQwode10+lwaSVTQke8nAdwG4TZ6RNP2SwA+DOAd2Cfm\nkeoComdbgEtAN0Zwj9YUyOTFtTai2Y7HbV4BZj4+g6XeJbRcbMHsn88icSEhrJl/yrbj7sRnlTZW\nYM5bAnwMwmIpvSPl8cxCgPDZ4vN3YcItl1zfNgiL8irE2qvnhovKcV3cSM3eO+srdnAJS0airRnM\n+HbXs3P3XL8owPPq1avIDmSxN7PXAowzR2e0APjSgy9hx9d2YP3qumXNpVUSsFoyCYR4JI6Opg7s\nG9uHxN6E4V4r4TfaEMVluoxGNOKZ5WfQ1NCEt68Iv+/eL/Ui3ZfGhcsXLNB5V+ddWHlnBSfXTqIt\n2obCWgFvXHkDpy8K3/Erp6+gd0svRnpG8PhvPI5HX3gUR39xFLd+9daS+M+SNcMMzt57FqmjKTx5\nz5OGG69cm+ZIMwCgI9aBP7n7T2zXfqB1ACfOn8CZS2fwyAuP1P1NkVChQoUKXW1DVaSNiH+cn593\n3WYjxnUtxYJKiD58WLBdzeXVv9luoBI2PQKzl3PKApC641v8/MdtwFRBJA4DIEBnClbonIGAphSE\na21n8X0OSVL3QcRGcuiM2YzxNHBvxyxu7ZzC/ZhDPAhXYg9wunTDEhZ2LODwnYcx84nicTgPYWmW\n4uNXEw51AXgDwhr5dxAAfwRinglYj2eLsq/dvdVZiGRF52ACPeD73CjHddHT+RSAnp+ZwTOZDJ6d\nmMB68YSTgOZlrDPPz+CVZ14BngWG1oewx+1GhUY6d0/pFgsApy6eQiwS08Yf6vb93A8+h46mDsQa\nYmiNtRrtvOdr70FibwI9X+zBDVtuAABQ0RRfuFjAh5/5MAAg1mj944g1xvDygy8j2hDFVVzF+tV1\nAzoBGBl5v3/q+wBE7Oium3Zh4bcW8OrHXkVPvEfEp75xBD85K4C3LdqG0xdP4+TaSbTGWi3xn4WL\nBRx5/Qhmjs7YuswuYQnH4sew0rSCkedGMPHsBGKRmLE2dyXvAgADKAH9OdXe1G7s0xxp3lQu4aE2\nXrX6ngoViisEz1AVqZ7iH4thgZiYAP7sz2o/rmqsBZ/Tag2vJTYcohMJzCT2IZNNOM+9lgPVAaTy\neb4H+OA54MkjjIN1oLMEEdu5AgFnKiTdBgFhq8VtzsCqXcWx3Fd8zeArfiqBsXP7vEGnCm3N7ruU\nKAa0bCkCxekR7PlK8Ti0ohSipSKa947BNibXolkAvcXfh2FaRFUlIDLvOrVlJxmXOwEkLngHuVpr\ndWkJKwsLOHH4MI6WcYdoaXVJlDo5Adxy9Jay3GxVmJx5fgYXrlxAU0OT5X1AQPzg1kHEI3E89K2H\nDEhUEw2dXDuJS3QJC79YMIB05Z0VI/bzbwt/CwCINIgTqSXSgr/+yF8DAF568CXEG+MARFbZ93W+\nD5954TPoaOqwnUNXvAvRBuEAdgVX8N03v4tbZm9B6sspIyvtUOcQvpcV8akS+BrRiJMXTmJ1fdWw\nwgJAW6wNf3L3n1jAevtfbjegsKV496RttQ2nVk7h8InDaI22Gjc4Ord0lqyLTvymyGtvv2YbMxoq\nVKhQ9aIwxjPUNaNrsezJRs2pHsr8eJp7PQyUqaT8y6PQx32qcYYfgACuyxDAdr64XQrCUsjjJ+8A\ncBSlcaKq+iGAVZdJlisGkTm2ubitHeQPF+dxDNa4zwZgdcsqZj4xgz3P7UHijoRwG5bZbFMAfhPA\n4zBLpXQW57gOsQZvFJ/txbn9Ozi7wrqVGOHuum0QcOrn9MhAHx9aZ3p2YgInDh9G98gI7p+bQ9zn\n30AQtSTVsiBOZVtmnp8pKW8Si8QsbsG8ruZQ5xDyv5XHoy88asRfNjc2Y3zbOI6uHMVtidvws7d/\nhu9MfscS3yjH9Ma5N4xMtxPbJnDk9SMGSBprsG0CL731Ek6unQQg3FuJCGcvn7Vs1xXvwvt73o/Z\ne2fxwDceMGIwARGHyfuS73135bs48c4JNKLRqDea2pLC9z72PTwSfwTHnj2GN068gY7uDizev4iB\n+IB2Tb1IdyzLqekZKlSoUOXKS4xnaPEMdc1ow610VdBGzWlDM9oW5WnuNRqoV8tzidVbzaAqLWmX\nIBLixAE8BJHBVrq08uviFQhw4vp7CBhahel2OgTTXRcQcaM/hD7hj6pLEMmLfg576ERxLj+E+K/B\n/3NQ0Sr43/Yh8U/F2M73K3P4GkzoBARM//PiPN4LM/PsWQjodHOFdXOXlevO3XX9yM2tuk507+ws\nbp2aKgs6Z56fwdmLZ5HaksKTu54sG0pU115uAVVrherKm6jW5J7mHnTFu9C7pRdP3/e04cYq4y9/\n/aZfx+n103hr/S0ce/MYzl48i1958lfQ80VR/gQwS5W8euZVAMI19uLVi/i1G3/NMvYHtj2A85fO\n45frph/4aGoUTZGmknmeXj+NwycO47a/vA2vrr5qvD/cNYw99+wxrKD8PQnDV5lv+craCn59/6/j\n5DMncf7qeWAAOHP/GTwSFy61M8/PIPtcFucuqumYnaVzCXfKnBsqVKhQG6EQPENtOtnFJdST229Q\nuhbn5FW1nLtbrIvXmNcSDlYB5iBMIHodpnspF0FkuZX7vU/5/HJx/9sB/BkEvOVhlhkBgGcgYKsd\nztK5vOqULG4rkwJdcdhuD4R1N1V8bwQi+yxXG4TFcw9EnVFpXIoCsMulwtxfHQEZqBwc3dyqXVSr\n2Kl4IoGxfft8QycgoOTYyWNYWVvBB576gOe4QKdSHyrMPvrCo5ZtJZQmm5L4wb/4QQnsvudr78Hj\n//A4Tq+L+EkZ3yj3645348z6GfyoIGoTjXSPYO3ymuGC+6EDHwIAfOUfvoKFlQWcWj+FKKJGDdFX\nTr2CRJPZ5wsnX8DCyoIBta2RVly8ehHfeuBb2rk3ohFvrb+FU+un0NTQhIltE/j2A99GIp7A7L2z\nmByYRHYga7zXHms32o01xIw2fn7+51hYWcCZN84AJ4Gt2Io/KZ74drDodk7pYnvLSYy12coHhSpf\nYYxnqI1QCJ6hrhnVg5UuaF2Lc/Kqepp72ZbnWQCDMC2bvP6mtHB2wATEhuL7FyHgKV58fwJmXKPU\nCoB3AXgOouYl/zZPQ2TCLcBZV2CfnEeqCSLm1KF2qDH2H8BMBvRjmPAmYy3vhEgkJGNZ+wAssjYu\nw5qQCDCB80l4r79ZITj6TUBUkfwAdYDiNSlX1lYMyJHgse0r2/DhAx8uTYyjAaOZ52dw45dvxF/8\n/V8YMPvIC49Ytn3fX70PZy+dRXOkGdGGKIb3D2PX13dZ2l55ZwVXinc1Yo0xS2zo1K1T2NGxA8dO\nHsOp9VPob+3H3P1zlvM30hDBjV++EReumCfrFXaX5OS6KLMCCJfa9ybfC0CAIQCcv3IeR14/gt9/\n8feNGFUubrm8SBfx49UfI/tcFhPPTgAAnr7vaTx131MG/M3eO4vueDfOXzmPS3TJaMNSsuUC8PbX\n3sbvrv+u5bi4waIXQCwnMVZoJQ0VKlQ1FYJnqE2nTCaz0UMI5UMblSDJj9zOKd/WVwkTD0HAlbRs\nxjXbnoGAxH4A0hNwBCKm8hgEoLVCuON2KvtegbAWnoIZFwoIq+QxuGekbURpjU6md1reAW0hEbN5\nyaWt4zDrac5AWGSPQABgd/F5A8S8pC7AClvDELGmGZggJt1mJUR7sWJK+M2i5kAH+PyOUt2xayBp\nmWxqLE0AJMHjxDsncOzNYyUAwsGoOdJsAOfKBRMak01J7Llnj2XbvpY+HHvzGC5cuYC31t8S2V/f\nOILbv3a7AU4y2VAEEbz02y8ZcYrS9bQ5JrJfNaIRFy5fwL8++q/REhF9vLfjvbh49SJWLqxY5krK\nCX71qoDHM5fO4Kdv/xTRhijOXzlv2eb7p76P4e5hy3vSeik11DmEvpY+R0hLxBP41Z5fLXm/OdKM\nni095htrQMNRQdCz985isG0Q8UaRgEmujTynJHA++dMnXQHRT4Zjqc1WPihU+QqvpUJthELwDBUq\nVFW14aVZApCr9VW1WnGY4Flaf0Oz7wgElL0LZu3KOZhWUAlaCQB3F98zKjAXL6ob1oBDZ0yw9epC\nKw04sr1GGBaky42XEb0YRcPZhtI22TUzUBzrv4EJeEsQFtkCBHwegYDjIxButnblYG6GcL3lICYN\nc8MAJuHdirkBQFeWNiCeVLrZXrx60bAcqjGajcXLg6HOIQuA8My0B187aAFOAEg0JQw3Wm5xk+Cm\nAtzK2gre/dV3Y+LZCXzrgW+hv7UfP/n4T3BX111GMiIJWPNvzAMQVsPT66fxV8t/ZSQB2p7YjsK6\ns4k/2hDFFTLH+sb5N6zWx6Le1/0+dMbFXZ7hrmFMDkzilY++gp64eeL/ePXHeOHkC8Y2KqRJQLx0\n9RJ6twh3hc6mTsQaYnh/7/vxN7/9N8b7w93D2HuPSM+ciCdwc9vNOHayFPoB88ZA4aKYa9CAWI6V\nNFSoUKG8KgTPUJtOYVzC5tJmSPpU8TnFIWc7gB8V3x8B8D2Ybp+/YPu0wwQpCVs8GY7OXbQHAlJH\nUYTFIhTSL4G9OQFugD4GU01SJNUB4OViX6cB/BK4HL2M6NUomi4X3Q2TMGEzBtFPb/F9QFhdea1M\nXmtzqPiU67EXwhUYEBl6e9lnX0ApiMl1+DaAp+Hd/XUDEwT5Op8qdQv2INUtk1u1fqPvNwx30dX1\nVcM9VLqV3rL1FguAJOIJ3Nx6M469ecyAH0AA5cS2Cfzs4z8zkupIi9ujLzyKl0+9jFhDDHcm78S2\n1m2W8Z2+KBL33HfoPvzwoz/EwNaBkgy4ACyQ2BJpMeCwPdaOP/3QnxrWTztdpssGJDei0QLMHTFR\nbiWCCL6z8h384NQPEG+M42dnf4bzl8+jo6kD8YjpsrB+dd0Yz9LqEm6ZvQVb/vsWfOCpD2Db49vw\ntX/8GhZWFnDkjSP44A0fxNStU7g9ebtRJmb7X27H7cnbRUzo/d92jc2U55T8TAJxJYCoc9ctx0oa\nanMqvJYKtREKwTNUqFBV1XWRIEle77ZBWPZOQbjOzkG4n15tJGsAACAASURBVMp4QbldEsI6+gKA\nWyGyxQJWSNLFGX6z2PYCGFyeA+4olpQ5aTO+LRAlWwABmlH2WSuAu4p9PQogC0QuC/MmgYQl96cQ\nFllAuNy+DNEXNzANsbHPinYwUGwfMC25CQBPQMDWUQgXYg5eKoiVG29ZA6ALRDWIJ1Xj9pzqP3L3\n0JHuEezNlBZK5fAzsW0C2YEsXnvoNRwaP2QBFu4WKmtzHjt5DG9eeFM7zpW1Fdy+T7je8gy4XM2R\nZqSaU/jIwEcMt9qzl84i93zOErfqpq54l+W17OsKrmD10ipW1lawfnUdq5dWceT1I+j5Uk+JGy8g\ngDXSEMGZS2ewfnUdL7z1Ak6cP2FYYiOI4Pyl89hzzx68du41Yz8JoAAMl2IJgX92z59Z6p3yGE55\n7L79wLeNrL/lKoznDBUqVK0V1vEMFSpUIJqZEW61LS0CNq8pyJR1IdV6nFKyrqR0LZX1OdXttgNY\nhrB2bocAT6l+CAB1WrdOWGEvfhW491PA4/9eLHgPBJhyNUMk+umAWftye3G7SHGsncV53Q7hIsvV\nCGFl/SVEzOhWiHqfjTBddbdAWHPVsWdgrYcpS8z8BAJKXyv+bId+XUMFIqd6nbrP3OpIrq6v4n37\n34e+lj60N7Xb1ojkNT25Yg0xI9mOTk0NTfhg6oMGmAFAAxoQa4zhgW0P4PT6abzyy1csFtcGNKCj\nqQOrF70F9EYQwVVcLYkB9aNIQwTff/D7uPfr9+LUuvmHF22I4jJdNn4C+lqfANC7pRevfuxVZJ/L\nGms1desUTr5z0ngdb4yjJdqCnd078cSuJyx1USup0xlEHddQoUKFkvJSxzMEz1ChQgWiTEbEcgLC\nwrlv34YOJ1hlYAUou7lJAJUxmaoSEMmEABGPuV78fQiiJIrbdd8uCLCVygJ4qvj7DIBXIJL8HAHw\nH4q/fwfA5wB8BSKZTxTC4roKAdJyPFPF/dwy4U5AWDG/DeGaG4EA18sAdkJYMxNs28MwQTwLcx1V\nOa1rHer5mRmsLi0h2tKCe2dnyyppUitpQbJ4p2i1I4aZ/6UNe+7d6ws8bvzyjYYFMDuQxVP3PVWy\njQSboc4hvP7O63hr7S0Mdw2jPdaOhZUFNKDBFvzijXG0xdpwev205X0Oc3aKNkSRfyCPe79+Ly6S\nl2K25Wvq1im8/NbL+OnbPy2ZT7IpicLFAhrRiGQ8iVhjTGs1HWwbxOrFVRQuFtAWbcMHej8ANABH\nXj+Ctmgbzl0+Z+lv35j4Q+Fgz9/3KrcbDKH0msEMlrCEFrRgFrNIhHfMQoUC4A08Q1fbUJtOYVxC\nfWozxHLayfWc8hov6OY2KZPqtAB4EQLEbv4B0PoA8JBN2l+euOi/w6yJuRXAf2bbLUFYUNcB/D8A\nDkHUCh0ofiYrTFwG8BaEy6yETjmvncXXt8NMVMRzwQxBWD9PQsRn9kFA51swrb3cY091d5Xr2K78\n9BKHuUFlR+y0urSElYUFnDh8GEeVrFn19h2ljdsrZv1KPHME+74Q04KHU8mO9Svrxu928CjdQvO/\nlcfSx5YsLqLd8W7LfhElI9b61XX8cv2Xlvca0egKnYCI5dz17C7c2XknGlzrBZWvtmgbCmsF/NO5\nfwJQug7SIiuTIZ1eMyFajqs10orCesHY9tzlczjyhqg5mh3I4gM3FH3kXxWJnpojzcYxiUXEF0q5\nCYbCeM7ytIQlLGABh3EYM3WducxZ9fY9Fer6UAieoUKFCkTXdCxnUPGCL0G41P4dRFzlUwAungCO\nfRo4/Cngd/730n144qJHYMLh28XXEsh4QiP1GtQu9K0NAuTU2MvvAPgYTPiMQ9TzXAbwTHE8CxCu\nttxjskPpWwVxuY6vABh8Cbj9Y0DqeeDJM+7rapeldoOANFq809I9MoJ7NtudFsDTnSK1DieH0J3d\n4kRsjbTi/OXzFjCVwPrQtx4yrGmJeAKJpgSyz2Xx0LcewlC3yDiVaEpgYtsEIg1W8GyLtpWAXKOP\nS5a1K2t4+fTLIBCaI83oineVwK1Ue6wdu2/d7bntCCKINkQNSOSZcrk6Yh1GPdBGNOL9ve8HYJ3b\n+SvnsXpp1RiH1Mn1k4hFYnhi7AlkB7IYvWEU+d/K45snvmkck6bGJl8ZaL3U/gzlrpbiF+oIRrCn\n1pnLQoXa5ApdbUOFClXfulaDR2Xc6LGzwOXiBWf2ItDTZI2DfBUiHlO6q94J4ASEtfAVAJ+E6b7a\nBFELlLu7AgLI/kcA34ewiHJ3WuniqsaxOrnFcm0BsAbhcvt9CKB2mq/Rfsafb7bqtivnl4E3N+iA\ntb66iqMzM7hnz566crP1HPe3uir+tvbssfxN8f0v0SUcef0IRrpHEI/EcexNEZ+Yak7he9nvYeSv\nRozYxsG2QVy4cgHr/z97bx8U13nne377HZoGGmhkhJBakkvWSyIZJBzJsRS1IyleEyd0XshcM3cs\nu2rdU8luJffurrh3tu7O3Jqb3Joqp27NTO2uK9pkxEzingQpkWLZZhRhCSThGFu2XhxJMQ6KiRBC\nvIgWIKBpoPePp5/T55w+p885/QIN+n1UlOjz8pznvAD97d/Ldy6MKKJCraVaami9tx5XR64KdaKd\ndzsxFmENeRxmB3Y9sktS4+k0OzE5P8nWmxwIR+MRVy2SpeceWHUALftbJDWWRpBfG47NZMN2z3a8\nN/SeIDJXOVcJ12ckPIIiWxHGImMoc5QhGo1ia9lW3Bi9gcHpQaHusqmrSXI/1//reiE66vf6UZ5f\nLlkv3158/9NNzSUYIYQQQACHcZjSbAlCBNV4EgSx9FmuxaM+SIVdzSxwxqos+KrAPEB7AVxBPLLX\nAGACTJC5Yt/z5fLLxIXfNcQbENWA1Wq6AaxEvLGQH0ygtsZe84ZCcmrBBCdvkpRM+InPtwHARB3Q\n6gdcO4Bd24CjtuRRT7X6WTVB+pCSrriQi0O7xY7Dew6j8e1GtPa1Cts1rG/AxMwEWvtaYYYZZlNi\nGqxcPF0bvYbh8DBcVheK7EWYmJkQur+uyFuBwel4W+byvHJE5iJCNJCLx+rSajgsDnQNdUGJUnsp\n7s3cU1wnF6Al9hLcfP4m3A43Vr+2Gn0P+gxdKwB43P04yp3luDBwAdPz03BZXfjCyi/g/sz9BDFq\nN9sxM89qTqsKqnDhqxdwqOuQpOmQ+Jq7HW7J/eDpySPhEaEWdGp2SthX3pTI4/DgifInBAFKzYQI\ngsgmVONJLEuoLkGFQICJtDqVWsGlygIUjy7YMyVOC+X1ntVgQu+MlYmmnthy/qu7CMAFxL0++a2V\n+1zuki2Xw1NVh8FqM/2Ii04g3ugIAKKIRz3rwbrt1iPuucnnfBqsxlN8XLXU1wR/ziDgeQqY2AG0\n2aBZKqVWP7tQtikGUnoX/HeUaG5CGqBK3Z9WuqXYQ7LZ1yzUAAb3BVGRXyEZO7gvCKvJinnMSwTd\nttJtEo9JnrI7HB6G3WzHxOwE+if7BdFZbCvGFyu/KJnH0PQQ8iysoNlldQnj90/249bELdVLoSQ6\nnWYnVuStSPD5HJ0ZxSP/8gieb3seY+Ex1TGTcSV0BW39bZienwbAajQvDl1EvjU/YdsnVzwJgF0/\n7lfasr8FRfYiYbn4mgPx+5H3hzwMh4cxEh4RrmFbfxt6xnqEfQ/vOSxs77K6MBwelliliG10SHQS\n9F6KWAxIeBLEciHWLAStrUyELheUikeXqsgW1ym6AKwD6xDLRV8AAH//yxNFxsBqOfl75howESj3\nueT1mWoCTCz8roHVl4q347Wj1QCaY+uOAzgBlvJ7AnHPzbOi/eXCT3yOm8FSgX1gtaCSebuBJz4T\nn5PaZwpagq8JrNlRo8p6AyQVZWo1ppkYO11Ecwv+PLm40PJuVBMnbocbN751Q7KO120CgNPixIFV\nB+D3+tHxlQ6Jx6RYzH5h5RcSjllkL0LrrVbJspqyGvgqfXCYHZianRKWD04PYnpuWtdlMcMME0yY\nnJ/E4PSgYofbmegMfnHzF4IIlrPKuUry2mlOLJiWNzAanB5Ed6g7YS4wAWsK1sBhZv6cB88ehPMn\nTvym7zewmWz4yd6foKmrSfKc8PuxpWQLgNg1rPiC8P27X3tXck/49rwpkfgDCLVmQlT7SRDEQkGp\ntgSxXKirY6KztnaZdvgRsVTTb3laqAssQjkFgGfjNYAJKJ6OagXrQMtTSL8X2/dxJNZwqiGuq3wV\nTMCqWb1oWcHohZ8jx4N4aq88FVfPMX1IXsOptd4ASdNU00zpzWp9nYG58XRLj8ODje6NKLIp+3Dq\nrRXtHe/F7td348JXL8Bb6FUcw2axocBagGJ7MXrGevDe4HuCj6fVZMUOzw5J6mx5Xjne/9r72HVi\nl6L9iBVWzEK7u61RtKxaHGYHyvLK0D/Zn3ScbaXbUGgtTPDs5GOE59knTfLzqHRWYkPRBsXnRGx9\nAkDTBsWIVQrVfhIEkQn0pNpaF2oyBEFkmWBQsVnIsmSpercEATwGZj/SBqAitpxH/Bpjr0sAtAP4\nPuKirFe0XwDKDYHkt51HwgAmOpO9n3RrrBeT7LhBsEjnQOy83LE5K0U19RxTy8pGr9WNDsSRuYQ0\n1SDSEuZJx04XA3ML7gsicD6A/gf9Qg1i4HxAEBvcn/S3X7qK33lGE9bL8RZ6cevP46mvYsE6FhkT\njtGwvgG9470JdY+z0Vl8OPwhAMBismAuOoeh6SEc6joksWyR7JNh0WmGOSFdWIlCWyEGJhOFsJx1\nheswM5cYXXVZXZLorfw8qsuqcXXkKgDW4faVna8I65q6mjA4OYjGtxvRebcTDyIPcOKPJ3Dx6xex\nrSyxoxePbuohlWdTdxMrgiAIEZRqSyw5qC5BBbebRf6Wu+gEMu7dsmDPlBtMICH2/7uQpqnytNWb\nYN1hxTWNSgJLK/1TTZSla0GS7LhuADcQPy+tFGAttGo4y2NfGXjsk9bAqdWYKqD0PKVaXxc4dw6+\n119H3VtvIRRW6eRqYG5ckHDrDrnY4P6k0Tujiuu1ONl7Ukjl/eT+J5IxuMApc5RJ9olEI6gqqMLT\nK5+WbM8tW7LNPOZ1bTccHtbc1mayodnXjOC+IErtpZJ1E7MTmEPceoVbrQBAHvIwMz+DVQUstXcs\nMoZDXYeE9e+df0+4rmORMcxhDpFoBDtP7ExIlRW/Pnj2oGYabSrPplbKNpFZAgjABx/qUIdQhnyj\n6L0UsRhQxJMgiKUHF9nZJADgJFj95Q7oT2/VQh6dEp9GsgigUlSLC0sPgH4wISmOQKpFwsSRUB49\n1WITmCCeRbzxUQ2Uo4zy80jnVmlFRZUiwRyDVjy6okRaUeZ0xlagOxRCxwCLsgXOn0fL/v2Gx1CC\nRz7lqZjcn/Q/fViNE8+uxU/2HTEklMVRytryWhTYCnB4z2E0dTVhLDKGivwKeF1ejAyNCNtZYMFE\nZAKRaAT13no0+5rhdrhRWVAJj8ODueicYCHisrowMTuRcFwxWimz2cICCy5+/aLQxddsSvxs3wIL\niu3FsJqsmJqbwswsi4xOYxptt9sSGjhxHBaHsPzKyBVEohGYYEKXvwvf7fyukCq74ecbJNer3FGO\nofAQAPXIdSrPZlYj+EQC3ehGR+yXdgABtCyUbxRBZBiKeBJLDp/Pt9hTIBaTdKN1Cig+U91g6aKj\niIuaTGAgOqW5H48GbgSrFZVHINWOlUp66gBYg6AoIAR9/gRpUx899ybT9y/ZuWSj4ZaOJkOZ/B3l\ntLLPh2s9Hhzes8fw/mqNY9QazewLBrG+oQHfevMsfll33HAK5Y5yFqWsKavBa198DS37W9DU1YSW\nnhZ03u3EwNQArt5j6aRWkxXFtmLMYQ6hmZDg28mP2Tvei+HwMEZnRuEwO7C6YDXyLdJusfJmP2ZI\nbV1K7aUosZdItrEqfOZugSWhSVDCNiYLDqw6gJ3lOxPWmWBCz/M92Fa2TdLFVz7mPOZxb+YeBsPx\nJknm2FuxWk8t3vW/KzQT8p/yC/dt085N8Dg8cDvcOPPcGeRZ8nD5G5exrWxbQidbLjprPbV4vOxx\n4ftMCkS9UVJqXJQZhG7VqMXhdGsKYtB7KWIxoOZCBEEsLXzIWDOZpIib5Ij9LnMRo41vUmkkVI54\nkyBA2TfUB+17o2cbI8jORVJ79qMI3K+3Zbbh1gL7hobCYQTOn8fhPXvgdjgM778QjWPE1/zVPa/i\nUNch5Fvy0Tvey2o9Z8aERjsl9hI8VvQYuoZZM6GK/ApJA6EyRxk+V/45BPcFE7xDxdE7Tt3qOpzu\nOy00K+LbiCOjJpgQRfx9yFv/01v4Hx/9D7TdbjN8rmL/Uo4JJkEEBs4FcOzmMYzOjMICiyS11hz7\nx2s7rbDCbDLj7efexj9e+0dJ9HnlT1cK16XeW49QOKR6H3kjodHwKNput6G6tBprC9fiiO8Iuz86\nmwxlA2pclBlCCCGAAA7jMNw5+4eIeNghH09iWUJ1CQ85GWwmw1F8poJg9h9+5LboBIx7WaYSdb0I\n5v95AOyaKPmGiu9NPvT5eRpBKVoqOxdJ7dlfujJaCwxA17XO5O8ot8OBlv37UxKdANAzznwei23F\nkmY1mSJwLoCWnhbhmh/qOoSW/S3oHe8VlnGvyRJ7CS594xJK81jtI4/wrchjBrEuqwsj4RG09rVi\n+6+2Y2xmDHazXdiWR+84BZYCXB65LLx22Vxoe65NYicCQCI6ASbEju4/mlBrqoXL6sJoeBSv7nkV\nDesbsKN0hzD+9y99HwB7/njEUZ5qO495mEzx92SzmMVMdAY/vPrDhOizOGX5VN8pXOy8CIBFkuWR\nSx69Prr/KBrWN+DsV87i+DPHBcsbpcj2QkEpuZnBDTda0JJR0UnvpYjFgGo8CYJYWqTZXVQ3bjDv\nyoXEaP1gKvWGKdYowgvgtui1UtRUfG/8UK4jTef+6ahNlbzR3XcEqMvwQ5Ks5pRf2ykAp5CR55N3\nmbU6ndgXDMJhUEB7C7zoe9CH+5H7gijUQq1jqdLy7lA37kfuA2DpqqPTowiFQ4LYLLIV4dSXT+H7\nl74vRN3k9aUf/9nH2P7L7bgXvgcAqC6tRoGtQOiAazfbcX30OiwmC2wmG56qeApXR67i3sw9PJh8\nIMx7IjKBr576KsJzYdybvqd6fo8WPYrvvfM9zEf1NRUqsBQgYolgYmYCbbfbsPZf16Iiv0LoUFtT\nVoPLw5fhPuLG5OwkAPb89T3ow8DUgBBxLbIVYWvp1oTOvkopvjvKdwgR2em5afAGuLcf3E7YlpNq\nHXE2UaslJgji4YRSbQmCIFIlVRGnhg/G0lCNbp/qPqmQjZRUHWMa8S/MOD6kdW2VhN3rPh8GYp61\n6xsasN9gUy3u21nrqcWWki1C+msyCwy19Eil5Xx8cUOfhvUNEruWdYXrsKZgTdLjisf2e/0Iz4XR\n2teq2EyoqqAKW0u2orWvFcW2YkH4AqxT7Ew00cpEjlLabqrUe+vR3t8umcfeir2YnpuW+JMC7Nyi\niOKdu+9gaHoIDrMDDosD4bkwqsuqUeooRXBfEACw+RebMTA9AJvJJqQSA5SyShBEbkKptgRBENlE\nR6MZQxhNQ00lbTULqcoJBACMgfmUHkPmItM60lwXNbUwzWurZFHBu8x6amuxJwXPWnETGHH6azIL\nDLX0SKXlfHzfSp9kndiupdJZqXlc8dhHfEeEcXetYCmzVhNL0HJanNj9yG6hQ+6+yn1Cs6CtJVuF\ncZJRYi9JSNtNlc+6P4tmXzNsZptkecdAh+BPajOxdTazDXce3MHM3Aze/9r7aFjfAIfFgbHIGMLz\nYXQNdQnXyO1w48af3UDD+gZs92yXzJ1SVgmCWKqQ8CSWHFSXQGSalJ+pTIs4o7WaRrfX2idTHWe7\nwbrsDgA4pLGtEZRqU7PQ5ThlYte2/W/aUxLbSsKOd5n98unThtNsAakQ11tvp9axNLgvCJfVhe5Q\nNzb8fAN6x3vj9YUHjkr2EY+h5Bkq9yWUH1M+7gdf/wBVBVW4/q3ruDN5R+iQe37gvNCsZ33RetSu\nYEa5ltg/JbaVbcOP9/5YELNizAbfFl0PXcf6f12Pje6NcJildbjcn3R7GROOkfkIuoa7JLWwvIZV\n3NmWXyN+DW4/uA18zMR3+1fa0/5QhTrNEgC9lyIWB0q1JZYc7e3t1Aac0I8OL8eUn6lUusPmMj4k\npIp+5Qeb8Mf5AeTBhjf/8iIeWeHVHmchO7/6sDCpwwZI9XnKdppwJsZ3H3ELKaVVBVW49ee3Ujqu\nDz50nOsAQkCFtQI39t2QzClZbas4fdhtd6Otvw21nlqc/vJpAMBjP39MkkZbZCvCWGQMFpMFc1HW\nZdbj8OD+zH1JCmuRtQhOm1PSZVeMFVZB5Crh9/rxzt13MDg9CIDVqj6YfYCbYzcl3W1L7CW4+fxN\nNHU14cQfT2A4PIzPrfgc7jy4g9UFq1FkL5KkJO/+9W50nusENmYmzZY6zRIAvZciMo+eVFsSngRB\nLG98PublCLAOpwZr5B4qFATjZ/+bG9ceYULjC3er0PF/aQuNjAvyZLW0C2xv8rBT/s/lGA4Pw2lx\n4vq3rsNb6FVtRpSM1edWo6+nj3nDIlEA8drWn/57YOyzHqza+oQwtljIAol2IVyY1pTVYI1rDfIt\n+Wi52YL5mAGt0+LE5BxrAmSCCSX2EhTYCrDGtQbXRq8hNJMYBXRanHhixRPouNORYM8CANtKt6Hj\nK+z3zOrXVmNqdgpuhxszczMYnx0XtjPBhO2l27HCuQJjkTFJoyFx3an4eoiFtpZvph4yPR5BEARA\nwpMgCAKoqwNaWzPr5bhcURCMtf+tHB88MoxHB53oDFzXF/HMND6oRzWXWNQ53S61mSQVwdg73ovd\nr+/Gha9egLeQPQvyCJrb7lYdlx/zyr0rgsDjEUDxdm/V1aGvtRX/8DcuXK+cEMbWE52TR1jF8xMj\nblwkbo6khAUWFNmL8OQjT6JrsAsj4RGYYRbEbL23HivyV6A71I3Ou53CWEoilSP2MK0pq0GZo0wS\nvXU73AicC+D6vevoGevBu197V7jm6bCoDbgIgli2UHMhYllCdQnLiIWozwsGNb0cl/UzZeQaK9RQ\nvvmXF/GFu1UZFZ2Ga8yS1dKm4kmaZZI9T6Hubgx0dKCvtRXnA5noSJU6Ss2MtPAWenHrz29JBJC8\ndjTZuCd7T6JjoEMiOi9941KCAOK1rVXbWXMh7qEpfl7UniN5gymlhkNOixMWE6sBLbAWCEKx2FYM\nv9ePYluxZPs5zGF0ZhRX710V6k0dlnhN5/DUMF7vfR0dAx3CWE6LE+e+ck6o4xRTXVqNd/3vot5b\nD7/XjzPPnUmokwXYPeoc7MTAlQEc6ooXTBv5GZJvu9jenkRusKz/7hE5C/l4EgSxeIh9GbdfBNb8\nH0lrMVPC7V6c9FodtaULgg7vy2SprI+s8OpLrzUypZgwAViapGYUa6G8WxeAdLvUZhK9zYa0kHs1\nJhs3PBcWvq90VuJawzVFAeRwu9Hyv7rxYLQfNpMNE7PMQ1P8vIifo+2/2o6p2SmE58LY4dmByoJK\nwTrm1T2v4onjT2BomqWxVpdWo8BagM5BluZaYC3Ag9kHggj2Fnpx4M0Dgo+mmP4H/dj4i42oLqvG\nnQd3hOWdg50SP06H2SGkIu+r3IfWvlbJOKMzozh49mBCVLhlf4skEm2zsI64RbYi9E/0o+6tOgT3\nBQ39DOnZNpXoN0EQhFEo1ZYgiMVDXJ/neA7ofJMtXw61mLlSW6qnBtKHBW3Q8zDXmIVDIZwPBLDn\n8GHNNFuxGPhfTpZj7kYvrE4nfvkfy9Ezpe3HqTXmq3texaGuQxlPueSpnPmW/ATfUC7oaspqcOa5\nM0mPK0+RlT8v4ufIYXFI6iXlvqKH9xzGS+0vIYoomn3N2Hp0K/om+1BkK8L5r57H9y99XzLfV/e8\nisd+/pguT1CA2arcenBLaLzk9/px/JnjwvXgnpwAS6t1WpyC8F3nWoc1rrjPqf+UXzjvem897BY7\n+if6he0b1jdgYmZC8WdISUDq+XmjhkMEQaQL1XgSBJHbiOvzGpdZLebq1UBfH1BUBFy9CnhFaarJ\nmuVkGn6N8wH0qhwz0w16NM7vYasxSzWaJBYDT/WW44X/ziJ2/8/feXC1ZBiAMZEQOBdAS0+LII6y\nLTCUxIyRey9vEtTsa5bsIx6r8e1GIapYYCnAg7kHAJTrR4FYp9i7cSHXsr8lYb6v7HwFW45uQWQu\nIul+y9lctBmjkVFYTVZ4XV78/v7vMRIeURTVoXAIL7a/CBNMOOI7Isy31lMLh9mhKSpX/2y1IJSv\nfvMqiu3FitfRyDUXP5eRaARtt9seyg+DCILIDFTjSSxLqC5hGSGuz9NRi5ktsvJMcaE5NgYckplZ\n8vTXVjCRlk34Ne5NcsxU/ECToXF+y73GTP48adVSqtXriVNWv/u7xwGwFN2KzdXCciMpst2hbkF0\nlthLDO2bivejUsptsnsvPwb39txauhWhcAiNbzeq1nIG9wXh9/pR761HsZ3VZybzveTeouLaUfl8\nvYVePOF5QlF0AsDGko248xd38GjRo+gc7MRIeARVBVWKkVy3w40Tz5zA8WeOY9eJXegc6ITdbMdP\n9v4ERXapz6mSj6r7U/b/WGQMh7oOqV5HI9dc/FwWWAsUvVuJ5Qu9lyIWAxKeBEHkBrwWc6lHOjlF\n7M0kamsBeS1fsmY52WIhG/QsxvnlMFq1lGrCVCxA6v/5KNY3NODLp0/jF88kNqExMg+1hj7JSKUR\nkZKAMnIMLph6x3s1j+12uHH8meM48cwJrCtaBwCYjc7i+5e+rzo3j8MjqR1Vmq9SYyKA1Yke8R2R\nbFPrqcVH3/xI81wHJgcwNjuGmfkZfPnfvpxwXCWhqLce18g1F4/Z7GvW/DAolQ8fCIIgxFCqLUEs\ndXKliQ0hJRRi9+bw4cR7YtQCJBP3eCFtR5aYxUm2EOlmbQAAIABJREFU0UovTaXmNZX03XRSnNOp\nyw0ggG50wwknggjCrfJQqB3D6LH1bq9nu1A4hJfaX8KD2Qf46N5H2Fq6FU6rU5L2K76uTV1NmlYy\n79x9B5FoROKFqoXeYxjB6PNAdaAEQSSDajwJ4mEgV5rY5DpLWaDTPRbIJR/MTJGKIFxoEaCnTlBN\nBPngQ0ese1UDGtCi0r1K7RjJmhUZGUc+32w0V0p2X8Tr8ix5+P23fp+SL+diCcCHuSkYQRDaUI0n\nsSyhugQZMXsGxZROIk53NxNvra1MhIrI+WdqOd9jg16uRn0wzwUCeN3nw1t1dQiHFiY90OjzlErN\na6asUPSip05QLQ3WGcu9rkUtDifJvVY7hpGU22TjyOd7qOtQwnbidNKDZw9mpK5Vad2df3/HkOgU\nP1MLfe85RlOnidwm5//uEcsS8vEkiKVOMKie0vmwI45y2pgfnm7xZrTzbDYjqnrvsWwOgSZ37gd5\n9fiMijDqg8mFKgCcDwSwfwGjxdn0RpR7Zy4kSj6TyURQEEEEEMBhHFZNs00WyebHuzZ6TfNYWuit\ntwWAckc5hsKsk7Auv1kkvy+ZumeLde+5oCcIgkgVSrUlCCKz5FJKqzhF1e9n4lOvQPfBmLdlLqTD\nyubgG2xZ9ClpYtDKxYgPJgC8VVeHvtZWeGpr8eXTpxc0NTeXa+LSEcXi8+I+k+mKoNd9PuEDgvUN\nDZIPCMTHqyqo0tXARw0j9bZuuxtt/Zm3GMnmBxIEQRCLhZ5UW4p4EgSRWXhKK8BE6GKqHXGK6pEj\nxkSw0c6suZAOK5uDs1FlSgEAJwGEAewAcBRAExbOW1RMEIYaETncbkNRy33BoCGhyslELelipUTq\nQRzZ0xvN48i7oaoJJyMCK1kkW3w8w42NzgVwsvckwnNh7PDswNEDR5OeqziaCCArkcV0rj1BEMRS\nhiKexJKjvb0dPp9vsadBqFFXx+ooa2sXxZNTQrLOsiIUnymjnVl1HiuryOagOiUf4tFcgEV0B2Es\nwrvMSRaB04I/T+l0kVUjU9GydBrF6D0vIxHfZJHsdK6jeA565pEJ+D3qGe+Bt8CLInsRyveVo9fR\nCyeciLwVQVtfGzwODza6N6LIVqR5L+nvHpFp6JkiMg1FPAmCWHhyqeaUe4OmtC+Mia90jpUpZHNQ\nnZLYmrAaTFzHoqPkvckwWkuqRDZq4lKNlsktTdKpE9R7XkoRX7VIcrJIdjrXUezDWV1avSCRZ/E9\n6nvQBwAoP1+Oof2sXrR+Xz0azjeg/0E/Ou92AqDIJ0EQDwcU8SQIgnjYCAF4CUAUQDOYyF6i3pvZ\nslcxWkuaCfScS6qRSr2WJplEKVKZTiQ51Tm81P4SoogmTQtOF3EkOhKNoO12G2xmGyLzERTbilH9\nzWp0FHagFrU4jdNww032JARBLCvIx5MgiMyRS02DlhpGO+QSulloIZNN9JxLqmmndahDK1olwidb\nJEsHXsxmT3pINZVZqeHSn8b+hK7hLgCAf70ftv02SWffbKRiEwRBLBbk40ksS8h7apFI4oO5JAkE\nWBfYujq0v/FGWvtDyx+SW4a0gonQ5YxBX850yURKbDLEvo56vRyN/o7ix/jbfdcwmZ/8XJJ5VCbz\nLA0iiAY0ZF10Asm9PfcFg1jf0JCTohPQ50uqhLzhUsv+FpTmlQrLjuw5gha0SK69Ef9W+rtHZBp6\npojFgIQnQRD6yIWurZlELKR/+MP09tcS4kY75C4gRvSzLhZYZGdbyKQqRFI5xgePDOP4X1WlfC7c\ns7SvtRXnZc+kG+4E4ZMtknXz5bWciyk6AwjABx/qUIeQ7NORVDsRB/cF0bC+QZIyq7SMIAjiYYaE\nJ7HkoC5si0QwyMwgF7tTbaYQCWnfiRNp7a8pxINgnWJ1+FQuNBkPZC+wyM62kElFiBj9HSU+xq+b\nPkr5XLId/dVLrguubnSjAx1oRSsCsk9HUp17U1cTBicH0fh2oxAZNxLR1IL+7hGZhp4pYjGgGk+C\nIB5O0rU/yaB9SiYa5KRagptx9xuNJkXZagaUjHSOuRB1eJk6hpGGSJmyZVmKZKPe1Yh1DEEQxHKE\najyJZQnVJRC6SZZH6nazL78f7Tt3Gs8z5V4lGRBOqimSBvJgU41cZjyQzW1oVMZKlg6aLdI5ZipR\nK6O/ozIVGTMS/V2IFOJcJRv1rqmm6OqF/u4RmYaeKWIxIOFJEMTyYNM5wH0ZKH8f6L3PlmmpMb7+\nvfcWtWGSaoqkATWZagluBvWzLvSmg6bS1CfdYz5MZFso5TJ6612NPIO5nl5MEASRC1CqLUEQywP3\nZeB+Nfu+6h3g1ue180i11i+QhYxqiqSBPNgMZv5mFb3poJlMXVwMT85ch6w8tKH0WYIgCP2QjydB\nEA8P5e8Dw08AzmvA9SrAW6ytxrTW+3ws4giwfFS3e2G9TJeKmswCdW/VobWvFbWeWooiEYsCPYME\nQRD6oRpPYllCdQmEIhcfY5HOr/7fwMF6Fi0EkueRxvJM2y9fVl7P81c9HqC/Hzh2bGG9TBc6DzaH\nWMqpi/Q7anmQS88gPVNEpqFnilgMUhaeJpOpwWQyXTOZTHMmk2l7JidFEMRDRKaMJL3FLL32zg1t\ncaj3mLzzzsaNQGcnMDrKli8XL9McJp2GO+cCAbzu8+GtujqEM2JOSjyMZNIOhSAIgkgj1dZkMm0C\nMA/gRwD+92g0+qHKdpRqSxBEHHndpN8fT2etqABu3EgvwqenLlKeQtuiUbvFx6ypAdasAZqbtee4\nQPWhRBxum3Lv6lXMxD4kWN/QgP1a93eRWAxrmUUnAKAbzO81iJzztSUIgiBSQ0+qrTXVwaPR6O/5\nQQiCWMIstEDinVr5sXk6KwAMDLBl6QiFYFC7LtJoC1i1MZNdO/l55qj4WYqoCTZum8LJ9S624vme\nDwQUBfKy89vsBsBvUQDMeocgCIJ4KKAaT2LJQXUJKqSaspqqAWSy4x88qD4XuegLBlmkU7wsHfTU\nRQaDwLp1gMMBNDai/Y03Uhsz2bVL1d+E0ETNl5PbppRWV8Pr9+PLp08vShRR7+8oPTYvy85vk3/O\nVAuAfix0Q3/3iExDzxSxGCSNeJpMptMAKhRW/Z/RaPSk3oO8+OKLWLt2LQDA7XajuroaPp8PQPzB\np9f0Wu/ry5cv59R8cuZ1dzfaY9ETXyzCpmv/qSn4AKC2Fu0vvAC0t6d2/JMn0T4wwF6XlQEjI2gH\nAL8fvth27e3twHe+A5/LBRw+LDT18X3pS0BrK9rn54ELF+B77rnsX681a4TrhclJ4LnnjI83NcVe\nx8Rle3s78MMfwjcxAdhsaH/kEWB6Gr7GRiAYjJ9vLjwvS/g1F2wDjz2GtS+8AI71O9/B+OQkDp44\nAYfbveDz+4fnnsNEXx8sDgeimzbhnStXYHE48B9PnVKcj575Tl2fAkqZ3+YL8y+gPdWfz1x5/R3A\n5/IBh4H2yzkwH3pNrx/S15fp7xG9TvP15cuXEYoFFz799FPoIW07FZPJdBZU40kQi48Bz0cJmbLs\nKC2NN99ZsQIYHIzPpakpeTqvz2es5jITpHq9xChdO/G5eDzA8DD7fqHO6yEgV305X/f5hNRZR3k5\nwkNDANKrMyW/TYIgCGIpsJB2KlToSRCLDe/AalREpWrZEQgAK1cywXngALBtG1teXQ289550Llrp\nvIuRlprq9RLDr11TU/xa/O53bF1tLbsW/HtKt9WNVldah9uN/S0tOSU6AWnqbNnjjwvfp1NnSp1V\nCYIgiOVCOl1tvwbgHwF4ANwHcCkajT6rsB1FPImM0i5KNSMWGHEznbExZjHC8fsBm005cqoVXcxU\n1DVF0nqmeOOg+/fjy6qqgI8+iq9fpPNaqogjh7nclVYOj8TOv/AC9u7enZNRWWJpQn/3iExDzxSR\nabLd1fY4gOOp7k8QxBJg0ybg5k0gGgWeegqYnY2LzQpR+XdNDXDkiLq40uo0yyOHegkEgJMngXAY\n2LEDOHpUeVx511mtlF+dh5YM0d0tFZ01NcCZM/Gxl4hoyiUSmu4EkLYFx0JYl/BIbHt7u/D9YqN1\n3g+lpQtBEASxKKRd46l5AIp4EsTSxe2WiiqbDYhEWArppk3Ab34DWK0stdbrTdxfrNLKy4He3tRF\nX7Joq1r9pLx2dHBQu5ZUw14moRx1IhbNdbuBz38eeO21hYtuZkCQsXGk53yuqWlRxUhCDacPcQuO\nBqRkwZHrUdRkAvBcIIDekycxFw7Ds2MHDhw9qvueaJ13rl8XgiAIYmmQ1YgnQRDLlE2bmJ+mzQaY\nRWXgBQXAgwfs+7VrgTt3gHv32Otdu4AbN9TtRuRs3qy8vRrydFZ5tFVWQye8ib92DfsAOHiNZWMj\n20Ct5lJ+HAX/zcRyVI1objbJlCeizHM0NDio6S+pRTqRNIfbDbvbjVN+P9vfFoQD7rQsOPRYlywm\nSp6e/Breu3oVM7HGXf1tbYbuidZ5q61fdv6hBEEQxKKTqeZCBLFg8JbORJYYGGDCa3iY+VxWVgKr\nV7PIJhBPq+UKjO+TrGmQ0jGMeIaK01lLSoB33wXq61ldqTitNYbg8zg8jPMOB3DsGNtGpaGQ8EzJ\nj6PwRj1hiFSbM2WCTHkiytR0JkSamtdmSvu7AizSeRopR3X3BYNY39CwIN6een5HyRsoKV1zfg24\n6ASYR6mRe6J13mrrl51/6BKH/u4RmYaeKWIxoIgnQTwsJEshFa/jAtPpZALP65Xml65ZExdxmzcz\nEcnDf/JjBIPAI48AMzNsX7MZmJ833uWVC6OSEvb1+OMsInvxoqLgE97EA9gTDgOHDsXFoVKk6Ic/\nBP7rfwWuXYsf59IlxbGNlqNK0EjjNUwQLNJ5GMqCTG8qrqwGd18wmHZjnHTFq2T/I4dTTyOOsRg1\nl8mivvIIp9I159egrKYGzpUrYbbZ4GtuNhw9TnbeauudVnbsWk8tDu/JvQgxQRAEsQSJRqNZ/WKH\nIIglzssvR6N790ajzz4bjY6OLvz+mWDv3miUtQmKRhsapOsqKuLrDhyIRquqotFPP42vf/ZZtq62\nVjr/0VE21ugoO8fi4sRjfPppNFpZGY3W1bHv+fZKbNzIxvB4pMfnx3nhhWjUYokfo6pKcZjp0dHo\n6YqK6LTSnJWOI742VVXZu0fJ7kFWjheN/zZegMOJmR4djZ5uaIhOp3gt090/F/j13r3RHwHRHwHR\n07L7/eazz0Z/BER/WVureo4LfQ06Xn45+uu9e6NvPvtsdODup9GG0w3R0emle/0JgiCIhSOm+ZLq\nQmouRBB6SOgoYzByku7+mSCZpUlpKcDT+errgRMnpPvqsTsRn2NJCeuGqxWZkUcA166Np7pWVQG3\nbqkfw2IBenpYRFYpksjnnP9ToNchjfqJmyZVVQFbt8avzZYt6k2Q9EQsk22jZSuTaeoAtIKl4hpN\nU00zOqsW7XuYuqi+VVeHvtZWeGprE1JZExoo5QDUaIggCIJIFT3NhajGk1hyLEpdQmJHmYXd3wiB\nABNodXVMfHFU6hsBMEsSgHWrLS5O3F9cx6g2fk9P/PstW/TN6+RJJiRbW4GXXmLpswC7XhcuJO4j\nPkZBQfx73hyntTVeO8rn3OtgDXhawVJPgYTjtH/nO/Fr09ubOFay48hJtk2ye5ANgki9NlLPuSZB\nrcYz3drPdJDXVWYL/jtKrX7yXCCAU34/ZiYmMjrPdM9PnN5syc9fkGtF6IPq8YhMQ88UsRhQjSdB\n6EHLhzLb+2uhZjUi7sra1MTsRBobEyNYR4/GooP5wK9/HY8GirvP8mNcvRqPjm7YADzxBBvP6wX6\n+tjyzk7gsceY0ObHOnmS1YMCwIsvsqhqOByfwzvvAG+/DXz5y8Du3axT7vAw8w7l5yI+xtgY2+7W\nreTCXqkBz8WLwO7dOLd7N0IHD+L61BSePHWKiYOkY+n4ACHZNmkViKaAG6l3uk3zwxK1Gs/F7C6r\n1Dk2E8ijuBy1+kmteYjX/3zDBpQ/8QTyy8sx3tsriRTLj5vu+YnrTE/5/Vm5VgRBEMTDC6XaEsRy\nQJyCWlERb/gjjqzJ033d7sRUSvE2HL6t2GZETkMD8NvfxkWh1RoXjGVlTNDydQBw4ABLq5WP6fEw\nISv36eSpu1u3xsfJywN+/3smRg8eZJG5xx9nIlosqkNgkc5LTwBDn8SbEnm9yqmFydKKldbJU1L5\nssWwV8kketKrk6CWSprNFFOtNN5kqa/pYDRFVWsefL3V5cJsLCrq8HgQHh6WHEN+3JmJiYydX7au\nFUEQBLE8oVRbglguqKW3csTRqXffVU7nFG+Tn89EnzyVkm/DO9vyaNfJk3GBaLFIj81tR7ze+DIu\nOgFgZEQqOgHWPVZsXQKwjrfDw2w+4pRaiwVob2fnIj7GF78Yf93bCwwNAW1tiWmhPOo39EncJmb3\nbnaaPPrmcmHP6Ci7tsnsUZTWyVNSNexVFirdMxm65pCmTQyP9skFi9ryTKCVxptfXg6zw4GRK1cQ\nXLcObxw4IJx/OvdFLYrLx3xt9Wqc2L1bGFuvxckju3YJ43qqqxOOwY/r8Hgw0d+P+UgEXr8/I0JR\nj/1MLjzLBEEQxNKBhCex5Hgo6xK06u3EtYNer7JgEG/T2yv1q8zPB1auZFHLFSuADz4A1q1jPp6N\njUw8cubm4t8XF8dtR3p79Z1Lfj5Lq+Uit6aGRUXn59lru52dAxe/c3PA/v1MdOfns2W1tcBrr8XH\nFAvm06dZRFX+Rnh6Ov691wtwAVBeDtfEBBxKolUPBlNSF7PGMZfmkA200njHe3sxHw4jGokgEgqh\nv61NOP+k1yQAwAfWrElBX8lFGv8dxcd80NeHwc5OYWwuvruamhSFG1+//+hRYVzx91wI8uMWb9yI\nwc5O9Le1wWKzZUTU6/mAYLk+R7nIQ/l3j8gq9EwRiwEJT4JYCmiJGz3RKfE2fDy7HZicBP7lX1h6\nbijE6kD/6q+YX2dnJxO78nR5sxlwudjy2lomOsXRSCXsdpYGvHIlS4k9c4bNpayMiU++jcMBdHXF\no6ZmM4tmtrYykVtRARw7Jj3XYJCl6c7OsnNQEpGxiBEAdl5cANTWwp7s2mphsGHQYtY4pjqHpRLZ\n0orS8fPm2AoLsfOVVyTrFK9JNxIbVIlQE2l8TFtxseLYWsJNPK7SMRxuN+xuN0LXrwNgfp/yuWfz\n3uXCs0wQBEEsHajGkyAyRZr2E0nHtNlYF9fmZmlt4cmTrEHPjh3x2kaleciXfe97wC9+wYSaOILJ\nsdlYNHN4mAm6SESaFnvlCvCFL0iXlZTEmw6p0dDAmgpFItLlK1YATz7Jjieu7RRjscTnqmRJw61K\nACYyz55VtjKRr1erZczG/URu2GgYncNSsNlIVt/J11lsNpjtdtz97W8xE3tW8yoq8Gc3bgCA+jVJ\n0ZaGX+edr7yCrkOHEsbORB2l+N546+vxjMgK6VwggJ6WFkRiP6da986o1U0uPMsEQRBEbqCnxpOE\nJ0Fkimx4dSYbU94IaN06FqUUd53l+8jHOX8+3mE2GZWVbFwuBk0m4PJlYNs21txH3JVWiVWrWAQ1\nEmFRzTNngPJyaQ0op6EBmJhg4pA3JyoqYo2GLBb2/eiougdmKMQsWaJRJtB37WLnyJsJFRdL12u9\nUc4F79UcIZcbzXCxdO/qVUFMygWWWhMejqaY5g2qDsO4LU0SMiHckt0b8XnDYkHl00/jwNGjqsda\nCh8wEARBELkJNRciliU5W5eQDa9OPdYeABN1lZVMKHHR6fEA/f0s0sd9K/k4WoKR87nPMcHHx/v8\n54H//J+ZyBOnrirhcrHj8OjmypVM7D31FHv9mc+wSKd4XsEgE7o7drCU2vPnmVCdm2PnVVWlntLq\ndgPHj7OIqtvNRKe4mZB8PSA0bWrfuTOxJnQhvVdzHD2NZhYLnq7KRadS2qfcnzIyNgaT3a66fQK8\nQZXo1M8FAvjpypVoLi3Fm6ImRYD+31GZaLSU7N5IUovn5iQ1rUrkYursUknzzjY5+3ePWLLQM0Us\nBiQ8CSJTGKz1S3vMYBCorwf8fhZJ5AKxpoYt37gxXqPpcknH2bFD/ZguV3wcHnHMz2ciko+3eTNQ\nWJh87hMT0qZEnBMn2FwuXAA+/pgJzT/9CVi/nonRkRFW4zkwwIRvzEICRUVsH73Xlottp5PtpwRv\n2vTee4k1odm4n6mi1dU4y8d0AFnrRJsuXCyVVlerdnQVi7Px3l7c7exEdGYGBVVVKYvpUHc3pgYG\nMDM6itttbfjl9u2CQJqJWaAsBGLxKhdp+4JBOMrLhW3tJSVJBWUufsBADYwIgiCWD5RqSxALQZbq\nBYVxe3pYWuuVK6xxT2kpizS2tbGI3ZYtrAGQ08kiiP/2b6xhj/xnc/VqlhobDrOmPkC826wSNhtb\nz2svS0uBe/fY99XVzHtzbIy9NplYumttLYvO8vnIPTs54ppOjpGU195eFum8cIE1PlK6B7zuUy19\nN1dYjLTfJZJqLE9XPRcIoPfkScyFw/Ds2JGQWpqptGE+DsCa+licTgzGnuNkaao8NXispweFXi9s\nRUXYFwyiq6nJUH2lEkqpsnye9pISfOPSJRRqNQHLMXI5zZsgCIKIQzWeBJErZPpNPBdR4npOJXh9\n43e/Gz++xxOPIsrhtZVGsNuBmRkm2jZuZOI3P599TUzEhaeY8nImfAGWUiuvNzWZWPMicQ1raSmL\ntBYVGRPvSteK3wO1xkK5xmII5KUiymVI6hqRKAL11lUqNdoRL9vz6qt453vfA0wm+I4cwduNjboE\nknx+fI6Tg4OK9ZVGGv6IRVrJli0Y7+2FxWaDtaAAvubmJSnatO6X0YZIBEEQRHYg4UksS9rb2+Hz\n+RZ7GsaimJl+Ey9vLKQUHeRUVbH/+/qYaKupie9rVGh+5jMsPVa8j9vN0m7v31cWmXK2bWPCt7+f\nRUDPnQP+5m+AN99kUVqLBfjwQ9Yo6cUX2TKbTdrx1oh4l18rhXvQ/txz8E1MZD4inSkWQyAvsihP\nVVCII5Gl1dX4ytmzhsRIsmZFyZrvcIFkyc/HO1euoKayUlGwRiMR3G5rg62oCJGxMUGoqglXpWOq\nXRuxSDvl9z8UjYIeloZIOfN3j1g20DNFZBo9wtO6UJMhiGUHrw8E2Bv0ZG94gkFjb+LVRC1ffu0a\ne22N/Qi7XEwoOJ1MqPGGPvn5LNV00yb2emyMRSi9XuDWLePRzU8+SdwnFFKuOzSZElN5AVbTWVjI\nhOf9+8AzzwA3brDvt2wBtm5lDYzKy+Pn1NwMNDay/fU0+xFfP17rWV0NrF0LHDmSeA/6+liklu+b\n7TevRlOvuQfrQqJxzGxHmnhtH8BsTvQKin3BINpj3YvVonzJ5i4+LgDYioo0vT7F6b2IRnEvFELf\nlSv45fbtcK1ZIxGx3vp6rG9okFisdDU1ITI2hvyKChw4dkwiVkdjP+viY6pdG17vmWyuelhKUcRc\nbIhEEARBqBCNRrP6xQ5BEMuQZ5+NRoFotLY2Gh0dTVz/8svR6N69bDul9cnYu5eNDUSjDQ3xsUpK\n4svlX3Z7/PuKimi0sjIa/fRTNp7JFF9nNkejFov6OJn4qqqKRgsLE5d7PNHoU09Fow6HdHlDQ+J5\nl5dL14+Oxv/Xus7icerrlfczci8zjfz+LkF+vXdv9EdA9EdA9HQWzuHNZ5+N/giI/rK2Njpt8J50\nvPxy9F8qKqJHSkqiJ/fvT9g/2dz5cX9kNidsMz06Gj3d0JB0PP71E5cr+k/FxZJlaueiNB/xsp9V\nVUn203Nt1Oaqh2zf20ySznkSBEEQmSOm+ZLqQop4EkSqaEUx5RFRt1t/lKunh/1vsbBmP/39yg14\nOG43iwTyZkI8lXTTJlY/KY48JmsWlAm4X+eGDcD4OFu2cyezU/ntbxPPw2pl5+nzxSO5tbVs/vx8\n+DUWR72Uajd5tFJshaLHs9NoRDpdloFVS7YjTfuCwZRrMXnHWQCChYg4Ypps7vuCQfx8wwaEY3XQ\nJosF06OjCIdCQkRRfswx/vPKMZsxK+psW1ZTA9eaNaoRWKX5iJd9+fRpSfOhPa++KkRL1a6NOPpp\nlGSR3VQjofJ9M9FMCUjvPAmCIIgFRkuZpvsFingSGebs2bOLOwG9kUweReNRPnG0ct06NkZVFVsn\nH+upp5SjmTU1iVFEkyka3bEjGt2/n0X3xOPYbOlFLs1m9XVWq/Jyj4fN4dNPpVHYhgb1iK04ullV\nxfZXi3ByxFFDebRydJRdY6Vrq0DCM5VOtFoPWue2BMiFSFPHyy9Looo8OidELYHo0erqhDlqzV3Y\n32JRjPyJI4L/XFER/dXOncLr/89uj/5vJpPw+qeVlZrXSGk+8mXZikJ2vPxy9Nd790bffPZZ4Vh6\nIrtKc1AaS23fpRRVzQUW/e8eseygZ4rINNAR8SQfT4IwCo9ktrYmej+K4T6Q3E+TR+W4nUhHB6st\n5N6Y4rG4JydnZoY1CTpzJl7XyYlGgQ8+YNHBq1eZr+fq1cxKhNd6pkqy6KhafejwMIt2fvvbrDMt\nEI/sKfmHFhay2k6+3Ucfsagj//L7lf0redSwupptI24Y1NTEbF2Urq0WPGqq5x6nCo/eLnLtnNz3\n0Qhi/8jFItTdjcj9+wCkHpX7gkF4/X546+sTmgudCwRwyu9P6rXJ/SxXPf00gMTI37gowjk9MICJ\n3l5hDmU1NZIMg+INGzTPw+F2w+5245TfL9wL+fXNVoRZySdT7d5qzSGZ56Z8X6rNJAiCeAjRUqbp\nfoEinsRyQ289II+aiaOGxcUsMrl/P3vNay1raqLRF16IR9k+/ZRFL/Py4tvt3cu2KS6W7iv+2rEj\nvQhnJr7E8yovj0b9/vh1euGFaLSsLDFa6vcrRwArKuLb1NdL1yWLGoqjoSUlxiKL6ey7hFCLFi4l\neGTySElJdIzXM2sgjrQ1ezyK0TmOOPInjuZCKrKxAAAgAElEQVSJI5z82Hw7cbRVHBXVinpqRQCz\nFWE2UkurN1KsNJZ831yImBMEQRCZAzoinmSnQhBG0WszIbfxEOP3s26z3E+zvp6NK/f6fOQRVuPJ\nsdniUUwlKxTuqcntVZLZrKSLy8W65g4Nsbns3s3sUTo6pNFJsfWJ0jXZto1FLXt7E+tfS0vjkWK/\nHzh+XN/cuH1NSQlw6RLr4quXdPZdQohtKOwlJXj+5s2Uo5eZ6IJqdIxzgQBGr1/HWE8P/O++i0KN\n+yTuEhseHobV5RLqMPMrKvCtGzeSHlN8vfIrKjA1MAB7SQmqnnkGk3fuwOp0Ir+8HPd7ejD0/vuI\nzsxI9tey+hB7cCbzAc0Ecj9SrXrRhRqLIAiCWLrosVOhVFtiydHe3r64E9CbJslTQS0W6fKaGmbp\n8cQT7DVvgCNuOJOfz0TavXvSfbnoNJmUU13n59k6LjazJTpNJpY2+/77TFgODbH02lAIMIt+rRQU\nMOHIhai8CQvA7FV6e5VTW/Pz2f+FhcDf/33ivoEAu07yVFye5nzzpi7hKDxTgQCznKmoWNaiE4in\nPtpLSvCNS5fSEgrJUiy14Om+N48dSxgjWSpwqLsbdzs7MTUwgK5Dh3TPMTw8jIKqKjyya5ewbmpg\nQHPe/HpZXS64N26Et74ez9+8ick7d4R5/+Ff/xWDnZ34/cwMnJWVMDscAKSWLGrw9F7eSCjVFGgl\n5NdRfL9+vmEDpvmHOykgHqvr0KFFT79eriz63z1i2UHPFLEYkPAkiGzBxc+HH7JIJGfNGiZa+Xpe\nmyh+zYWYWh2lWhbB7Kz6OjW4f6URolE2v8ceA155Jd6xt6ODiWWnkwnuBw9Y7WkgEBd1YqxW4B/+\nQb3LKxfO4+PA976XOA+1etvYhwPnjL6B7+5mdaEDA4AOMbOU4ULn+Zs3NaOFWqRTr8eFC/e5FI/R\ne/KkIGraX3oprWOKt//mRx9h/9GjyK+oUBxDSfDuCwbh8HgwOzGBOx0dGHjnHbzd2Agz94kFEI19\nMFT82GNouHYN5bW1AIDI2JimOBbXVeoR8mqiXGm5fDx+LficeeffVKBaTYIgCEIvJDyJJYfP51vs\nKeiDR0a3bQP27WPLeHRTvJ5HB8Sv+RvDmhqWbqqF1crScI1gsQD/7t8BzzxjbD8xMzMsxTYQYFYp\nAItObt0aF40lJSxy2dKSKDxnZ5nAk4tw8fgck0L2hoYtid5InPBMLQObE72k2hxITZTxaJ3R8bhw\nKa2uhtfvl4wxFw7HN5R9oKJ1TPk85ds73G5868YNxTHUGu5Y8/KEbcJDQ+hrbYXN5YJJ9LPnrKzE\nf+rqgsPtxnis6ZCtqAiwWJJ+CMLn+7PVqzES+zCotLpaVcypPdtKy+XicF8wKIhureMoXUsx6dx7\nQj9L5u8esWSgZ4pYDEh4EsRCoCasxIjTRouLgfJyoKwM2L6drd+4Mb5tYaF03y99KS6a9GC1srTX\nO3dYdM8oXAQ6nUzw/tM/xUXi+DiL2ALxOsneXiDWfVQ4PpDo0Sm/NrwLLk9PlqNxXQ1HY/Tcp4cc\nI11QAe3OuVy4fOXsWTxz/LhkDE/s/pdWV8NeXCwZR0s4y+fZ1dSEycFBvN3YKMzDSPfWc4EAwvIP\nTiwWRCYmUPH5zwNgfp0N164J4/FIcmRsDLfb2pJ+CMLnO9nXh0hsfoVr10rm9otNm3DE7cY/l5eD\nfwwjf7aV5q4mutU6/2pdSzG50N2YIAiCWBpQcyFiydHe3r60P6kLBFhKp7yRzsqVcRFYVgaMjLDv\n6+uZTUplJYscfvIJS2HljYkA1njnvfeA/n59c9i5k0VSOzqAyUlj83e7gS9+EXjjDeDJJ5mwFL8h\nt1qZcB4bAz7/eeDECaCxkaXDihsiVVXFrVPU0NvISYVwKITzgYBms5OMPFNq93UByUSTHy2MNsER\nN+XRarAjR3z/Tvn9muOIz38+lkLK56lnf6Xj8vMTn4ccl9eLyIMHsNjtcK1bh99HIti5aRN6T57E\nzOgoSqurke/x4LZoPvLrxq+rragIkbExxe2OuN2CfYyzshIVTz2FPYcPo6upKasNfhay8RGhzJL/\nu0fkHPRMEZlGT3Mha7KVBEGkiZIY4XWJAItmrlnD1k9Px/fjzT6sVuBv/xb47nfj+4g72wIsZfbU\nKUCclqhFV1d8fD3w7rglJezr+PF4nac8xXd2Ni6aOzqAF19k5x4IsPNqa2ORTj1RRR4JTREejVkQ\nxPeVe4EuMDwyBQDnA4G0z11JyO4LBnWJeY44AmfJz8frPp9uYSy+f3qi1+Lz9/r9WN/QIMxTvr/8\n3MTibV8wmHDtrCoZBVaXCzP372MmFqWc7O/HEIBP3ntP2KZw7Vr4jhwRrpv8WOLruvOVVwTh2NXU\nhN6TJzEXDqN8xw6YYo3KLE4n6t95R4iois/7V088IdSWZgqj95wgCIIglKCIJ0Gkgt7oltg+pKGB\nbXfsGBNgSnYoAEujjUYBbnBvtwNFRdIIZyawWlkHWpntAwAgL48dl0cyy8pYLWdzM7B2rTRt9sAB\ndo5K4wCsM+zatSy1d9Uqlnb77rvGO8bmQEQxKdyGRa+ozgKZjkylE63kGI1aJhvnl9u3w1lZCXtR\nkaJwVTp/LjDvf/IJ5sNhWBwOFK5bh9Hr14WGRo7yckRnZ4XXJquV+Y2Zzfj6xYso27YN4VAIv9i8\nGdOxrAT3Zz+LqTt3EOYfsqhhsaDy6adRUFmJ8d5eWJ1ODH/wAaZjNkkurxeutWsFOxa+zb5gUHK9\nAGB1XR3uXb2Kr164IGkIxc9bbBGT6v0iCIIgiFTQE/Ek4UkQRuHRLC6+xD6VcrgY8XhY1HB4WJ/F\nicnExCf/P1uYzcyCRUxBAUuhjUSknpvl5Uz4bdgQF8EWC/PztFji1i9btjB7laGh+DqxUAWSXzM1\n5CI+195Up5kWrIaR9NlwKIRfxcSZTUWcGUGPkFWbn9Jy8XglW7ZIRJaeeWoJYaUU2Z84nZibmlId\nUyzWABYRHf7wQ+HnwpKXhw1/8RcIdXfDbLPBYrfDbLPB19yMtxsb0dfaitLqajy4dStBhJosFkRj\nP++O8nKEh4bYcptN6IDrKCsT9hNv4ygvR2RsDPOxTIbSbdvwlY4OxevEz3t6dFSSXkzRSYIgCGKh\nIB9PYlmyKN5TgQCrwSwtlYpOi0XqUynfh3tCPvoocPductHpcsW/56JzxYr4cZKxebOx8+FjykWn\n2Ry3QCkokK4bGmLndPEiqze129n53L/PRGdFBas17exkArW8nEVt+bUqLmb/p9oxNosdZzPyTOn1\ndzWIWmMXpaY9DrcbBWvW4G5np2YnX62mP4C+jqVGuquKxxvv7ZWs1zMftXRbvu/bjY0J6aBzski8\nragIAJjHpsWC2ViKO++qW7Jli+TnwrNjB+5dv46Bjg70t7XBVlCAZ06cENJjC9etg62gQNJ1+Q9O\nJ1bX1WHl008L8y17/HHh+0dizYhKq6vhqalJ2MbqciE8NCSITgAoXLdOiOAq3ff9LS0oiHmH3v/D\nH3C6oUFYr+fayklln6XCUjw38lwkMg09U8RiQMKTIPTQ3c0a/4yOSkXn3Fzcp1JpH+4JKar3SsBu\nZ+mqzz0nXR6NxlNxuWC125VF6I0bxs7HYmHpu0C8ztPlkgrRU6ek+xQVMcHn9QK3bycK0127WO2n\n282+HA62vLCQRX6vXEmvY+xD2nFWTWypCT69nXz1WM3o6Viqdryxnh4ATOjtfOWVhPHk++mZj5oQ\n1uq6CgDmvDysrqvDN69exfqGBiY85+aA2VlY8vKErrrcAoVzt7MTY598IpkrFy7Htm7F1MgI7nZ2\nIjw8DJPdjrwVK+D7yU9QsGoVZqemkFdRgQPHjuHA0aMoXLcOD27dwr0rV5C3YgXcmzYhIttmfUMD\nVuzaJZmDvaQEvuZmnAsE8HFzs6q36XhvL+bDYURCIfS3taFl82aEQyHdtkJiUtlnqbCcz40gCCKX\nIeFJLDkWpQubuLHI1q2s02wsmpEQgeO2KNeuxZclS5edmWEi7s4d6fKaGvYlJhLRl6qrhLgJ0Nxc\nvIGRy8UaBonSDYVtOEVFTDz6/ez/UChudQIwr1K53QmvQRsfZ+fn9TLBuHkzixwfOBBPT+U2Msmi\nD1mKKAK57WemJrbUBJ9eX0XDVjMa8yvZsgUtmzejubQUbxw4gIJVqwAwK5GuQ4c0z0vPfMTCVRy1\nssSebaV9v/7BByioqsKf/f73ePbNN1Ho9WJ/SwssdjsAlg5b+vjjgs2KUhOhsscfl8yVC5cHfX2Y\nFXV0js7MYHpwEJaf/xyh7m4MdnZiemAAXYcOCdHoqbt3MRMKYXpwELfffluyDbd8MQFwxLIdzHY7\nih97DG83NmL0+nUhRZcdUPp7RT73qYEBnA8EUrrXmXo+cpGleG65/DuKWJrQM0UsBlTjSRB6CIWA\nl15ib/Sam5n4UavpE9cickwmZi3yySdArKmIhKoqZoUijjiaTCwyqdSAKBWS1Ys6HKwrrrzhkdnM\nROLFiyyiye1e/H4mNF98kY175EiiIFRqtiO/NuXl7HhcBOdi7WaOotcqJp39jdSXyu1G8isqMDUw\noLvekM/Hkp+vWvspns/M2BgGOzsBAN76eljs9qT7yml7/nl8+qtfwZKfL1iU8C645wMB/OnUKUFU\neuvrkb9iBULd3Rjv6cHM+LiwjxJevx/DFy/iQV8fYLFg5e7d+NKJE0JNKMDSaedmZhCdmYGtuBjf\nvHIFZw8elHTlvd3WJqk/zVuxQmhK5P7sZ1F//rzkHMOhENpfegl333kH04ODcHg8KN64Edb8fNhc\nLviOHNH9rKT7fOUyy/ncCIIgFgtqLkQsS3Lee4oLLpcrMYro97N1cusTLvz0UlwMrF4N/O536c/X\nbGbNhPr6WK3m+HjiNuvWAX/8Y/y11cpE5NGj6hFIJWHOrw3AoqAPHsS3V+sGuwDdbHP+mVokjHS1\n5Y2DAFa7+MyJEyn5SSY7pnhdXkUFpmXCVu98zwUC6GlpkYhHuUB+48AB9Le1wVJQAEdxMaYGBxHV\n8SFQ6bZtKPrBDzD5d38nCGM+nz2HD6P9xRcxcOFCQiOi9Q0NmJmYkDRzCq5dK5nj6ro6mO12IBqF\nr7k5QZT3njyJ8L17sOTnwxzr3jscs06iLrdLG/odRWQaeqaITEM+ngRhlEyIHLlnJaekhKWsyhv6\nFBayqKER4enzAR98YHxuQLyTLY+Azs+zWtSyMmXRWVLCmgmJhefsLDu3DRuAJ55QvlZKHpzBYDxy\nzJsYVVczuxWlqCmQE/6YDytGUhL3BYOs5lAkivQKHXEk05wkbVY8nwPHjqHr0CFY8vNxyu9nkcjY\nBz1WlwufvvEGjhQXw2y34+sXL+LSD34giZZyQWcrLkbl008L0UA+F4vNBntpKWbu3cOk+AOSGLai\nIkTGxmCy2WArKGAdb/PyMDkwgIvPP49NsVReACirqREE+DMnTkhEOgDAYkF4dBRf+PGPcfLpp2F2\nOPB2YyPMIp9dW3Exwvfvw15UhPzycpzy+yWR3d6TJzEVy0iYjzVUMpnNqteSIAiCIBYaingShBgt\nyw49wpRvY7MBV6+y1NqSEuDSJVbfqGTtoGRrokZNDXDmDGtGJIqoaKJlzbJiBZurON3W7QYuXwa+\n/e14pJIjjlimkiKr134kB/wxH1YynY6rtq0kkrliBR558smEiB4AnD14EH9qbUXZ44/jwNGjCVFO\nNQqqqjA9MiLYqpgdDsyHwzBZrfj6Bx+gbNs2YdufrlwpCDie2spFJsAEoMlqRelnPwtHSQmmh4Zw\nN/ZzaLJaJVFRS34+rE4nympqhPny69qyeTOmBgYklivrGxowOTgonI/JaoXJbMbKvXsRmZwUIqgO\njwfhmKURj2Q2l5YKPqQAizq7N23C7bY2eKqrsV90fIIgCILINGSnQhBG0bLs4NG31lblTrYAcPIk\n26atjY2zbh3ztvz2t1kjoXT53e9YZ13elZajZbnCRSfvNiumpoZ13m1oAP7wB9Y8ye9nUU6vl4ls\nbu1iszEhnZfHXqdqb6K3WZBWN1u9zYkIw+jpamukQ6hWJ14ArDmP3a54zL7f/AbhoSH0t7Wh/cUX\nAQDjse65ptjzb5P/XAB40Ncn8fLkNiXR2Vlc/Ou/lmwb5n60iDcVWl1XB0dZGfJWrEDxpk2YGRnB\nQEcHbre14e4777CNzWaJ6LQVFqJ02zaER0bQ39YmOV+H241v3biB9Q0NEsuVPYcPS65FdHYW8zMz\ncLjdsMfOy1NbC091tWQfACiPNfsy2Wywud3I93iYt+jwMG7Ljp8NlqJFCUEQBLGwkPAklhwZ8Z5S\nEyvl5eyLv+mVb6fHS1KcMtvWBty6xSKTra0sssnhvp1ud/JIpJxIBHjsMfZ/XR378npZ859kIq6m\nhglKufCsq2Odeg8eZDWpxcXAiRNxaxQ+x48/ZgLwc59jacQjI6wpUrajkFoCVc+HARqQn1nqGEnH\nTdaJ15KfD4BFFGGx4KcrV6K5tBQ/W7UKJ3bvxlt1dYLnJgDBN7Mg1j05OjeHgqoqwS7FKvbFTcLA\nhQv42erV+PXu3Xht9WohTRUAzDYb9re0YPLOHYRHRjA9OIiJmN2Kp7YWc+Fw/GdXlLHwMYDI+Dju\nXbkiLLv1m9/gzQMHEA6F8ItNm/AvK1bgj8ePY25qCt76eqG+dF8wiLyKivg1KyhAeHQUe159Veis\nuz9muyKuSXVWVsJRXg6r04lIKITbbW2CpY3V5UJ4dFRTEKYjHsmiJLvQ7ygi09AzRSwGJDyJhxM1\nsdLbCwwNxb055dvp8ZLkNiMFBSzCyaMgJSWsO2xVFbBzZ7zx0NSUMeFpNrNx29rYMVatYqK4s5P9\nz43sucjlgvPMGSYoRbVnOH8eePNNdt5a4o0LQB5Rqq0FPvoo+6mvWhFNPR8GEFlDr31Lsm0dbjfK\ntm8HAETu38fttjZMDQxgZnQUk/39GOzsRF9rq2CBUlZTA3tREV73+TD4298K40zevYtjjz+O6dFR\nWJQi+3LMZoRHRjDZ14e7nZ2sC62IW7/5DX62apUkqln86KOCUNT6uXV/5jPC9/y82l98EZMDA4hG\nIojOzuJuZ6ckwutwu7H6S1+CKXausw8e4HZbG7oOHYLd7cYpvx9vNzYK6c9cLPaePInw0JBQu2p1\nueDeuBGOsjLMTkzoinqmIx6XokUJQRAEsbBQjSfxcKJWNyhf3thovL6Q1y52djKLFIClwX74IfO7\nlB/nD38wliJaUsIijnxOmzfHbU4A4K232PHffBP4/vcTayh7e4Hdu4ELF4Af/ICJ62vXgOFhfZ1l\nX30VOHRIuzbTKGr1s1p1t3prRYkFwUjNpxjecMdTWwuH243b4sZcgGA5wjvl8hpJNRxlZZidnITZ\nbsfc1JQkkmkpLMScqJGWvDZTC0teHsp27EDo+nVWV2mxKPrrmmw2qe8mgILVqzF5545wPJPVCmtB\nAeYjEZisVljsdhQ9+iiGYt1oAcBeUoLnb97EKb8/oWuvvMbV5nLBZLMJ9Z6Ttgo4IwMoqanFV88k\n/3BAfA/0fJAghixKCIIgHm7IToUg1FASK4EAcP060NMDvPsuE2Xi17GUPmFbuUjiy3p62La8FpPj\n9bLurVy8Pf00a85z7x6Liqq8eVWkspKJRbebpc6Ka0erqlh6rx7Eoq6qSj2CqSX+MoHaMai5kCp6\nRF6qQjDVfY1YsIgRCxcAaH/pJfS+8YaQMbC6rg7PvvmmsL28mY4SjrKyuG2JqGkW9xi1FBRgTqFj\nrRgtUWp1OmEpKEB4aEjzHIF4Y6PkB403AjPZbPBs3w5HaSnmIxH0t7UJwrCrqQk3jx1LuA7cambE\nVYsfThzDN3AIE/WHETyhz0uVxCNBEARhFGouRCxLMlKXoFQ32N3NopQDAyyiJ38tRilVly/r62P7\nyQ3mx8fj+xw6BKxZw7rI8je1zzyj3PhHic99Lj73WG2cgOjNuSbiNNVkabPZSmcVp9HGbDQSjqEn\nvTlNlmqti57UyHTSJ1PZVy3lUqt+UNzIyOF245njx7Eq5jFXVlODL772mmR7TyylvWjzZpjz82Er\nKYGJP0MxTOKGW7GfM09tLfzvvov1DQ2oePJJzfOp2L0bXr9f+VwLC1GydWuC6PwYTLACLOXVHEub\nNVmtmJdFQBWJRuGsrIS1oADRSARDXV1CqvH6hgaUbNmCU36/ougsq6nB12Ln9+6u07gHLy7VtuD/\nbU7ebfh1nw9vNzYK9jTUJCi3WKq/o4jchZ4pYjEgH09i+WLUk1Murhobpa/FY167xl57PCyddvXq\neM2mEtXVbFve6VY8Pl//2mvA+vXafp5mM0u1fewxJlwnJ6Xr//qvWS2nEvJrwj1HtdJU9W5nBO7J\nyQV6fT0TmPJjKPmBLkNSiS7qqatLp/YulX33BYOKUTMuYgHgfCCgGAnl12CspweFXi+s+fnw1tcr\nWqscOHoUv9q+HfmlpZjo6UGEC7BYtNBRVobCdesQDoUQnZlBWU0NXGvWwNfcjK6mJtw5fx6zU1OK\n6bBiBi5cQP6KFYp2RLPj47h39ariftHZWZjtdsyKfi8ki5yu9Plw97e/xXw4LEQ0g2vXSra529UF\na34+ImNjgr0LwDrorti1C9aCAsGP1O5242C/HzsrnPifjwXhFnmUyp8x8b0RW7Wo3SeCIAiCSAWK\neBJLDl8sCqIJtzVpbQVi1gtJkUfWlCJtPKo5PMxSUzduZNHNvj7lOk2Xi0Xztm1jTYQqKoBjx+Lj\n+/1McJ09y5bxxkQAi2TyRkDV1ay2E2DdMzs6WG3o/fusu60YU5IsB3mkVq+lid7tdHIuEMDrLS14\n6/59hAF2bs3NGT2GEczB4KJbQaQSXdTT2MdI8x+1fXmETc/14aJHvr0eEcuvAW/2c7utTdVaxeF2\no2DNGtzt7JTUbyIaRUFVFdybNmGoqwvRmRlYnU5YnU7MxbYLdXdjamAAkfv3EY1EYLLZhAilnOjs\nLCb7+1UbCc2Ju+Da7XCUl2MjWKSTd9a1FRezDVQsjyxOJ8xWK/7s44+Fe9XV1CQRl6b8fMzEGiEJ\ny2PjRcbHhSixWEwOd3bAM9CKq4cCkuurZmejZtVCLD66/+4RhE7omSIWAxKexPJFHDlMJsY4bjf7\n8vuZWOTL+Gu5ncpHH8U7vPL/a2rYtqWl7PXEBOs829ubmLbrdjPLkhUr4sf48Y/Z93Y7E6ozM6ye\n8+xZZpciPh/xG+Gysvjxi4rUu8DmSAfYUHc3Bu7fRx+A8zYbcOnSotZu5oIVRCrRRT0+m3q20dp3\nvLfX0PVRup77gkEUrlsHi8OBtxsbFQUsvwbci9Ph8aC/owPNpaV4I2ZFwjkXCMQ72ooEXWl1Nb75\n0UfCGJ7aWpTV1OBurDPuzyorMXL5srC9yWoVOsymhKgue35mBiaTSegkO3PvHqxOJ9ybNiG/ogJ5\n/OdUPsTkJG63teGd//AfsL+lBV1NTehpaZH8jDsKCyXXxl5SgpW7d7Pr5nJhWmaXovQ8cXsVW1ER\ndr7yirCt+MMJJasWgiAIgsgEJDyJJYfuugRe+1hYCPz93+vbRx4R1LJT4a+vXmX/nznDaiy5wCsu\nBl55Jf7a5WJpsuI33eJjHDrExGhBQXw9r+cMBuMNjsSi0+Vix+XHF1ujbN8uFaELUC+pB+FNcUkJ\n9nzyibRxUwYw6kd4fWqKzWcRozzpRCazjVFRrLS9OEKpJmD5NeBenCazGdODg5gZHUW/zA4k1N0d\nj3TOzcFst2N1XR2+cvas4IfpWrcOZocDoY8/Fvabm5oSLEdgMglzNYJadBQApgcHcZU3NAIwGw5j\nqKsLUwMDmB4cTD5G7Oc61N0dnyOYsCzZvBkurxfuzZuRX1GBb1y6hC+dOAFHeTlmJyYSro/S81QY\n+zmLjI2hS1S3Lq+vTfWDCiJ7UD0ekWnomSIWAxKexPJl3Tr2//h4YnMgNd5/n/1vtQL/5b9II4T/\nf3v3HhzVeeZ5/PdKfdENqYUkLMsYGceY4AQb2fgaKGvWJo4xDp148SSe3eCdyqomrtp1qiZ4s5PL\nTtXEtalJpWaSmirXpioLGSfEBmKIMSYuZK7GNg4bcBJDjA22bAxCCCSEuLRuZ/84fY5Ot7p1aZ1W\nq8X3U0WZVp8+5+3Tr4Ueve/zPMXF9mqkN5fzqafsPMtFi+xcz8ceswM8J5A6d86+9tq1Uk2N/drm\nZmnOnNSrqM4P9c6W24YGafVq+++RiF0VN1l3t902xdmm6g1yz57NbGttlrk/FB87prDPQac09hXM\nW7/3vZwHfZP5B/7kIGakwD5dED1SAOvcg2n19bp/3bqEQjzBioqE1yQHjAM9PTr9hz8knKts1iy1\n7d3r5iwOYVmD21a9uyIKhv+ncUyro2kqVYeSPufK+fPVuGaNJM/Kb0WFVFCgvu5undy1S93Hj7tB\n7Kb4DoiahQsl2avDF06ccD+T5Pm0u6lJHYcOSbILELGNFgAw0WingsllrAWBhpNJG46KCsn5QdRp\nL+IU1YlGB9t91NZKhw8nfs2xYoUdDCZf2xlPWdlg8BoMSvfcYz+/Zs3gGJPbvXiLGrW326+74w57\n+27y++vstAsPeSttLlwo3XSTvRrqx72d5MbTjxAj86Nlymg+k5eWLNGJ5mbJGLdCbe3nPqfPx4tn\nPTd3rmKeVUTJbj9SXFOjstmz1b5//8itS5IU19YqMneuTib/f+2nggLNuPtuheO5nwWhkFsUSJJ2\nrFypo88/r8LiYvUOs2I/bfZsldTVqevoUQ309bkBdn00qgc2bkw49tmrr3b7nia3pgEAYLxop4L8\nk6pNSaa820qfeip93qOXU8ynpER67bXEFULvCktrq11YyGnf4OR4OquWqba0Ol/z5mr29trvNxRK\nXcnVCTrXrUssatTWJr30Uupts5GIPffB4sIAACAASURBVA7JXjFdvtw+xrsFN/neetuaTIEWCpls\nWx3r9tx8Nt736qzIhaur1e1ZZRvJWFd1S+vqFK6pkQoKZPX1yerr08ldu7SnqUnhSERfefddlc2a\nlbBt1ert1cUTJ9S2d2/KoNMEg27Rn1Rm3HmnHdiOJi98rJyV1IEBte3dq/Y//EHn3ntPJ3bs0HNz\n5uh8S4sk6XxLiwZisZRBp/NeqxcuVO+FCzq1d68utbaqx3PsqddfV6yzM+Fz7otvJ5fktncZyZX0\n/wQAIPtY8cTkMopVyp07d469Gltj4+DK5IoV6dtztLTY22Zfe21o3mFnp10IyFtFNhCwCwlt22Zv\nd03VbiR5FVeS5s2zg1fJDg63b098nfc1XV32yqZkV389diz9sc5KZvKKqTT8vR3t/Zmidu7cqa5/\n/MeMVvHy0XArlqNp6+KsXHbHA7zk84ylNUy6Y3c3NenounUJuY6SvSW1uqFB51taFCgpUW9Xl045\n/384Cgrs6s/Ofx2FhTIFBWnbpwQjEVV+5jPqbmnRxZMn026TTSdQXq6+ri69K2muJBMKSZalUHm5\nZtx5p/p7euwVXA8TCLhbdwMlJaq+/XZ1vPPO0O3BhYUyxmjGnXeqqLpajWvW6NfXX+/28fS2QZHs\n1dDLZ8+6969oxgxdbmtTVUODlm3fPqrgP9OVbfgvo3/3gGEwp+A3VjyRf7JV/Ga01Vzr66WPPx4a\ndDY12dtq45UlXX199uqjN8cyWaoWJocP2yuR0ejQoDP5NfFKlKqsTF39NdUqcXIuZ1OTHcB627lk\ncn+msPH0u8w3w73X0eTHOiuXIU/lWO95xpJjm+5Yb4GdYHm5CouLFaqsVMlVV+nc0aPua5xKrdMX\nLNC1S5cqVFXlBptOFdnpN99sf72/P2XQaYJBFc2YoYJgUG179+ri8eNu0BmKRNxKsl4F4fCQr/Wd\nP5/w2OrpkdXbq9iZMwqWlmrJ+vUKJ1W2teLXKSwpUaC0VK27dtkBZHzFtaCkxF7l7O+X1denU3v3\nui1mquO54NMXLNCX9+9XcW2tJPvzKKmrc+9fqLJSX3rrLV2/YsWog04p9TxhFRQAkCkCT0wuoyh+\nk9Fv6JyA9qabEtujjJYT3J09a2+vdbbYSnaPzeEClVRBXSRir552dAwWJHI0NdlVciV7NbW+3g4Y\nDxxIXf11NEHjkSND27l4TZJqt7nS2Ng4qavK+m249zqWADzTIkKjud75eEBpAgF9cc8e1dxxh3o6\nOvRJc7O7yjp9wQJF33xT169YoYd37NCDW7YoGK9mHayo0EPNzfZzu3YpEP+6kyvqLSBk9fbqclub\nYt686Pi1p99yi6obGoaMO5hqu258d8/cFO+z4bvfVTgSUc0ddwx5TWFRkR49dEgD3qJF8XMNXLyY\nUMzIWxhoSbz1ycM7dmhafb0ePXzY/Ty649t2TSCgh3fudAs2jWVup/p8J0ProSsRK1PwG3MKuUDg\niSuDE9AOl+c4HG+l2N5e+09dnb1quWPH8MFauqAuXT7rkSN2QCrZqx779qUNGHc3NenFri69XFur\nWKqVzOTxpwtOJ0m121yazFVl/Zbuve5ualJvV5eKa2u1ZMOGEe9FuvOk69mZarUsXfBaGv8li9XX\npwM/+EHKticXT5xQqKJCoUhEr0SjennpUhVfc40kqffcOR34wQ/c8V08ccI+X3+/CouKFHT6YsYD\nSG/epyksVKiyUlZfn1p37VIoEhmywhnztEwZjd899JB7fwuKitxczekLFug/nTypA08/LSctxRlL\nMF58SIWFUiCggnBYJhRy72ny/Q9HIgpFIlo3b54uOO83fv9SGWn1MtXneyXtDAAA+Ct9MzJgkso4\nL8G7ktjQMLYtpWvX2q/v6LDboYylUq4T1HnH4VSolYYGg94gMRIZvF6K8XYeOaLW+OrPnlWr0udg\nOeNPlYMKcl3iOo8ccfMl9w03n1JIztUsnTXLzQ/c09Sk+9etc1fLvF9zgptkqbbxPj9vni47udGy\ne2b+e02NpMEWJ0We7aaLf/Yzd1zeXM/+y5fVf/myJNnVZSMRxeKrqZIdnDqBZvXChWpcs0a/W7Zs\naC5pGk6Op1fJNdeo49ChIeeYdt11Ckci9tbiePAXKC7WNffdp3t+8hO9sHChm7s50Nen9n37Eu6f\n974X19Tow9/+NiEvNlRZOSRAdF5z9o9/dHNEnfOl4r3G4mee0b5Vq9zKxGPJ50Xm+B4FvzGnkAsE\nnrhyeFcSZ80aWwDmBI+pivZIY2sD46x0SnaF2uQA1hskOudOEzAG4tsRqysqtPhHPxp5/JOFn21z\n4JvxrGYlB5WpzjXS+Xc3Nall82b1x2Kquvlm1Uejaly9WvueekqdR46o6rOfVcGtt+r0/v263Nbm\nVrt1FRaq4lOf0lV33qlQRYVeiUbV9sYbGujpGfY9hyIRdZ84oYJQSAM9PQpXV2tafb2MpPIbbtAr\n0ag633039QmSCxilcXrfPhV6tvta/f12UBvv0+td0b18+rQ+2rJFH//udxrwFDgKlJWpr7tbgbIy\nxTo6FOvsTLjv4ZqahKAzWFGhRw4cGBIMel8jjfx5e49P/oVEql8mAACQClVtceXIpK/naHmrwobD\ndkB1223S+vVDr+PjOGKLFmnP3r1aLCk8nmq0Ex0IXuFVdCcrb59NJ9gb7UpWcu9USQk9O3c3Nanj\n0CF1HT2q6JtvalpSvnKqKrZOJdXk6qrtBw+q6/333TzI5ODv+hUrdLGtLSG4SiUYiWggFlO/p9VI\n6cyZKquvd1cmvVVnTTA4WJyooEChigrJGPWcPatQZaWqbr45ff/PggIFy8rUG+8TXDpzpv7jn/7k\n3tdYZ6fWzZvn9tpMJVRZqd7ubncM4epq9Z4/b7eNKSxUqLxcPR0dCkUiuuqee/QffvWrlJ+b81lV\nNTSobNYsNa5ZM+znm/zZeufGQG+vTjQ30zMXAK5wVLXFlWe4fpRr10qzZ9uBYXJBn/FyVisCASkW\nG9ySmyqP1MdCPuHyct0vKTzaarTp7o+f/VNHgyq6WTHeiqPenL6xFpFJztVMzg90tvFeam3VvhT5\nyt4qtlJiEZ3kvqFdx44NBp2SwtOnu3+fvmCBCouLddbZVl+Q+p+5wqIiTf/MZxKCTkn64muvuf00\nvSuqocpK1d177+CBAwPq6ehQz9mzCpSUKHLTTTLBYPoemQMDbtAZLC/XF197LSFIC0cievTwYYWr\nq1O+vKCkRD0dHW7QGSgrU6y9fbBXaX+/ejo6VFJXp69+8IEe3LLFDfjT5dUu275dD2zaNGKwmPzZ\neudGsKzsiinKBQAYHwJP5J2dO3emf3K4ACoSsbfY7t3rT4DlDeKeecYOJr2VLisqsl/IZ6xBbLr7\nM9GB4CSrojvsnMojflYcHeu225GKM410Puf5YEWFrl26NKHtR9f778sEAurp7LQr2nq2n1bcdJO+\nvH+/yurrFaqqUlF1tbqOHh3sb+kJSh2FJSV69C9/Sdkm5fUnnxxcjY2vooYqK/XIgQO6f/16t2WJ\niVe2DpaXq3L+fLXt3asTzc0a6O1Vmk25dpEgSb1dXUOC791NTVo3b5560vzCIOQUQSotlQoK1BcP\nmANJ1XVrbr894TNINSfGUkhrd1OTXolG1dPd7X7N+1k2rl59xRTlyqWp8j0KkwdzCrlA4ImpJVUA\n5Q0QnTYofgRYmzcPBnFPPmkHkwsX2s8VFtptVrJtrEFsugBzogNBquhmhZ8VR/1uLzPS+ZznH/vw\nQ3e1znGprU1WX5+7+ljozYc8dUp7vvENlcycqZ4zZ3SiuVltb70labC/pVNwqHL+fJXU1enRQ4c0\nrb7eLoI0c2biQIxxA9LC0lIVzZihRw4c0LT6endV8voVK1R9662S7CDSWSFNDgK9XwtFIiqOF0IK\nVlRIhYXuSuSOlSt1dN06XWptdd9jcW2tTHy11gQCWvKb3yhcXa2+CxfsgDgefAfLyhSeMcN9v41r\n1iRef5xzIlXgeiW1HgIA+IccT0wNTo5iMCiVlkpr1gwGNd58wuXLpVDIn+qu06cPFiuKRqWNG+3t\nq3PmSPEqlJMufzFdcSRMCd4czYkOCLJZ3fQXNTWKtbersKREdY2NGujp0SfNzW6xHUl26yHLSsj3\nrI9G9cDGjdqxcqU+2rpVVbfcoiXr1yeMzZs/agIBfeX997X/+9/Xe88+627nrV++XA9s2pTwPjve\neUex9nZ7a6wxQ3qASlLRjBn60ltvuVVgty5b5vYg9eaOhmtqEl5f1dCgZdu361f19erz5IRWzp+v\n41u3uq8tLClR/Re/qAsff5wyd3Z3U5POxvNqvxR/bqyfU3J+J4EmACAVcjxx5XC2kDY324Gl94cj\n7yrfmjX+rbTddpv934YGKV6ZUpGIdPvtg9cb6wrDcDmqfmClcUrLZS9Sv7b5pspJ/PL+/SqdOVOP\nHjqkB7ds0f3r16ts9myZ+NbVwtLSwZzPeNBZvXChQuXlerGxUe/98peKnT6tE83N+vWcOQn5r95q\nslZfn15/8kl7BdPzC9OBeF6lUwCpddcuxdrbVRAKyerrSxl0StJVd9+tA08/rYttbXr1scd05sCB\nhGtJ9kpo1S23SBq6zbgwni9aWFKiqxYtUk9Xl4pqa7Vsxw73flw8edLNnX3h9tsT7lvnkSNq27tX\nlz15tePN3QUAIFMEnsg7KfMShstRzNY20vXr7fNu3z60HUqm15voIj+QRK7LcEZbsCiTLZ3ec+9Y\nuVIvNjbq2IYNQwKjafX1+puPP3ZX7F6JRtXT2ekWIwqWlrrnrJw/X/XRqB7atk3nW1rs1UxPxdue\n9nY9W1en3y5apJeXLtXiZ56xV0vjBnp7E4JRSTr7xz+6Y3MLIBUWaqCnJyEnM1hRoaIZM/SupOC0\nabrnJz9JCPRStXW56p57VFpXZ/cNNUb9nmO8AffFkyfdIPKdn/7UvR/OWANlZYqdPq3jW7dq3bx5\ninV2ZtTSJlkuf5mBQXyPgt+YU8gF+nhiavD2vkz+ASlbPSzTnXc816PaKyaZ0fZpvG/t2jFv803o\nQVldrZizRV3pA6NUPSiXbNig1598UjJGjatXu9d3giynb6Zj4NIlt13KvlWrFKqocAPIglBIjatX\n6xc1NVJ8VfLC8eO6cPy4+3pv65SqhgZdbm9X78WLqm5oUO/581Jbm3rPn9e+VasSAr3zH3yg2Jkz\ng9uCJX28dav794FYTCeam/X83Ln663ffdQNuSeqK9+wNlpfrTk/PXue+X+7o0InmZknSpdZW7Wlq\nSvmZZPI5AQDgB3I8gVxK7p/pfM3vHMyJ7tOJKSObOX7ec4cjEX3S3Dykt2RyTuKG+fN14fhxBadN\nU+3ixWl7VUqDOa8N3/2utj74oPp6etTjCW5DlZX66rFjal6xwr32su3bte+pp3Rs/fohFWZDkYiu\nvvdet4CPE8C9Eo26wXBxba0utbYqXF0tU1CggZ4eFYRC+lK84NGLixbpC1u26KX77ksItJM5PUwd\nv120yA2Wk59z3qvTB9Tbb7Nl82b1x2Kqvu22IfmtAAD4ZTQ5ngSeQC55Cx9lsxDRRF0HEy6bRX2k\n7BYs8p5bUsrreIv/XL9ihbpPnHAL9JTNnq2yWbNG/d5jnZ16ft48XW5ttStPO/82FRTomr/6K3dL\nqfeajlAkokcOHkwo3iPZ9//Yhg3q6eiQCQQUKClRYVGRps2erdP79rnHeYNF72tchYVupVonAPa+\nn9H8AiD5s0p+H6kCVgAA/EBxIUxJ485LyHYBn7EYaWutX2NlC++w8jnXxc/enamMJ8dvpPxQ77nT\nXSc5JzEUb3VSvXChSurqRvXedzc16dmrr9avr79elXPnKlxVZQd5AwP2n74+ffLqq0OuWdXQoGuX\nLlV9NKqvfvCBDjz99JD303nkiBtAWn196u3q0tttbeqOt1iR7DYn3m3D3tdI0jVLluirR4+qfvly\n1UejQ4JOyd4iO232bBWGw3r1sccU6+wccn+T76E3V7WwtFSxjo5h83QxeeXz9yhMTswp5AKBJ648\nk6mAz0iFiPwa60T36byCjLb4Trb42bvTT94KsOMJipOrqnofe4PQ4d5755EjutTaqp6ODp3ctUsF\nTj9fr4GBIX0ql23frge3bNEDGzcqHIkkBPnP3XijXl661D2X0/tTkspvuEHRN99UWX29QlVVKqqu\ndu/Ji42Nat2zJ+HS4UhE0+rr9cCmTe61vMe/vHSpJKl01iyd2rvXvZ8j/dLhvrVrVR+NKlxVpf4L\nF/RJc3NWfjkBAMBosNUWV56lS+1AbuHCkQOxXOdGjmWsyInkraATvZUxl707h+O9L04uZfL4xrtN\neLTv3dmmKtmrmJ/fuFH7Vq3S+Y8+GtwOW1CgqxcvVll9vY6tX6/+S5dkAgFd9bnP6YFNmxSORNzz\nePuHmkBA4enT9dC2bdr//e8nFDjy3oPi2lpZlqXLp04ljM0Eg/paW1vK8SfPrZ7ubvf6M+66S5J0\norl5xPxbenECALKNHE8glc7O0RfwyXVu5FjGipzgh/rUnPsSqqzUIwcODMmNlMYftI82cI11dmrn\n448PqXob6+zUr2+4QT1nzrjHhquq7MqzHsW1tXr08GFJGlJB1nvMzM9/XudbWhQoKVFxTY1aNm9O\n2FJrgkFZ8Z6gjof37NHVixalHHfy3JKk52680e0bWh+NqjAYHDHwnqy/nAAATB3keGJKGndegtPu\nZDQ/gOU6N3IsY0XGxjOnkreCwlZcU6NwTY2qb7tNoYqKlMckbxMe67bl0ea3hiMRFc+YobY339Sz\nV12l1ZGIXlqyRJI04447Eo41hYVDXn+ptVXPz5snSbp/3TotWb9exbW1Q475aOtWte7apVe3btVH\nW7cmBJ3B8nIVTZ8ev8jgv8vv/PSnacedPLfCkYhqFi6UZN+zxtWrR5V/Sy/O/Ec+HvzGnEIuZBx4\nGmN+ZIw5bIx52xjzgjEm9U8WQD4jNxIj4If61M63tCh2+rRODJNXmBxYjbVQ0ljyW508z4GeHvWe\nO+eO6761a1VQVGSfb9o0PbRtm0qvvVYmGJQCg62uL8d7Y0r2Z/7o4cOqX75cRTNmuGOouuUWSVLF\njTeq0MkjLbD/me3t6tKltjb7a/FdQN5xpwq6U80tftEBAMhXGW+1NcYskfSqZVkDxpgfSpJlWd9O\ncRxbbQHgCpPJFuSxvmbHypX66OWXVb1ggUrq6txtrqm23XrzPCUpNH26KufNU7C8XKd//3u3p2Z9\nNKpYR4e7BbggHNZALJZ2TOlawmxdtsxt+5JKSV2dVrzzjnu+Z6++WpdaWyXZ231r7rgjK+1xAADI\nhtFstQ0M9+RwLMva5nm4T9IjmZ4LADC13Ld27ajyCr15moufeUb7Vq1yXzNSDuf5lhbF2tv1SXOz\nwjU1bu7jnqYm3b9u3ZBzv/7Nb2qgp0cFwaAut7frlBMYera+DvT0JKyklt9wg33+NO/BWZV0OH93\nKu4Wlpaq/8KFhNdMX7BAVTffrFeiUfe99cdi7vOxM2fcVV/6bgIApgq/cjz/VtLLPp0LGNao8hIm\nU69OTHrkuvhvtFuQvdtr961alfCakbbenj96VJIUrKjQ9JtukpS4fbVl82b39a9/85t6YONGuz3K\npk1u+5NwdXVC4FkQDCZsZ7148qQb3I62FcnOnTvdc9TefXfCc6UzZ+rhHTt0vqUl4b1V33abJCkw\nTIuYXLfuSWeyjmsq4XsU/MacQi4Mu+JpjNkmqTbFU/9gWdbm+DHfkdRjWdbadOd5/PHHdd1110mS\nIpGIFixYoMbGRkmDE5/HPB7t44MHD458fLz/5U5JikbVGP/6ZBg/jyffY8dkGc+V9PjQpUuaLjvQ\nGvja17Rz5073+UOXLum0pM/Fg7Dk138Qiajj+HHNPXdOoUhE5++9V9d961tu4PpOd7d6Jc2VdHL3\nbv3wzjt16/e+p88vW6b71q7Vv0WjajtzRjPi22yPlpbquq9/3Q2aveMLlJXp90ePauCll1T04ovq\nPHJEhy5dcs/nfX+SHXgHnnhCA93dKvrzn3W5tVWtN96ou378Y/u5khK9KzsfdGU8wPy3aFTz/u7v\nFHrhBS3+2c/0xsGDCe93z1tv6ezbb2uu7FXdwBNP5Pzzk6Su+C8I3pV0OBrV3/P9lsc8nvSPDyZ9\nf8n1eHicf48PHjyozvgvGz/88EONxrjaqRhjHpf0XyXdZ1nW5TTHkOOJiUf/y7HJdb9SXLGGa/Ux\nUhuQkXJCX1qyRCeamxO2u16/YoVC8UJGgZISDfT26kRzs0KVlZr5wAO6ePJkwtbeWGennpszx80B\nLZs9WxeOH3fbotQvX64HNm3S85/+tC62tqogGNSX9+9PaB+T6n1k0uJksrbumazjAgBMnKz28TTG\nfEHSjyXda1lW+zDHEXhi4tH/cmwaG3PbrxTwGEt/zuGCN+f5WEeHPmluVqCsTDPuukv9ly65+Z3e\nXpivRKMp+4p6A6uCcDihaFB9NKoHNm7U6khEvefOSbK30/7Nxx/7ek9G835zZbKOCwAwcbIdeL4n\nKSTpbPxLb1iW9USK4wg84audnq148MkVvkLMnJpcXmxsTBkAjiRdwBrr7NRzN97oFh8qrq3VpdbW\nISt06VbuvIHVq4895lbHnX7zzXp41y6FIxH9oqZGsfZ2FZaU6Oqf/1xLv/KVMY9zrMeM5/zIL3yP\ngt+YU/DbaALPgkxPblnWHMuy6i3Laoj/GRJ0AsgT9CvFJDKW/pxe6YoRhSMR1Sxc6J4z+uabKXth\npuuR6S2UdN/atapfvlz10agbdO5uatK0T31KBeGwom+8oZLaVKURRh7nWI8Zz/kBAJho48rxHNUF\nWPEEAIxBpls3h8s1zOZ20LGu0CaPc99TTw1ZoRxP3iQ5lwCAiZbVFU8AALJhtK1YkqVbsRzPOVNJ\nbh8y1hXa5HGmWqEc7r2MpLimRuHqagJOAMCkQuCJvOOUdAb8wpyaGvwMLoeTHCgmB4kjzafkcaYK\nXMfzXs63tIy59ygmN75HwW/MKeQCgScAAGOQHCiON+Adz+rmaMYHAMBkQI4nAABjMNnbh0z28QEA\npp6stlMZwyAIPAEAAABgiqK4EKYk8hLgN+YU/MR8gt+YU/Abcwq5QOAJAAAAAMgqttoCAAAAADLG\nVlsAAAAAQM4ReCLvkJcAvzGn4CfmE/zGnILfmFPIBQJPAAAAAEBWkeMJAAAAAMgYOZ4AAAAAgJwj\n8ETeIS8BfmNOwU/MJ/iNOQW/MaeQCwSeAAAAAICsIscTAAAAAJAxcjwBAAAAADlH4Im8Q14C/Mac\ngp+YT/Abcwp+Y04hFwg8AQAAAABZRY4nAAAAACBj5HgCAAAAAHKOwBN5h7wE+I05BT8xn+A35hT8\nxpxCLhB4AgAAAACyihxPAAAAAEDGyPEEAAAAAOQcgSfyDnkJ8BtzCn5iPsFvzCn4jTmFXCDwBAAA\nAABkFTmeAAAAAICMkeMJAAAAAMg5Ak/kHfIS4DfmFPzEfILfmFPwG3MKuUDgCQAAAADIKnI8AQAA\nAAAZI8cTAAAAAJBzBJ7IO+QlwG/MKfiJ+QS/MafgN+YUcoHAEwAAAACQVeR4AgAAAAAyRo4nAAAA\nACDnCDyRd8hLgN+YU/AT8wl+Y07Bb8wp5AKBJwAAAAAgq8jxBAAAAABkjBxPAAAAAEDOEXgi75CX\nAL8xp+An5hP8xpyC35hTyAUCTwAAAABAVpHjCQAAAADIGDmeAAAAAICcI/BE3iEvAX5jTsFPzCf4\njTkFvzGnkAsEngAAAACArCLHEwAAAACQMXI8AQAAAAA5R+CJvENeAvzGnIKfmE/wG3MKfmNOIRcI\nPAEAAAAAWUWOJwAAAAAgY+R4AgAAAAByjsATeYe8BPiNOQU/MZ/gN+YU/MacQi4QeAIAAAAAsooc\nTwAAAABAxsjxBAAAAADkHIEn8g55CfAbcwp+Yj7Bb8wp+I05hVwg8AQAAAAAZBU5ngAAAACAjJHj\nCQAAAADIOQJP5B3yEuA35hT8xHyC35hT8BtzCrlA4AkAAAAAyCpyPAEAAAAAGSPHEwAAAACQcwSe\nyDvkJcBvzCn4ifkEvzGn4DfmFHKBwBMAAAAAkFXkeAIAAAAAMkaOJwAAAAAg5wg8kXfIS4DfmFPw\nE/MJfmNOwW/MKeQCgScAAAAAIKvI8QQAAAAAZIwcTwAAAABAzmUceBpj/skY87Yx5qAx5lVjzLV+\nDgxIh7wE+I05BT8xn+A35hT8xpxCLoxnxfOfLcu6xbKsBZI2SfpfPo0JGNbBgwdzPQRMMcwp+In5\nBL8xp+A35hRyIePA07Ks856HZZLaxz8cYGSdnZ25HgKmGOYU/MR8gt+YU/Abcwq5EBjPi40xT0v6\nz5IuSrrLlxEBAAAAAKaUYVc8jTHbjDF/SvHnYUmyLOs7lmXNkrRG0r9MwHgBffjhh7keAqYY5hT8\nxHyC35hT8BtzCrngSzsVY8wsSS9blvXZFM/RSwUAAAAAprCR2qlkvNXWGDPHsqz34g+XSzqQyQAA\nAAAAAFNbxiuexpgNkuZK6pd0VNI3LMtq83FsAAAAAIApwJettgAAAAAApDOePp6jZoz5J2PM28aY\ng8aYV40x107EdTE1GWN+ZIw5HJ9TLxhjKnI9JuQ3Y8wKY8w7xph+Y8ytuR4P8pcx5gvGmL8YY94z\nxvyPXI8H+c0Y83+NMaeMMX/K9VgwNRhjrjXG7Ij/m/dnY8x/z/WYkL+MMUXGmH3xGO+QMeZ/D3v8\nRKx4GmOmOX0/jTH/TdItlmV9PesXxpRkjFki6VXLsgaMMT+UJMuyvp3jYSGPGWM+LWlA0v+R9PeW\nZf0hx0NCHjLGFEp6V9L9kj6R9HtJX7Us63BOB4a8ZYxZLKlb0r9bljU/1+NB/jPG1EqqtSzroDGm\nTNL/kxTl+xQyZYwpsSzrojEm47bo4wAAAphJREFUIOk1Sd+yLOu1VMdOyIqnE3TGlUlqn4jrYmqy\nLGubZVkD8Yf7JM3M5XiQ/yzL+otlWUdyPQ7kvTskvW9Z1oeWZfVKek528T0gI5Zl7ZHUketxYOqw\nLKvVsqyD8b93SzosqS63o0I+syzrYvyvIUmFks6mO3ZCAk9JMsY8bYz5SNJKST+cqOtiyvtbSS/n\nehAAIOkaSR97Hh+Pfw0AJh1jzHWSGmT/Eh/IiDGmwBhzUNIpSTssyzqU7tiM26mkuOg2SbUpnvoH\ny7I2W5b1HUnfMcZ8W9K/SPovfl0bU89I8yl+zHck9ViWtXZCB4e8NJo5BYwT1foA5IX4NtsNkp6M\nr3wCGYnvQlwQr7nyijGm0bKsnamO9S3wtCxrySgPXStWqDCCkeaTMeZxSUsl3TchA0LeG8P3KCBT\nn0jyFs+7VvaqJwBMGsaYoKTfSPqlZVmbcj0eTA2WZZ0zxmyRtFDSzlTHTFRV2zmeh8slHZiI62Jq\nMsZ8QdIqScsty7qc6/FgyjG5HgDy1n5Jc4wx1xljQpL+WtKLOR4TALiMMUbSzyUdsizrX3M9HuQ3\nY0y1MSYS/3uxpCUaJs6bqKq2GyTNldQv6aikb1iW1Zb1C2NKMsa8JzuB2UlefsOyrCdyOCTkOWPM\nlyT9VFK1pHOSDliW9WBuR4V8ZIx5UNK/yi6w8HPLsoYtLQ8Mxxjza0n3SqqS1Cbp+5Zlrc7tqJDP\njDGLJO2W9EcNpgf8T8uyfpe7USFfGWPmS/qF7MXMAknPWpb1o7THT0TgCQAAAAC4ck1YVVsAAAAA\nwJWJwBMAAAAAkFUEngAAAACArCLwBAAAAABkFYEnAAAAACCrCDwBAAAAAFlF4AkAAAAAyCoCTwAA\nAABAVv1/lzHCzGUnjVoAAAAASUVORK5CYII=\n", + "text": [ + "" + ] + } + ], + "prompt_number": 5 + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/examples/triplet/mnist_triplet.prototxt b/examples/triplet/mnist_triplet.prototxt new file mode 100644 index 00000000000..0e903f85909 --- /dev/null +++ b/examples/triplet/mnist_triplet.prototxt @@ -0,0 +1,113 @@ +name: "mnist_siamese" +input: "data" +input_dim: 10000 +input_dim: 1 +input_dim: 28 +input_dim: 28 +layer { + name: "conv1" + type: "Convolution" + bottom: "data" + top: "conv1" + param { + lr_mult: 1 + } + param { + lr_mult: 2 + } + convolution_param { + num_output: 20 + kernel_size: 5 + stride: 1 + } +} +layer { + name: "pool1" + type: "Pooling" + bottom: "conv1" + top: "pool1" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "conv2" + type: "Convolution" + bottom: "pool1" + top: "conv2" + param { + lr_mult: 1 + } + param { + lr_mult: 2 + } + convolution_param { + num_output: 50 + kernel_size: 5 + stride: 1 + } +} +layer { + name: "pool2" + type: "Pooling" + bottom: "conv2" + top: "pool2" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "ip1" + type: "InnerProduct" + bottom: "pool2" + top: "ip1" + param { + lr_mult: 1 + } + param { + lr_mult: 2 + } + inner_product_param { + num_output: 500 + } +} +layer { + name: "relu1" + type: "ReLU" + bottom: "ip1" + top: "ip1" +} +layer { + name: "ip2" + type: "InnerProduct" + bottom: "ip1" + top: "ip2" + param { + lr_mult: 1 + } + param { + lr_mult: 2 + } + inner_product_param { + num_output: 10 + } +} +layer { + name: "feat" + type: "InnerProduct" + bottom: "ip2" + top: "feat" + param { + lr_mult: 1 + } + param { + lr_mult: 2 + } + inner_product_param { + num_output: 2 + } +} diff --git a/examples/triplet/mnist_triplet_solver.prototxt b/examples/triplet/mnist_triplet_solver.prototxt new file mode 100644 index 00000000000..39222b89cf0 --- /dev/null +++ b/examples/triplet/mnist_triplet_solver.prototxt @@ -0,0 +1,25 @@ +# The train/test net protocol buffer definition +net: "examples/triplet/mnist_triplet_train_test.prototxt" +# test_iter specifies how many forward passes the test should carry out. +# In the case of MNIST, we have test batch size 100 and 100 test iterations, +# covering the full 10,000 testing images. +test_iter: 100 +# Carry out testing every 500 training iterations. +test_interval: 500 +# The base learning rate, momentum and the weight decay of the network. +base_lr: 0.01 +momentum: 0.9 +weight_decay: 0.0000 +# The learning rate policy +lr_policy: "inv" +gamma: 0.0001 +power: 0.75 +# Display every 100 iterations +display: 100 +# The maximum number of iterations +max_iter: 50000 +# snapshot intermediate results +snapshot: 5000 +snapshot_prefix: "examples/triplet/mnist_triplet" +# solver mode: CPU or GPU +solver_mode: CPU diff --git a/examples/triplet/mnist_triplet_train_test.prototxt b/examples/triplet/mnist_triplet_train_test.prototxt new file mode 100644 index 00000000000..3cea2376c11 --- /dev/null +++ b/examples/triplet/mnist_triplet_train_test.prototxt @@ -0,0 +1,498 @@ +name: "mnist_triplet_train_test" +layer { + name: "triplet_data" + type: "Data" + top: "triplet_data" + top: "sim" + include { + phase: TRAIN + } + transform_param { + scale: 0.00390625 + } + data_param { + source: "examples/triplet/mnist_triplet_train_leveldb" + batch_size: 64 + } +} +layer { + name: "triplet_data" + type: "Data" + top: "triplet_data" + top: "sim" + include { + phase: TEST + } + transform_param { + scale: 0.00390625 + } + data_param { + source: "examples/triplet/mnist_triplet_test_leveldb" + batch_size: 100 + } +} +layer { + name: "slice_triplet" + type: "Slice" + bottom: "triplet_data" + top: "data" + top: "data_true" + top: "data_false" + slice_param { + slice_dim: 1 + slice_point: 1 + slice_point: 2 + } +} +layer { + name: "conv1" + type: "Convolution" + bottom: "data" + top: "conv1" + param { + name: "conv1_w" + lr_mult: 1 + } + param { + name: "conv1_b" + lr_mult: 2 + } + convolution_param { + num_output: 20 + kernel_size: 5 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "pool1" + type: "Pooling" + bottom: "conv1" + top: "pool1" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "conv2" + type: "Convolution" + bottom: "pool1" + top: "conv2" + param { + name: "conv2_w" + lr_mult: 1 + } + param { + name: "conv2_b" + lr_mult: 2 + } + convolution_param { + num_output: 50 + kernel_size: 5 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "pool2" + type: "Pooling" + bottom: "conv2" + top: "pool2" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "ip1" + type: "InnerProduct" + bottom: "pool2" + top: "ip1" + param { + name: "ip1_w" + lr_mult: 1 + } + param { + name: "ip1_b" + lr_mult: 2 + } + inner_product_param { + num_output: 500 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "relu1" + type: "ReLU" + bottom: "ip1" + top: "ip1" +} +layer { + name: "ip2" + type: "InnerProduct" + bottom: "ip1" + top: "ip2" + param { + name: "ip2_w" + lr_mult: 1 + } + param { + name: "ip2_b" + lr_mult: 2 + } + inner_product_param { + num_output: 10 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "feat" + type: "InnerProduct" + bottom: "ip2" + top: "feat" + param { + name: "feat_w" + lr_mult: 1 + } + param { + name: "feat_b" + lr_mult: 2 + } + inner_product_param { + num_output: 2 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "conv1_true" + type: "Convolution" + bottom: "data_true" + top: "conv1_true" + param { + name: "conv1_w" + lr_mult: 1 + } + param { + name: "conv1_b" + lr_mult: 2 + } + convolution_param { + num_output: 20 + kernel_size: 5 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "pool1_true" + type: "Pooling" + bottom: "conv1_true" + top: "pool1_true" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "conv2_true" + type: "Convolution" + bottom: "pool1_true" + top: "conv2_true" + param { + name: "conv2_w" + lr_mult: 1 + } + param { + name: "conv2_b" + lr_mult: 2 + } + convolution_param { + num_output: 50 + kernel_size: 5 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "pool2_true" + type: "Pooling" + bottom: "conv2_true" + top: "pool2_true" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "ip1_true" + type: "InnerProduct" + bottom: "pool2_true" + top: "ip1_true" + param { + name: "ip1_w" + lr_mult: 1 + } + param { + name: "ip1_b" + lr_mult: 2 + } + inner_product_param { + num_output: 500 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "relu1_true" + type: "ReLU" + bottom: "ip1_true" + top: "ip1_true" +} +layer { + name: "ip2_true" + type: "InnerProduct" + bottom: "ip1_true" + top: "ip2_true" + param { + name: "ip2_w" + lr_mult: 1 + } + param { + name: "ip2_b" + lr_mult: 2 + } + inner_product_param { + num_output: 10 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "feat_true" + type: "InnerProduct" + bottom: "ip2_true" + top: "feat_true" + param { + name: "feat_w" + lr_mult: 1 + } + param { + name: "feat_b" + lr_mult: 2 + } + inner_product_param { + num_output: 2 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "conv1_false" + type: "Convolution" + bottom: "data_false" + top: "conv1_false" + param { + name: "conv1_w" + lr_mult: 1 + } + param { + name: "conv1_b" + lr_mult: 2 + } + convolution_param { + num_output: 20 + kernel_size: 5 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "pool1_false" + type: "Pooling" + bottom: "conv1_false" + top: "pool1_false" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "conv2_false" + type: "Convolution" + bottom: "pool1_false" + top: "conv2_false" + param { + name: "conv2_w" + lr_mult: 1 + } + param { + name: "conv2_b" + lr_mult: 2 + } + convolution_param { + num_output: 50 + kernel_size: 5 + stride: 1 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "pool2_false" + type: "Pooling" + bottom: "conv2_false" + top: "pool2_false" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "ip1_false" + type: "InnerProduct" + bottom: "pool2_false" + top: "ip1_false" + param { + name: "ip1_w" + lr_mult: 1 + } + param { + name: "ip1_b" + lr_mult: 2 + } + inner_product_param { + num_output: 500 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "relu1_false" + type: "ReLU" + bottom: "ip1_false" + top: "ip1_false" +} +layer { + name: "ip2_false" + type: "InnerProduct" + bottom: "ip1_false" + top: "ip2_false" + param { + name: "ip2_w" + lr_mult: 1 + } + param { + name: "ip2_b" + lr_mult: 2 + } + inner_product_param { + num_output: 10 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "feat_false" + type: "InnerProduct" + bottom: "ip2_false" + top: "feat_false" + param { + name: "feat_w" + lr_mult: 1 + } + param { + name: "feat_b" + lr_mult: 2 + } + inner_product_param { + num_output: 2 + weight_filler { + type: "xavier" + } + bias_filler { + type: "constant" + } + } +} +layer { + name: "loss" + type: "ContrastiveLoss" + bottom: "feat" + bottom: "feat_false" + bottom: "sim" + top: "loss" + contrastive_loss_param { + margin: 1 + } +} diff --git a/examples/triplet/readme.md b/examples/triplet/readme.md new file mode 100644 index 00000000000..ce98ec10819 --- /dev/null +++ b/examples/triplet/readme.md @@ -0,0 +1,179 @@ +--- +title: Siamese Network Tutorial +description: Train and test a siamese network on MNIST data. +category: example +include_in_docs: true +layout: default +priority: 100 +--- + +# Siamese Network Training with Caffe +This example shows how you can use weight sharing and a contrastive loss +function to learn a model using a siamese network in Caffe. + +We will assume that you have caffe successfully compiled. If not, please refer +to the [Installation page](../../installation.html). This example builds on the +[MNIST tutorial](mnist.html) so it would be a good idea to read that before +continuing. + +*The guide specifies all paths and assumes all commands are executed from the +root caffe directory* + +## Prepare Datasets + +You will first need to download and convert the data from the MNIST +website. To do this, simply run the following commands: + + ./data/mnist/get_mnist.sh + ./examples/siamese/create_mnist_siamese.sh + +After running the script there should be two datasets, +`./examples/siamese/mnist_siamese_train_leveldb`, and +`./examples/siamese/mnist_siamese_test_leveldb`. + +## The Model +First, we will define the model that we want to train using the siamese network. +We will use the convolutional net defined in +`./examples/siamese/mnist_siamese.prototxt`. This model is almost +exactly the same as the [LeNet model](mnist.html), the only difference is that +we have replaced the top layers that produced probabilities over the 10 digit +classes with a linear "feature" layer that produces a 2 dimensional vector. + + layers { + name: "feat" + type: INNER_PRODUCT + bottom: "ip2" + top: "feat" + blobs_lr: 1 + blobs_lr: 2 + inner_product_param { + num_output: 2 + } + } + +## Define the Siamese Network + +In this section we will define the siamese network used for training. The +resulting network is defined in +`./examples/siamese/mnist_siamese_train_test.prototxt`. + +### Reading in the Pair Data + +We start with a data layer that reads from the LevelDB database we created +earlier. Each entry in this database contains the image data for a pair of +images (`pair_data`) and a binary label saying if they belong to the same class +or different classes (`sim`). + + layers { + name: "pair_data" + type: DATA + top: "pair_data" + top: "sim" + data_param { + source: "examples/siamese/mnist-siamese-train-leveldb" + scale: 0.00390625 + batch_size: 64 + } + include: { phase: TRAIN } + } + +In order to pack a pair of images into the same blob in the database we pack one +image per channel. We want to be able to work with these two images separately, +so we add a slice layer after the data layer. This takes the `pair_data` and +slices it along the channel dimension so that we have a single image in `data` +and its paired image in `data_p.` + + layers { + name: "slice_pair" + type: SLICE + bottom: "pair_data" + top: "data" + top: "data_p" + slice_param { + slice_dim: 1 + slice_point: 1 + } + } + +### Building the First Side of the Siamese Net + +Now we can specify the first side of the siamese net. This side operates on +`data` and produces `feat`. Starting from the net in +`./examples/siamese/mnist_siamese.prototxt` we add default weight fillers. Then +we name the parameters of the convolutional and inner product layers. Naming the +parameters allows Caffe to share the parameters between layers on both sides of +the siamese net. In the definition this looks like: + + ... + param: "conv1_w" + param: "conv1_b" + ... + param: "conv2_w" + param: "conv2_b" + ... + param: "ip1_w" + param: "ip1_b" + ... + param: "ip2_w" + param: "ip2_b" + ... + +### Building the Second Side of the Siamese Net + +Now we need to create the second path that operates on `data_p` and produces +`feat_p`. This path is exactly the same as the first. So we can just copy and +paste it. Then we change the name of each layer, input, and output by appending +`_p` to differentiate the "paired" layers from the originals. + +### Adding the Contrastive Loss Function + +To train the network we will optimize a contrastive loss function proposed in: +Raia Hadsell, Sumit Chopra, and Yann LeCun "Dimensionality Reduction by Learning +an Invariant Mapping". This loss function encourages matching pairs to be close +together in feature space while pushing non-matching pairs apart. This cost +function is implemented with the `CONTRASTIVE_LOSS` layer: + + layers { + name: "loss" + type: CONTRASTIVE_LOSS + contrastive_loss_param { + margin: 1.0 + } + bottom: "feat" + bottom: "feat_p" + bottom: "sim" + top: "loss" + } + +## Define the Solver + +Nothing special needs to be done to the solver besides pointing it at the +correct model file. The solver is defined in +`./examples/siamese/mnist_siamese_solver.prototxt`. + +## Training and Testing the Model + +Training the model is simple after you have written the network definition +protobuf and solver protobuf files. Simply run +`./examples/siamese/train_mnist_siamese.sh`: + + ./examples/siamese/train_mnist_siamese.sh + +# Plotting the results + +First, we can draw the model and siamese networks by running the following +commands that draw the DAGs defined in the .prototxt files: + + ./python/draw_net.py \ + ./examples/siamese/mnist_siamese.prototxt \ + ./examples/siamese/mnist_siamese.png + + ./python/draw_net.py \ + ./examples/siamese/mnist_siamese_train_test.prototxt \ + ./examples/siamese/mnist_siamese_train_test.png + +Second, we can load the learned model and plot the features using the iPython +notebook: + + ipython notebook ./examples/siamese/mnist_siamese.ipynb + diff --git a/examples/triplet/train_mnist_triplet.sh b/examples/triplet/train_mnist_triplet.sh new file mode 100755 index 00000000000..e005970824a --- /dev/null +++ b/examples/triplet/train_mnist_triplet.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +TOOLS=./build/tools + +$TOOLS/caffe train --solver=examples/triplet/mnist_triplet_solver.prototxt diff --git a/include/caffe/data_layers.hpp b/include/caffe/data_layers.hpp index 3958cb7ecb0..2bb9d948169 100644 --- a/include/caffe/data_layers.hpp +++ b/include/caffe/data_layers.hpp @@ -14,6 +14,7 @@ #include "caffe/filler.hpp" #include "caffe/internal_thread.hpp" #include "caffe/layer.hpp" +#include "caffe/net.hpp" #include "caffe/proto/caffe.pb.h" #include "caffe/util/db.hpp" @@ -28,6 +29,7 @@ template class BaseDataLayer : public Layer { public: explicit BaseDataLayer(const LayerParameter& param); + virtual ~BaseDataLayer() {} // LayerSetUp: implements common data layer setup functionality, and calls // DataLayerSetUp to do special data layer setup for individual layer types. // This method may not be overridden except by the BasePrefetchingDataLayer. @@ -56,6 +58,7 @@ class BasePrefetchingDataLayer : public: explicit BasePrefetchingDataLayer(const LayerParameter& param) : BaseDataLayer(param) {} + virtual ~BasePrefetchingDataLayer() {} // LayerSetUp: implements common data layer setup functionality, and calls // DataLayerSetUp to do special data layer setup for individual layer types. // This method may not be overridden. diff --git a/include/caffe/data_transformer.hpp b/include/caffe/data_transformer.hpp index 0ad68c80216..880356601a4 100644 --- a/include/caffe/data_transformer.hpp +++ b/include/caffe/data_transformer.hpp @@ -62,7 +62,6 @@ class DataTransformer { */ void Transform(const vector & mat_vector, Blob* transformed_blob); - /** * @brief Applies the transformation defined in the data layer's * transform_param block to a cv::Mat @@ -88,41 +87,6 @@ class DataTransformer { */ void Transform(Blob* input_blob, Blob* transformed_blob); - /** - * @brief Infers the shape of transformed_blob will have when - * the transformation is applied to the data. - * - * @param datum - * Datum containing the data to be transformed. - */ - vector InferBlobShape(const Datum& datum); - /** - * @brief Infers the shape of transformed_blob will have when - * the transformation is applied to the data. - * It uses the first element to infer the shape of the blob. - * - * @param datum_vector - * A vector of Datum containing the data to be transformed. - */ - vector InferBlobShape(const vector & datum_vector); - /** - * @brief Infers the shape of transformed_blob will have when - * the transformation is applied to the data. - * It uses the first element to infer the shape of the blob. - * - * @param mat_vector - * A vector of Mat containing the data to be transformed. - */ - vector InferBlobShape(const vector & mat_vector); - /** - * @brief Infers the shape of transformed_blob will have when - * the transformation is applied to the data. - * - * @param cv_img - * cv::Mat containing the data to be transformed. - */ - vector InferBlobShape(const cv::Mat& cv_img); - protected: /** * @brief Generates a random integer from Uniform({0, 1, ..., n-1}). diff --git a/include/caffe/filler.hpp b/include/caffe/filler.hpp index ff3542e1f99..bb18e8e1e28 100644 --- a/include/caffe/filler.hpp +++ b/include/caffe/filler.hpp @@ -126,18 +126,17 @@ class PositiveUnitballFiller : public Filler { }; /** - * @brief Fills a Blob with values @f$ x \sim U(-a, +a) @f$ where @f$ a @f$ is - * set inversely proportional to number of incoming nodes, outgoing - * nodes, or their average. + * @brief Fills a Blob with values @f$ x \sim U(-a, +a) @f$ where @f$ a @f$ + * is set inversely proportional to the number of incoming nodes. * * A Filler based on the paper [Bengio and Glorot 2010]: Understanding - * the difficulty of training deep feedforward neuralnetworks. + * the difficulty of training deep feedforward neuralnetworks, but does not + * use the fan_out value. * - * It fills the incoming matrix by randomly sampling uniform data from [-scale, - * scale] where scale = sqrt(3 / n) where n is the fan_in, fan_out, or their - * average, depending on the variance_norm option. You should make sure the - * input blob has shape (num, a, b, c) where a * b * c = fan_in and num * b * c - * = fan_out. Note that this is currently not the case for inner product layers. + * It fills the incoming matrix by randomly sampling uniform data from + * [-scale, scale] where scale = sqrt(3 / fan_in) where fan_in is the number + * of input nodes. You should make sure the input blob has shape (num, a, b, c) + * where a * b * c = fan_in. * * TODO(dox): make notation in above comment consistent with rest & use LaTeX. */ @@ -149,16 +148,7 @@ class XavierFiller : public Filler { virtual void Fill(Blob* blob) { CHECK(blob->count()); int fan_in = blob->count() / blob->num(); - int fan_out = blob->count() / blob->channels(); - Dtype n = fan_in; // default to fan_in - if (this->filler_param_.variance_norm() == - FillerParameter_VarianceNorm_AVERAGE) { - n = (fan_in + fan_out) / Dtype(2); - } else if (this->filler_param_.variance_norm() == - FillerParameter_VarianceNorm_FAN_OUT) { - n = fan_out; - } - Dtype scale = sqrt(Dtype(3) / n); + Dtype scale = sqrt(Dtype(3) / fan_in); caffe_rng_uniform(blob->count(), -scale, scale, blob->mutable_cpu_data()); CHECK_EQ(this->filler_param_.sparse(), -1) @@ -166,47 +156,6 @@ class XavierFiller : public Filler { } }; -/** - * @brief Fills a Blob with values @f$ x \sim N(0, \sigma^2) @f$ where - * @f$ \sigma^2 @f$ is set inversely proportional to number of incoming - * nodes, outgoing nodes, or their average. - * - * A Filler based on the paper [He, Zhang, Ren and Sun 2015]: Specifically - * accounts for ReLU nonlinearities. - * - * Aside: for another perspective on the scaling factor, see the derivation of - * [Saxe, McClelland, and Ganguli 2013 (v3)]. - * - * It fills the incoming matrix by randomly sampling Gaussian data with std = - * sqrt(2 / n) where n is the fan_in, fan_out, or their average, depending on - * the variance_norm option. You should make sure the input blob has shape (num, - * a, b, c) where a * b * c = fan_in and num * b * c = fan_out. Note that this - * is currently not the case for inner product layers. - */ -template -class MSRAFiller : public Filler { - public: - explicit MSRAFiller(const FillerParameter& param) - : Filler(param) {} - virtual void Fill(Blob* blob) { - CHECK(blob->count()); - int fan_in = blob->count() / blob->num(); - int fan_out = blob->count() / blob->channels(); - Dtype n = fan_in; // default to fan_in - if (this->filler_param_.variance_norm() == - FillerParameter_VarianceNorm_AVERAGE) { - n = (fan_in + fan_out) / Dtype(2); - } else if (this->filler_param_.variance_norm() == - FillerParameter_VarianceNorm_FAN_OUT) { - n = fan_out; - } - Dtype std = sqrt(Dtype(2) / n); - caffe_rng_gaussian(blob->count(), Dtype(0), std, - blob->mutable_cpu_data()); - CHECK_EQ(this->filler_param_.sparse(), -1) - << "Sparsity not supported by this Filler."; - } -}; /** * @brief Get a specific filler from the specification given in FillerParameter. @@ -227,8 +176,6 @@ Filler* GetFiller(const FillerParameter& param) { return new UniformFiller(param); } else if (type == "xavier") { return new XavierFiller(param); - } else if (type == "msra") { - return new MSRAFiller(param); } else { CHECK(false) << "Unknown filler name: " << param.type(); } diff --git a/include/caffe/layer.hpp b/include/caffe/layer.hpp index 8f924a75755..2d13ef97c05 100644 --- a/include/caffe/layer.hpp +++ b/include/caffe/layer.hpp @@ -406,7 +406,6 @@ template inline Dtype Layer::Forward(const vector*>& bottom, const vector*>& top) { Dtype loss = 0; - Reshape(bottom, top); switch (Caffe::mode()) { case Caffe::CPU: Forward_cpu(bottom, top); diff --git a/include/caffe/loss_layers.hpp b/include/caffe/loss_layers.hpp index 86c34241168..2f9c1f567a1 100644 --- a/include/caffe/loss_layers.hpp +++ b/include/caffe/loss_layers.hpp @@ -210,6 +210,73 @@ class ContrastiveLossLayer : public LossLayer { Blob summer_vec_; // tmp storage for gpu forward pass }; +template +class TripletLossLayer : public LossLayer { + public: + explicit TripletLossLayer(const LayerParameter& param) + : LossLayer(param), diff_() {} + virtual void LayerSetUp(const vector*>& bottom, + const vector*>& top); + + virtual inline int ExactNumBottomBlobs() const { return 4; } + virtual inline const char* type() const { return "TripletLoss"; } + /** + * Unlike most loss layers, in the TripletLossLayer we can backpropagate + * to the first three inputs. + */ + virtual inline bool AllowForceBackward(const int bottom_index) const { + return bottom_index != 3; + } + + protected: + /// @copydoc TripletLossLayer + virtual void Forward_cpu(const vector*>& bottom, + const vector*>& top); + virtual void Forward_gpu(const vector*>& bottom, + const vector*>& top); + + /** + * @brief Computes the Triplet error gradient w.r.t. the inputs. + * + * Computes the gradients with respect to the two input vectors (bottom[0] and + * bottom[1]), but not the similarity label (bottom[2]). + * + * @param top output Blob vector (length 1), providing the error gradient with + * respect to the outputs + * -# @f$ (1 \times 1 \times 1 \times 1) @f$ + * This Blob's diff will simply contain the loss_weight* @f$ \lambda @f$, + * as @f$ \lambda @f$ is the coefficient of this layer's output + * @f$\ell_i@f$ in the overall Net loss + * @f$ E = \lambda_i \ell_i + \mbox{other loss terms}@f$; hence + * @f$ \frac{\partial E}{\partial \ell_i} = \lambda_i @f$. + * (*Assuming that this top Blob is not used as a bottom (input) by any + * other layer of the Net.) + * @param propagate_down see Layer::Backward. + * @param bottom input Blob vector (length 2) + * -# @f$ (N \times C \times 1 \times 1) @f$ + * the features @f$a@f$; Backward fills their diff with + * gradients if propagate_down[0] + * -# @f$ (N \times C \times 1 \times 1) @f$ + * the features @f$b@f$; Backward fills their diff with gradients if + * propagate_down[1] + */ + virtual void Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom); + virtual void Backward_gpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom); + + Blob diff_; // cached for backward pass + Blob diff_pos; + Blob diff_neg; + Blob dist_sq_; // cached for backward pass + Blob dist_sq_pos; + Blob dist_sq_neg; + Blob diff_sq_; // tmp storage for gpu forward pass + Blob diff_sq_pos; + Blob diff_sq_neg; + Blob summer_vec_; // tmp storage for gpu forward pass +}; + /** * @brief Computes the Euclidean (L2) loss @f$ * E = \frac{1}{2N} \sum\limits_{n=1}^N \left| \left| \hat{y}_n - y_n @@ -605,6 +672,8 @@ class SigmoidCrossEntropyLossLayer : public LossLayer { /// @copydoc SigmoidCrossEntropyLossLayer virtual void Forward_cpu(const vector*>& bottom, const vector*>& top); + virtual void Forward_gpu(const vector*>& bottom, + const vector*>& top); /** * @brief Computes the sigmoid cross-entropy loss error gradient w.r.t. the diff --git a/include/caffe/net.hpp b/include/caffe/net.hpp index 5665df1edf2..075afebc9b0 100644 --- a/include/caffe/net.hpp +++ b/include/caffe/net.hpp @@ -137,9 +137,6 @@ class Net { inline const vector& blob_loss_weights() const { return blob_loss_weights_; } - inline const vector& layer_need_backward() const { - return layer_need_backward_; - } /// @brief returns the parameters inline const vector > >& params() const { return params_; diff --git a/include/caffe/neuron_layers.hpp b/include/caffe/neuron_layers.hpp index c2e0774aaa2..323215134c7 100644 --- a/include/caffe/neuron_layers.hpp +++ b/include/caffe/neuron_layers.hpp @@ -8,6 +8,7 @@ #include "caffe/blob.hpp" #include "caffe/common.hpp" #include "caffe/layer.hpp" +#include "caffe/net.hpp" #include "caffe/proto/caffe.pb.h" #define HDF5_DATA_DATASET_NAME "data" @@ -267,72 +268,6 @@ class ExpLayer : public NeuronLayer { Dtype inner_scale_, outer_scale_; }; -/** - * @brief Computes @f$ y = log_{\gamma}(\alpha x + \beta) @f$, - * as specified by the scale @f$ \alpha @f$, shift @f$ \beta @f$, - * and base @f$ \gamma @f$. - */ -template -class LogLayer : public NeuronLayer { - public: - /** - * @param param provides LogParameter log_param, - * with LogLayer options: - * - scale (\b optional, default 1) the scale @f$ \alpha @f$ - * - shift (\b optional, default 0) the shift @f$ \beta @f$ - * - base (\b optional, default -1 for a value of @f$ e \approx 2.718 @f$) - * the base @f$ \gamma @f$ - */ - explicit LogLayer(const LayerParameter& param) - : NeuronLayer(param) {} - virtual void LayerSetUp(const vector*>& bottom, - const vector*>& top); - - virtual inline const char* type() const { return "Log"; } - - protected: - /** - * @param bottom input Blob vector (length 1) - * -# @f$ (N \times C \times H \times W) @f$ - * the inputs @f$ x @f$ - * @param top output Blob vector (length 1) - * -# @f$ (N \times C \times H \times W) @f$ - * the computed outputs @f$ - * y = log_{\gamma}(\alpha x + \beta) - * @f$ - */ - virtual void Forward_cpu(const vector*>& bottom, - const vector*>& top); - virtual void Forward_gpu(const vector*>& bottom, - const vector*>& top); - - /** - * @brief Computes the error gradient w.r.t. the exp inputs. - * - * @param top output Blob vector (length 1), providing the error gradient with - * respect to the outputs - * -# @f$ (N \times C \times H \times W) @f$ - * containing error gradients @f$ \frac{\partial E}{\partial y} @f$ - * with respect to computed outputs @f$ y @f$ - * @param propagate_down see Layer::Backward. - * @param bottom input Blob vector (length 1) - * -# @f$ (N \times C \times H \times W) @f$ - * the inputs @f$ x @f$; Backward fills their diff with - * gradients @f$ - * \frac{\partial E}{\partial x} = - * \frac{\partial E}{\partial y} y \alpha \log_e(gamma) - * @f$ if propagate_down[0] - */ - virtual void Backward_cpu(const vector*>& top, - const vector& propagate_down, const vector*>& bottom); - virtual void Backward_gpu(const vector*>& top, - const vector& propagate_down, const vector*>& bottom); - - Dtype base_scale_; - Dtype input_scale_, input_shift_; - Dtype backward_num_scale_; -}; - /** * @brief Computes @f$ y = (\alpha x + \beta) ^ \gamma @f$, * as specified by the scale @f$ \alpha @f$, shift @f$ \beta @f$, @@ -799,8 +734,7 @@ class PReLULayer : public NeuronLayer { const vector& propagate_down, const vector*>& bottom); bool channel_shared_; - Blob multiplier_; // dot multiplier for backward computation of params - Blob backward_buff_; // temporary buffer for backward computation + Blob multiplier_; // dot multipler for backward computation of params Blob bottom_memory_; // memory for in-place computation }; diff --git a/include/caffe/python_layer.hpp b/include/caffe/python_layer.hpp index 19cf18c9742..816ef453720 100644 --- a/include/caffe/python_layer.hpp +++ b/include/caffe/python_layer.hpp @@ -14,12 +14,12 @@ template class PythonLayer : public Layer { public: PythonLayer(PyObject* self, const LayerParameter& param) - : Layer(param), self_(bp::handle<>(bp::borrowed(self))) { } + : Layer(param), self_(self) { } virtual void LayerSetUp(const vector*>& bottom, const vector*>& top) { try { - self_.attr("setup")(bottom, top); + bp::call_method(self_, "setup", bottom, top); } catch (bp::error_already_set) { PyErr_Print(); throw; @@ -29,7 +29,7 @@ class PythonLayer : public Layer { virtual void Reshape(const vector*>& bottom, const vector*>& top) { try { - self_.attr("reshape")(bottom, top); + bp::call_method(self_, "reshape", bottom, top); } catch (bp::error_already_set) { PyErr_Print(); throw; @@ -42,7 +42,7 @@ class PythonLayer : public Layer { virtual void Forward_cpu(const vector*>& bottom, const vector*>& top) { try { - self_.attr("forward")(bottom, top); + bp::call_method(self_, "forward", bottom, top); } catch (bp::error_already_set) { PyErr_Print(); throw; @@ -51,7 +51,8 @@ class PythonLayer : public Layer { virtual void Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) { try { - self_.attr("backward")(top, propagate_down, bottom); + bp::call_method(self_, "backward", top, propagate_down, + bottom); } catch (bp::error_already_set) { PyErr_Print(); throw; @@ -59,7 +60,7 @@ class PythonLayer : public Layer { } private: - bp::object self_; + PyObject* self_; }; } // namespace caffe diff --git a/include/caffe/solver.hpp b/include/caffe/solver.hpp index c2ced487d6f..4dcdc3dc20b 100644 --- a/include/caffe/solver.hpp +++ b/include/caffe/solver.hpp @@ -11,7 +11,7 @@ namespace caffe { /** * @brief An interface for classes that perform optimization on Net%s. * - * Requires implementation of ApplyUpdate to compute a parameter update + * Requires implementation of ComputeUpdateValue to compute a parameter update * given the current state of the Net parameters. */ template @@ -39,8 +39,8 @@ class Solver { int iter() { return iter_; } protected: - // Make and apply the update value for the current iteration. - virtual void ApplyUpdate() = 0; + // Get the update value for the current iteration. + virtual void ComputeUpdateValue() = 0; // The Solver::Snapshot function implements the basic snapshotting utility // that stores the learned net. You should implement the SnapshotSolverState() // function that produces a SolverState protocol buffer that needs to be @@ -80,10 +80,7 @@ class SGDSolver : public Solver { protected: void PreSolve(); Dtype GetLearningRate(); - virtual void ApplyUpdate(); - virtual void Normalize(int param_id); - virtual void Regularize(int param_id); - virtual void ComputeUpdateValue(int param_id, Dtype rate); + virtual void ComputeUpdateValue(); virtual void ClipGradients(); virtual void SnapshotSolverState(SolverState * state); virtual void RestoreSolverState(const SolverState& state); @@ -105,7 +102,7 @@ class NesterovSolver : public SGDSolver { : SGDSolver(param_file) {} protected: - virtual void ComputeUpdateValue(int param_id, Dtype rate); + virtual void ComputeUpdateValue(); DISABLE_COPY_AND_ASSIGN(NesterovSolver); }; @@ -119,7 +116,7 @@ class AdaGradSolver : public SGDSolver { : SGDSolver(param_file) { constructor_sanity_check(); } protected: - virtual void ComputeUpdateValue(int param_id, Dtype rate); + virtual void ComputeUpdateValue(); void constructor_sanity_check() { CHECK_EQ(0, this->param_.momentum()) << "Momentum cannot be used with AdaGrad."; diff --git a/include/caffe/test/test_caffe_main.hpp b/include/caffe/test/test_caffe_main.hpp index fc156091476..bd5f31e063f 100644 --- a/include/caffe/test/test_caffe_main.hpp +++ b/include/caffe/test/test_caffe_main.hpp @@ -40,36 +40,34 @@ class MultiDeviceTest : public ::testing::Test { typedef ::testing::Types TestDtypes; -template -struct CPUDevice { - typedef TypeParam Dtype; +struct FloatCPU { + typedef float Dtype; static const Caffe::Brew device = Caffe::CPU; }; -template -class CPUDeviceTest : public MultiDeviceTest > { +struct DoubleCPU { + typedef double Dtype; + static const Caffe::Brew device = Caffe::CPU; }; #ifdef CPU_ONLY -typedef ::testing::Types, - CPUDevice > TestDtypesAndDevices; +typedef ::testing::Types TestDtypesAndDevices; #else -template -struct GPUDevice { - typedef TypeParam Dtype; +struct FloatGPU { + typedef float Dtype; static const Caffe::Brew device = Caffe::GPU; }; -template -class GPUDeviceTest : public MultiDeviceTest > { +struct DoubleGPU { + typedef double Dtype; + static const Caffe::Brew device = Caffe::GPU; }; -typedef ::testing::Types, CPUDevice, - GPUDevice, GPUDevice > - TestDtypesAndDevices; +typedef ::testing::Types + TestDtypesAndDevices; #endif diff --git a/include/caffe/test/test_gradient_check_util.hpp b/include/caffe/test/test_gradient_check_util.hpp index cc5dcbad0ee..22937711b58 100644 --- a/include/caffe/test/test_gradient_check_util.hpp +++ b/include/caffe/test/test_gradient_check_util.hpp @@ -80,14 +80,11 @@ void GradientChecker::CheckGradientSingle(Layer* layer, CHECK_EQ(top_count, bottom[blob_id]->count()); } } - // First, figure out what blobs we need to check against, and zero init - // parameter blobs. + // First, figure out what blobs we need to check against. vector*> blobs_to_check; vector propagate_down(bottom.size(), check_bottom < 0); for (int i = 0; i < layer->blobs().size(); ++i) { - Blob* blob = layer->blobs()[i].get(); - caffe_set(blob->count(), static_cast(0), blob->mutable_cpu_diff()); - blobs_to_check.push_back(blob); + blobs_to_check.push_back(layer->blobs()[i].get()); } if (check_bottom < 0) { for (int i = 0; i < bottom.size(); ++i) { diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp index 2cacd8e72cd..f43036fcebc 100644 --- a/include/caffe/util/math_functions.hpp +++ b/include/caffe/util/math_functions.hpp @@ -88,9 +88,6 @@ void caffe_rng_bernoulli(const int n, const Dtype p, unsigned int* r); template void caffe_exp(const int n, const Dtype* a, Dtype* y); -template -void caffe_log(const int n, const Dtype* a, Dtype* y); - template void caffe_abs(const int n, const Dtype* a, Dtype* y); @@ -206,9 +203,6 @@ void caffe_gpu_abs(const int n, const Dtype* a, Dtype* y); template void caffe_gpu_exp(const int n, const Dtype* a, Dtype* y); -template -void caffe_gpu_log(const int n, const Dtype* a, Dtype* y); - template void caffe_gpu_powx(const int n, const Dtype* a, const Dtype b, Dtype* y); diff --git a/include/caffe/util/mkl_alternate.hpp b/include/caffe/util/mkl_alternate.hpp index 3355b6658a3..32fdbf79932 100644 --- a/include/caffe/util/mkl_alternate.hpp +++ b/include/caffe/util/mkl_alternate.hpp @@ -33,7 +33,6 @@ extern "C" { DEFINE_VSL_UNARY_FUNC(Sqr, y[i] = a[i] * a[i]); DEFINE_VSL_UNARY_FUNC(Exp, y[i] = exp(a[i])); -DEFINE_VSL_UNARY_FUNC(Ln, y[i] = log(a[i])); DEFINE_VSL_UNARY_FUNC(Abs, y[i] = fabs(a[i])); // A simple way to define the vsl unary functions with singular parameter b. diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp index a6bd86a93f5..cd0ab8babb0 100644 --- a/include/caffe/vision_layers.hpp +++ b/include/caffe/vision_layers.hpp @@ -453,72 +453,6 @@ class CuDNNPoolingLayer : public PoolingLayer { }; #endif -/** - * @brief Does spatial pyramid pooling on the input image - * by taking the max, average, etc. within regions - * so that the result vector of different sized - * images are of the same size. - */ -template -class SPPLayer : public Layer { - public: - explicit SPPLayer(const LayerParameter& param) - : Layer(param) {} - virtual void LayerSetUp(const vector*>& bottom, - const vector*>& top); - virtual void Reshape(const vector*>& bottom, - const vector*>& top); - - virtual inline const char* type() const { return "SPP"; } - virtual inline int ExactNumBottomBlobs() const { return 1; } - virtual inline int MinTopBlobs() const { return 1; } - // MAX POOL layers can output an extra top blob for the mask; - // others can only output the pooled inputs. - virtual inline int MaxTopBlobs() const { - return (this->layer_param_.pooling_param().pool() == - PoolingParameter_PoolMethod_MAX) ? 2 : 1; - } - - protected: - virtual void Forward_cpu(const vector*>& bottom, - const vector*>& top); - virtual void Backward_cpu(const vector*>& top, - const vector& propagate_down, const vector*>& bottom); - // calculates the kernel and stride dimensions for the pooling layer, - // returns a correctly configured LayerParameter for a PoolingLayer - virtual LayerParameter GetPoolingParam(const int pyramid_level, - const int bottom_h, const int bottom_w, const SPPParameter spp_param); - - int pyramid_height_; - int bottom_h_, bottom_w_; - int channels_; - int kernel_h_, kernel_w_; - int pad_h_, pad_w_; - - /// the internal Split layer that feeds the pooling layers - shared_ptr > split_layer_; - /// top vector holder used in call to the underlying SplitLayer::Forward - vector*> split_top_vec_; - /// bottom vector holder used in call to the underlying PoolingLayer::Forward - vector*>*> pooling_bottom_vecs_; - /// the internal Pooling layers of different kernel sizes - vector > > pooling_layers_; - /// top vector holders used in call to the underlying PoolingLayer::Forward - vector*>*> pooling_top_vecs_; - /// pooling_outputs stores the outputs of the PoolingLayers - vector*> pooling_outputs_; - /// the internal Flatten layers that the Pooling layers feed into - vector*> flatten_layers_; - /// top vector holders used in call to the underlying FlattenLayer::Forward - vector*>*> flatten_top_vecs_; - /// flatten_outputs stores the outputs of the FlattenLayers - vector*> flatten_outputs_; - /// bottom vector holder used in call to the underlying ConcatLayer::Forward - vector*> concat_bottom_vec_; - /// the internal Concat layers that the Flatten layers feed into - shared_ptr > concat_layer_; -}; - } // namespace caffe #endif // CAFFE_VISION_LAYERS_HPP_ diff --git a/src/caffe/blob.cpp b/src/caffe/blob.cpp index 94fdcc35fb6..6d2b3f502d9 100644 --- a/src/caffe/blob.cpp +++ b/src/caffe/blob.cpp @@ -26,7 +26,6 @@ void Blob::Reshape(const vector& shape) { shape_.resize(shape.size()); for (int i = 0; i < shape.size(); ++i) { CHECK_GE(shape[i], 0); - CHECK_LE(shape[i], INT_MAX / count_) << "blob size exceeds INT_MAX"; count_ *= shape[i]; shape_[i] = shape[i]; } diff --git a/src/caffe/data_transformer.cpp b/src/caffe/data_transformer.cpp index 6f75bdb3852..b0b98e478c1 100644 --- a/src/caffe/data_transformer.cpp +++ b/src/caffe/data_transformer.cpp @@ -125,31 +125,10 @@ void DataTransformer::Transform(const Datum& datum, template void DataTransformer::Transform(const Datum& datum, Blob* transformed_blob) { - // If datum is encoded, decoded and transform the cv::image. - if (datum.encoded()) { - CHECK(!param_.force_color() && !param_.force_gray()) - << "cannot set both force_color and force_gray"; - cv::Mat cv_img; - if (param_.force_color() || param_.force_gray()) { - // If force_color then decode in color otherwise decode in gray. - cv_img = DecodeDatumToCVMat(datum, param_.force_color()); - } else { - cv_img = DecodeDatumToCVMatNative(datum); - } - // Transform the cv::image into blob. - return Transform(cv_img, transformed_blob); - } else { - if (param_.force_color() || param_.force_gray()) { - LOG(ERROR) << "force_color and force_gray only for encoded datum"; - } - } - - const int crop_size = param_.crop_size(); const int datum_channels = datum.channels(); const int datum_height = datum.height(); const int datum_width = datum.width(); - // Check dimensions. const int channels = transformed_blob->channels(); const int height = transformed_blob->height(); const int width = transformed_blob->width(); @@ -160,6 +139,8 @@ void DataTransformer::Transform(const Datum& datum, CHECK_LE(width, datum_width); CHECK_GE(num, 1); + const int crop_size = param_.crop_size(); + if (crop_size) { CHECK_EQ(crop_size, height); CHECK_EQ(crop_size, width); @@ -215,12 +196,10 @@ void DataTransformer::Transform(const vector & mat_vector, template void DataTransformer::Transform(const cv::Mat& cv_img, Blob* transformed_blob) { - const int crop_size = param_.crop_size(); const int img_channels = cv_img.channels(); const int img_height = cv_img.rows; const int img_width = cv_img.cols; - // Check dimensions. const int channels = transformed_blob->channels(); const int height = transformed_blob->height(); const int width = transformed_blob->width(); @@ -233,6 +212,7 @@ void DataTransformer::Transform(const cv::Mat& cv_img, CHECK(cv_img.depth() == CV_8U) << "Image data type must be unsigned byte"; + const int crop_size = param_.crop_size(); const Dtype scale = param_.scale(); const bool do_mirror = param_.mirror() && Rand(2); const bool has_mean_file = param_.has_mean_file(); @@ -317,23 +297,11 @@ void DataTransformer::Transform(const cv::Mat& cv_img, template void DataTransformer::Transform(Blob* input_blob, Blob* transformed_blob) { - const int crop_size = param_.crop_size(); const int input_num = input_blob->num(); const int input_channels = input_blob->channels(); const int input_height = input_blob->height(); const int input_width = input_blob->width(); - if (transformed_blob->count() == 0) { - // Initialize transformed_blob with the right shape. - if (crop_size) { - transformed_blob->Reshape(input_num, input_channels, - crop_size, crop_size); - } else { - transformed_blob->Reshape(input_num, input_channels, - input_height, input_width); - } - } - const int num = transformed_blob->num(); const int channels = transformed_blob->channels(); const int height = transformed_blob->height(); @@ -345,7 +313,7 @@ void DataTransformer::Transform(Blob* input_blob, CHECK_GE(input_height, height); CHECK_GE(input_width, width); - + const int crop_size = param_.crop_size(); const Dtype scale = param_.scale(); const bool do_mirror = param_.mirror() && Rand(2); const bool has_mean_file = param_.has_mean_file(); @@ -427,82 +395,6 @@ void DataTransformer::Transform(Blob* input_blob, } } -template -vector DataTransformer::InferBlobShape(const Datum& datum) { - if (datum.encoded()) { - CHECK(!param_.force_color() && !param_.force_gray()) - << "cannot set both force_color and force_gray"; - cv::Mat cv_img; - if (param_.force_color() || param_.force_gray()) { - // If force_color then decode in color otherwise decode in gray. - cv_img = DecodeDatumToCVMat(datum, param_.force_color()); - } else { - cv_img = DecodeDatumToCVMatNative(datum); - } - // InferBlobShape using the cv::image. - return InferBlobShape(cv_img); - } - - const int crop_size = param_.crop_size(); - const int datum_channels = datum.channels(); - const int datum_height = datum.height(); - const int datum_width = datum.width(); - // Check dimensions. - CHECK_GT(datum_channels, 0); - CHECK_GE(datum_height, crop_size); - CHECK_GE(datum_width, crop_size); - // Build BlobShape. - vector shape(4); - shape[0] = 1; - shape[1] = datum_channels; - shape[2] = (crop_size)? crop_size: datum_height; - shape[3] = (crop_size)? crop_size: datum_width; - return shape; -} - -template -vector DataTransformer::InferBlobShape( - const vector & datum_vector) { - const int num = datum_vector.size(); - CHECK_GT(num, 0) << "There is no datum to in the vector"; - // Use first datum in the vector to InferBlobShape. - vector shape = InferBlobShape(datum_vector[0]); - // Adjust num to the size of the vector. - shape[0] = num; - return shape; -} - -template -vector DataTransformer::InferBlobShape(const cv::Mat& cv_img) { - const int crop_size = param_.crop_size(); - const int img_channels = cv_img.channels(); - const int img_height = cv_img.rows; - const int img_width = cv_img.cols; - // Check dimensions. - CHECK_GT(img_channels, 0); - CHECK_GE(img_height, crop_size); - CHECK_GE(img_width, crop_size); - // Build BlobShape. - vector shape(4); - shape[0] = 1; - shape[1] = img_channels; - shape[2] = (crop_size)? crop_size: img_height; - shape[3] = (crop_size)? crop_size: img_width; - return shape; -} - -template -vector DataTransformer::InferBlobShape( - const vector & mat_vector) { - const int num = mat_vector.size(); - CHECK_GT(num, 0) << "There is no cv_img to in the vector"; - // Use first cv_img in the vector to InferBlobShape. - vector shape = InferBlobShape(mat_vector[0]); - // Adjust num to the size of the vector. - shape[0] = num; - return shape; -} - template void DataTransformer::InitRand() { const bool needs_rand = param_.mirror() || diff --git a/src/caffe/layers/base_data_layer.cpp b/src/caffe/layers/base_data_layer.cpp index 26a1118282f..352200915d7 100644 --- a/src/caffe/layers/base_data_layer.cpp +++ b/src/caffe/layers/base_data_layer.cpp @@ -2,6 +2,7 @@ #include #include "caffe/data_layers.hpp" +#include "caffe/net.hpp" #include "caffe/util/io.hpp" namespace caffe { @@ -20,11 +21,11 @@ void BaseDataLayer::LayerSetUp(const vector*>& bottom, } else { output_labels_ = true; } + // The subclasses should setup the size of bottom and top + DataLayerSetUp(bottom, top); data_transformer_.reset( new DataTransformer(transform_param_, this->phase_)); data_transformer_->InitRand(); - // The subclasses should setup the size of bottom and top - DataLayerSetUp(bottom, top); } template @@ -62,15 +63,13 @@ void BasePrefetchingDataLayer::Forward_cpu( JoinPrefetchThread(); DLOG(INFO) << "Thread joined"; // Reshape to loaded data. - top[0]->ReshapeLike(prefetch_data_); + top[0]->Reshape(this->prefetch_data_.num(), this->prefetch_data_.channels(), + this->prefetch_data_.height(), this->prefetch_data_.width()); // Copy the data caffe_copy(prefetch_data_.count(), prefetch_data_.cpu_data(), top[0]->mutable_cpu_data()); DLOG(INFO) << "Prefetch copied"; if (this->output_labels_) { - // Reshape to loaded labels. - top[1]->ReshapeLike(prefetch_label_); - // Copy the labels. caffe_copy(prefetch_label_.count(), prefetch_label_.cpu_data(), top[1]->mutable_cpu_data()); } diff --git a/src/caffe/layers/base_data_layer.cu b/src/caffe/layers/base_data_layer.cu index 9335a5bc9a9..775f6c47f7e 100644 --- a/src/caffe/layers/base_data_layer.cu +++ b/src/caffe/layers/base_data_layer.cu @@ -10,14 +10,12 @@ void BasePrefetchingDataLayer::Forward_gpu( // First, join the thread JoinPrefetchThread(); // Reshape to loaded data. - top[0]->ReshapeLike(this->prefetch_data_); + top[0]->Reshape(this->prefetch_data_.num(), this->prefetch_data_.channels(), + this->prefetch_data_.height(), this->prefetch_data_.width()); // Copy the data caffe_copy(prefetch_data_.count(), prefetch_data_.cpu_data(), top[0]->mutable_gpu_data()); if (this->output_labels_) { - // Reshape to loaded labels. - top[1]->ReshapeLike(prefetch_label_); - // Copy the labels. caffe_copy(prefetch_label_.count(), prefetch_label_.cpu_data(), top[1]->mutable_gpu_data()); } diff --git a/src/caffe/layers/concat_layer.cu b/src/caffe/layers/concat_layer.cu index 8f2e85d8f52..dbadb5aeb30 100644 --- a/src/caffe/layers/concat_layer.cu +++ b/src/caffe/layers/concat_layer.cu @@ -6,41 +6,21 @@ namespace caffe { -template -__global__ void Concat(const int nthreads, const Dtype* in_data, - const bool forward, const int num_concats, const int concat_size, - const int top_concat_axis, const int bottom_concat_axis, - const int offset_concat_axis, Dtype* out_data) { - CUDA_KERNEL_LOOP(index, nthreads) { - const int total_concat_size = concat_size * bottom_concat_axis; - const int concat_num = index / total_concat_size; - const int concat_index = index % total_concat_size; - const int top_index = concat_index + - (concat_num * top_concat_axis + offset_concat_axis) * concat_size; - if (forward) { - out_data[top_index] = in_data[index]; - } else { - out_data[index] = in_data[top_index]; - } - } -} - template void ConcatLayer::Forward_gpu(const vector*>& bottom, const vector*>& top) { Dtype* top_data = top[0]->mutable_gpu_data(); int offset_concat_axis = 0; const int top_concat_axis = top[0]->shape(concat_axis_); - const bool kForward = true; for (int i = 0; i < bottom.size(); ++i) { const Dtype* bottom_data = bottom[i]->gpu_data(); const int bottom_concat_axis = bottom[i]->shape(concat_axis_); - const int bottom_concat_size = bottom_concat_axis * concat_input_size_; - const int nthreads = bottom_concat_size * num_concats_; - Concat // NOLINT_NEXT_LINE(whitespace/operators) - <<>>( - nthreads, bottom_data, kForward, num_concats_, concat_input_size_, - top_concat_axis, bottom_concat_axis, offset_concat_axis, top_data); + for (int n = 0; n < num_concats_; ++n) { + caffe_copy(bottom_concat_axis * concat_input_size_, + bottom_data + n * bottom_concat_axis * concat_input_size_, + top_data + (n * top_concat_axis + offset_concat_axis) + * concat_input_size_); + } offset_concat_axis += bottom_concat_axis; } } @@ -51,17 +31,15 @@ void ConcatLayer::Backward_gpu(const vector*>& top, const Dtype* top_diff = top[0]->gpu_diff(); int offset_concat_axis = 0; const int top_concat_axis = top[0]->shape(concat_axis_); - const bool kForward = false; for (int i = 0; i < bottom.size(); ++i) { if (!propagate_down[i]) { continue; } Dtype* bottom_diff = bottom[i]->mutable_gpu_diff(); const int bottom_concat_axis = bottom[i]->shape(concat_axis_); - const int bottom_concat_size = bottom_concat_axis * concat_input_size_; - const int nthreads = bottom_concat_size * num_concats_; - Concat // NOLINT_NEXT_LINE(whitespace/operators) - <<>>( - nthreads, top_diff, kForward, num_concats_, concat_input_size_, - top_concat_axis, bottom_concat_axis, offset_concat_axis, bottom_diff); + for (int n = 0; n < num_concats_; ++n) { + caffe_copy(bottom_concat_axis * concat_input_size_, top_diff + + (n * top_concat_axis + offset_concat_axis) * concat_input_size_, + bottom_diff + n * bottom_concat_axis * concat_input_size_); + } offset_concat_axis += bottom_concat_axis; } } diff --git a/src/caffe/layers/contrastive_loss_layer.cpp b/src/caffe/layers/contrastive_loss_layer.cpp index 25e167819d3..0692c11c257 100644 --- a/src/caffe/layers/contrastive_loss_layer.cpp +++ b/src/caffe/layers/contrastive_loss_layer.cpp @@ -41,8 +41,6 @@ void ContrastiveLossLayer::Forward_cpu( diff_.mutable_cpu_data()); // a_i-b_i const int channels = bottom[0]->channels(); Dtype margin = this->layer_param_.contrastive_loss_param().margin(); - bool legacy_version = - this->layer_param_.contrastive_loss_param().legacy_version(); Dtype loss(0.0); for (int i = 0; i < bottom[0]->num(); ++i) { dist_sq_.mutable_cpu_data()[i] = caffe_cpu_dot(channels, @@ -50,12 +48,7 @@ void ContrastiveLossLayer::Forward_cpu( if (static_cast(bottom[2]->cpu_data()[i])) { // similar pairs loss += dist_sq_.cpu_data()[i]; } else { // dissimilar pairs - if (legacy_version) { - loss += std::max(margin - dist_sq_.cpu_data()[i], Dtype(0.0)); - } else { - Dtype dist = std::max(margin - sqrt(dist_sq_.cpu_data()[i]), 0.0); - loss += dist*dist; - } + loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0)); } } loss = loss / static_cast(bottom[0]->num()) / Dtype(2); @@ -66,8 +59,6 @@ template void ContrastiveLossLayer::Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) { Dtype margin = this->layer_param_.contrastive_loss_param().margin(); - bool legacy_version = - this->layer_param_.contrastive_loss_param().legacy_version(); for (int i = 0; i < 2; ++i) { if (propagate_down[i]) { const Dtype sign = (i == 0) ? 1 : -1; @@ -85,20 +76,10 @@ void ContrastiveLossLayer::Backward_cpu(const vector*>& top, Dtype(0.0), bout + (j*channels)); } else { // dissimilar pairs - Dtype mdist(0.0); - Dtype beta(0.0); - if (legacy_version) { - mdist = margin - dist_sq_.cpu_data()[j]; - beta = -alpha; - } else { - Dtype dist = sqrt(dist_sq_.cpu_data()[j]); - mdist = margin - dist; - beta = -alpha * mdist / (dist + Dtype(1e-4)); - } - if (mdist > Dtype(0.0)) { + if ((margin-dist_sq_.cpu_data()[j]) > Dtype(0.0)) { caffe_cpu_axpby( channels, - beta, + -alpha, diff_.cpu_data() + (j*channels), Dtype(0.0), bout + (j*channels)); diff --git a/src/caffe/layers/contrastive_loss_layer.cu b/src/caffe/layers/contrastive_loss_layer.cu index 931239316ac..78a55995a0a 100644 --- a/src/caffe/layers/contrastive_loss_layer.cu +++ b/src/caffe/layers/contrastive_loss_layer.cu @@ -32,20 +32,12 @@ void ContrastiveLossLayer::Forward_gpu( Dtype(0.0), dist_sq_.mutable_gpu_data()); // \Sum (a_i-b_i)^2 Dtype margin = this->layer_param_.contrastive_loss_param().margin(); - bool legacy_version = - this->layer_param_.contrastive_loss_param().legacy_version(); Dtype loss(0.0); for (int i = 0; i < bottom[0]->num(); ++i) { if (static_cast(bottom[2]->cpu_data()[i])) { // similar pairs loss += dist_sq_.cpu_data()[i]; } else { // dissimilar pairs - if (legacy_version) { - loss += std::max(margin - dist_sq_.cpu_data()[i], Dtype(0.0)); - } else { - Dtype dist = std::max(margin - sqrt(dist_sq_.cpu_data()[i]), - Dtype(0.0)); - loss += dist*dist; - } + loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0)); } } loss = loss / static_cast(bottom[0]->num()) / Dtype(2); @@ -53,8 +45,8 @@ void ContrastiveLossLayer::Forward_gpu( } template -__global__ void CLLBackward(const int count, const int channels, - const Dtype margin, const bool legacy_version, const Dtype alpha, +__global__ void CLLForward(const int count, const int channels, + const Dtype margin, const Dtype alpha, const Dtype* y, const Dtype* diff, const Dtype* dist_sq, Dtype *bottom_diff) { CUDA_KERNEL_LOOP(i, count) { @@ -62,18 +54,8 @@ __global__ void CLLBackward(const int count, const int channels, if (static_cast(y[n])) { // similar pairs bottom_diff[i] = alpha * diff[i]; } else { // dissimilar pairs - Dtype mdist(0.0); - Dtype beta(0.0); - if (legacy_version) { - mdist = (margin - dist_sq[n]); - beta = -alpha; - } else { - Dtype dist = sqrt(dist_sq[n]); - mdist = (margin - dist); - beta = -alpha * mdist / (dist + Dtype(1e-4)) * diff[i]; - } - if (mdist > 0.0) { - bottom_diff[i] = beta; + if ((margin-dist_sq[n]) > 0.0) { + bottom_diff[i] = -alpha * diff[i]; } else { bottom_diff[i] = 0; } @@ -89,14 +71,12 @@ void ContrastiveLossLayer::Backward_gpu(const vector*>& top, const int count = bottom[0]->count(); const int channels = bottom[0]->channels(); Dtype margin = this->layer_param_.contrastive_loss_param().margin(); - const bool legacy_version = - this->layer_param_.contrastive_loss_param().legacy_version(); const Dtype sign = (i == 0) ? 1 : -1; const Dtype alpha = sign * top[0]->cpu_diff()[0] / static_cast(bottom[0]->num()); // NOLINT_NEXT_LINE(whitespace/operators) - CLLBackward<<>>( - count, channels, margin, legacy_version, alpha, + CLLForward<<>>( + count, channels, margin, alpha, bottom[2]->gpu_data(), // pair similarity 0 or 1 diff_.gpu_data(), // the cached eltwise difference between a and b dist_sq_.gpu_data(), // the cached square distance between a and b diff --git a/src/caffe/layers/conv_layer.cpp b/src/caffe/layers/conv_layer.cpp index 928ef5ee468..c0c9f6f3371 100644 --- a/src/caffe/layers/conv_layer.cpp +++ b/src/caffe/layers/conv_layer.cpp @@ -39,6 +39,13 @@ void ConvolutionLayer::Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) { const Dtype* weight = this->blobs_[0]->cpu_data(); Dtype* weight_diff = this->blobs_[0]->mutable_cpu_diff(); + if (this->param_propagate_down_[0]) { + caffe_set(this->blobs_[0]->count(), Dtype(0), weight_diff); + } + if (this->bias_term_ && this->param_propagate_down_[1]) { + caffe_set(this->blobs_[1]->count(), Dtype(0), + this->blobs_[1]->mutable_cpu_diff()); + } for (int i = 0; i < top.size(); ++i) { const Dtype* top_diff = top[i]->cpu_diff(); const Dtype* bottom_data = bottom[i]->cpu_data(); diff --git a/src/caffe/layers/conv_layer.cu b/src/caffe/layers/conv_layer.cu index b8a98ff7cc9..3902fdf3930 100644 --- a/src/caffe/layers/conv_layer.cu +++ b/src/caffe/layers/conv_layer.cu @@ -31,6 +31,13 @@ void ConvolutionLayer::Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) { const Dtype* weight = this->blobs_[0]->gpu_data(); Dtype* weight_diff = this->blobs_[0]->mutable_gpu_diff(); + if (this->param_propagate_down_[0]) { + caffe_gpu_set(this->blobs_[0]->count(), Dtype(0), weight_diff); + } + if (this->bias_term_ && this->param_propagate_down_[1]) { + caffe_gpu_set(this->blobs_[1]->count(), Dtype(0), + this->blobs_[1]->mutable_gpu_diff()); + } for (int i = 0; i < top.size(); ++i) { const Dtype* top_diff = top[i]->gpu_diff(); // Bias gradient, if necessary. diff --git a/src/caffe/layers/cudnn_conv_layer.cu b/src/caffe/layers/cudnn_conv_layer.cu index b4e802e13d1..4a1a4c4f4f2 100644 --- a/src/caffe/layers/cudnn_conv_layer.cu +++ b/src/caffe/layers/cudnn_conv_layer.cu @@ -101,10 +101,12 @@ void CuDNNConvolutionLayer::Backward_gpu(const vector*>& top, if (this->param_propagate_down_[0]) { weight = this->blobs_[0]->gpu_data(); weight_diff = this->blobs_[0]->mutable_gpu_diff(); + caffe_gpu_set(this->blobs_[0]->count(), Dtype(0), weight_diff); } Dtype* bias_diff = NULL; if (this->bias_term_ && this->param_propagate_down_[1]) { bias_diff = this->blobs_[1]->mutable_gpu_diff(); + caffe_gpu_set(this->blobs_[1]->count(), Dtype(0), bias_diff); } for (int i = 0; i < top.size(); ++i) { const Dtype* top_diff = top[i]->gpu_diff(); diff --git a/src/caffe/layers/data_layer.cpp b/src/caffe/layers/data_layer.cpp index 161a75e0c8c..0f2d66776a9 100644 --- a/src/caffe/layers/data_layer.cpp +++ b/src/caffe/layers/data_layer.cpp @@ -38,17 +38,32 @@ void DataLayer::DataLayerSetUp(const vector*>& bottom, cursor_->Next(); } } - // Read a data point, to initialize the prefetch and top blobs. + // Read a data point, and use it to initialize the top blob. Datum datum; datum.ParseFromString(cursor_->value()); - // Use data_transformer to infer the expected blob shape from datum. - vector top_shape = this->data_transformer_->InferBlobShape(datum); - this->transformed_data_.Reshape(top_shape); - // Reshape top[0] and prefetch_data according to the batch_size. - top_shape[0] = this->layer_param_.data_param().batch_size(); - this->prefetch_data_.Reshape(top_shape); - top[0]->ReshapeLike(this->prefetch_data_); + bool force_color = this->layer_param_.data_param().force_encoded_color(); + if ((force_color && DecodeDatum(&datum, true)) || + DecodeDatumNative(&datum)) { + LOG(INFO) << "Decoding Datum"; + } + // image + int crop_size = this->layer_param_.transform_param().crop_size(); + if (crop_size > 0) { + top[0]->Reshape(this->layer_param_.data_param().batch_size(), + datum.channels(), crop_size, crop_size); + this->prefetch_data_.Reshape(this->layer_param_.data_param().batch_size(), + datum.channels(), crop_size, crop_size); + this->transformed_data_.Reshape(1, datum.channels(), crop_size, crop_size); + } else { + top[0]->Reshape( + this->layer_param_.data_param().batch_size(), datum.channels(), + datum.height(), datum.width()); + this->prefetch_data_.Reshape(this->layer_param_.data_param().batch_size(), + datum.channels(), datum.height(), datum.width()); + this->transformed_data_.Reshape(1, datum.channels(), + datum.height(), datum.width()); + } LOG(INFO) << "output data size: " << top[0]->num() << "," << top[0]->channels() << "," << top[0]->height() << "," << top[0]->width(); @@ -71,17 +86,25 @@ void DataLayer::InternalThreadEntry() { CHECK(this->prefetch_data_.count()); CHECK(this->transformed_data_.count()); - // Reshape according to the first datum of each batch - // on single input batches allows for inputs of varying dimension. + // Reshape on single input batches for inputs of varying dimension. const int batch_size = this->layer_param_.data_param().batch_size(); - Datum datum; - datum.ParseFromString(cursor_->value()); - // Use data_transformer to infer the expected blob shape from datum. - vector top_shape = this->data_transformer_->InferBlobShape(datum); - this->transformed_data_.Reshape(top_shape); - // Reshape prefetch_data according to the batch_size. - top_shape[0] = batch_size; - this->prefetch_data_.Reshape(top_shape); + const int crop_size = this->layer_param_.transform_param().crop_size(); + bool force_color = this->layer_param_.data_param().force_encoded_color(); + if (batch_size == 1 && crop_size == 0) { + Datum datum; + datum.ParseFromString(cursor_->value()); + if (datum.encoded()) { + if (force_color) { + DecodeDatum(&datum, true); + } else { + DecodeDatumNative(&datum); + } + } + this->prefetch_data_.Reshape(1, datum.channels(), + datum.height(), datum.width()); + this->transformed_data_.Reshape(1, datum.channels(), + datum.height(), datum.width()); + } Dtype* top_data = this->prefetch_data_.mutable_cpu_data(); Dtype* top_label = NULL; // suppress warnings about uninitialized variables @@ -89,31 +112,48 @@ void DataLayer::InternalThreadEntry() { if (this->output_labels_) { top_label = this->prefetch_label_.mutable_cpu_data(); } - timer.Start(); for (int item_id = 0; item_id < batch_size; ++item_id) { - // get a datum + timer.Start(); + // get a blob Datum datum; datum.ParseFromString(cursor_->value()); + + cv::Mat cv_img; + if (datum.encoded()) { + if (force_color) { + cv_img = DecodeDatumToCVMat(datum, true); + } else { + cv_img = DecodeDatumToCVMatNative(datum); + } + if (cv_img.channels() != this->transformed_data_.channels()) { + LOG(WARNING) << "Your dataset contains encoded images with mixed " + << "channel sizes. Consider adding a 'force_color' flag to the " + << "model definition, or rebuild your dataset using " + << "convert_imageset."; + } + } read_time += timer.MicroSeconds(); timer.Start(); + // Apply data transformations (mirror, scale, crop...) int offset = this->prefetch_data_.offset(item_id); this->transformed_data_.set_cpu_data(top_data + offset); - this->data_transformer_->Transform(datum, &(this->transformed_data_)); - // Copy label. + if (datum.encoded()) { + this->data_transformer_->Transform(cv_img, &(this->transformed_data_)); + } else { + this->data_transformer_->Transform(datum, &(this->transformed_data_)); + } if (this->output_labels_) { top_label[item_id] = datum.label(); } trans_time += timer.MicroSeconds(); - timer.Start(); - // go to the next item. + // go to the next iter cursor_->Next(); if (!cursor_->valid()) { DLOG(INFO) << "Restarting data prefetching from start."; cursor_->SeekToFirst(); } } - timer.Stop(); batch_timer.Stop(); DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms."; DLOG(INFO) << " Read time: " << read_time / 1000 << " ms."; diff --git a/src/caffe/layers/deconv_layer.cpp b/src/caffe/layers/deconv_layer.cpp index a4612963b6b..e6d65ab526b 100644 --- a/src/caffe/layers/deconv_layer.cpp +++ b/src/caffe/layers/deconv_layer.cpp @@ -39,6 +39,13 @@ void DeconvolutionLayer::Backward_cpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) { const Dtype* weight = this->blobs_[0]->cpu_data(); Dtype* weight_diff = this->blobs_[0]->mutable_cpu_diff(); + if (this->param_propagate_down_[0]) { + caffe_set(this->blobs_[0]->count(), Dtype(0), weight_diff); + } + if (this->bias_term_ && this->param_propagate_down_[1]) { + caffe_set(this->blobs_[1]->count(), Dtype(0), + this->blobs_[1]->mutable_cpu_diff()); + } for (int i = 0; i < top.size(); ++i) { const Dtype* top_diff = top[i]->cpu_diff(); const Dtype* bottom_data = bottom[i]->cpu_data(); diff --git a/src/caffe/layers/deconv_layer.cu b/src/caffe/layers/deconv_layer.cu index 39bc4de8c66..9198dd64c72 100644 --- a/src/caffe/layers/deconv_layer.cu +++ b/src/caffe/layers/deconv_layer.cu @@ -31,6 +31,13 @@ void DeconvolutionLayer::Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom) { const Dtype* weight = this->blobs_[0]->gpu_data(); Dtype* weight_diff = this->blobs_[0]->mutable_gpu_diff(); + if (this->param_propagate_down_[0]) { + caffe_gpu_set(this->blobs_[0]->count(), Dtype(0), weight_diff); + } + if (this->bias_term_ && this->param_propagate_down_[1]) { + caffe_gpu_set(this->blobs_[1]->count(), Dtype(0), + this->blobs_[1]->mutable_gpu_diff()); + } for (int i = 0; i < top.size(); ++i) { const Dtype* top_diff = top[i]->gpu_diff(); const Dtype* bottom_data = bottom[i]->gpu_data(); diff --git a/src/caffe/layers/flatten_layer.cpp b/src/caffe/layers/flatten_layer.cpp index f7e5c9c2172..745f271ea45 100644 --- a/src/caffe/layers/flatten_layer.cpp +++ b/src/caffe/layers/flatten_layer.cpp @@ -9,19 +9,9 @@ namespace caffe { template void FlattenLayer::Reshape(const vector*>& bottom, const vector*>& top) { - const int start_axis = bottom[0]->CanonicalAxisIndex( - this->layer_param_.flatten_param().axis()); - const int end_axis = bottom[0]->CanonicalAxisIndex( - this->layer_param_.flatten_param().end_axis()); - vector top_shape; - for (int i = 0; i < start_axis; ++i) { - top_shape.push_back(bottom[0]->shape(i)); - } - const int flattened_dim = bottom[0]->count(start_axis, end_axis + 1); - top_shape.push_back(flattened_dim); - for (int i = end_axis + 1; i < bottom[0]->num_axes(); ++i) { - top_shape.push_back(bottom[0]->shape(i)); - } + vector top_shape(2); + top_shape[0] = bottom[0]->num(); + top_shape[1] = bottom[0]->count() / bottom[0]->num(); top[0]->Reshape(top_shape); CHECK_EQ(top[0]->count(), bottom[0]->count()); } diff --git a/src/caffe/layers/image_data_layer.cpp b/src/caffe/layers/image_data_layer.cpp index 18c035cba9d..38ebbd5ec14 100644 --- a/src/caffe/layers/image_data_layer.cpp +++ b/src/caffe/layers/image_data_layer.cpp @@ -62,15 +62,21 @@ void ImageDataLayer::DataLayerSetUp(const vector*>& bottom, // Read an image, and use it to initialize the top blob. cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first, new_height, new_width, is_color); - // Use data_transformer to infer the expected blob shape from a cv_image. - vector top_shape = this->data_transformer_->InferBlobShape(cv_img); - this->transformed_data_.Reshape(top_shape); - // Reshape prefetch_data and top[0] according to the batch_size. + const int channels = cv_img.channels(); + const int height = cv_img.rows; + const int width = cv_img.cols; + // image + const int crop_size = this->layer_param_.transform_param().crop_size(); const int batch_size = this->layer_param_.image_data_param().batch_size(); - top_shape[0] = batch_size; - this->prefetch_data_.Reshape(top_shape); - top[0]->ReshapeLike(this->prefetch_data_); - + if (crop_size > 0) { + top[0]->Reshape(batch_size, channels, crop_size, crop_size); + this->prefetch_data_.Reshape(batch_size, channels, crop_size, crop_size); + this->transformed_data_.Reshape(1, channels, crop_size, crop_size); + } else { + top[0]->Reshape(batch_size, channels, height, width); + this->prefetch_data_.Reshape(batch_size, channels, height, width); + this->transformed_data_.Reshape(1, channels, height, width); + } LOG(INFO) << "output data size: " << top[0]->num() << "," << top[0]->channels() << "," << top[0]->height() << "," << top[0]->width(); @@ -101,19 +107,19 @@ void ImageDataLayer::InternalThreadEntry() { const int batch_size = image_data_param.batch_size(); const int new_height = image_data_param.new_height(); const int new_width = image_data_param.new_width(); + const int crop_size = this->layer_param_.transform_param().crop_size(); const bool is_color = image_data_param.is_color(); string root_folder = image_data_param.root_folder(); - // Reshape according to the first image of each batch - // on single input batches allows for inputs of varying dimension. - cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first, - new_height, new_width, is_color); - // Use data_transformer to infer the expected blob shape from a cv_img. - vector top_shape = this->data_transformer_->InferBlobShape(cv_img); - this->transformed_data_.Reshape(top_shape); - // Reshape prefetch_data according to the batch_size. - top_shape[0] = batch_size; - this->prefetch_data_.Reshape(top_shape); + // Reshape on single input batches for inputs of varying dimension. + if (batch_size == 1 && crop_size == 0 && new_height == 0 && new_width == 0) { + cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first, + 0, 0, is_color); + this->prefetch_data_.Reshape(1, cv_img.channels(), + cv_img.rows, cv_img.cols); + this->transformed_data_.Reshape(1, cv_img.channels(), + cv_img.rows, cv_img.cols); + } Dtype* prefetch_data = this->prefetch_data_.mutable_cpu_data(); Dtype* prefetch_label = this->prefetch_label_.mutable_cpu_data(); diff --git a/src/caffe/layers/inner_product_layer.cpp b/src/caffe/layers/inner_product_layer.cpp index 83c3235eb71..89e0c8fbad7 100644 --- a/src/caffe/layers/inner_product_layer.cpp +++ b/src/caffe/layers/inner_product_layer.cpp @@ -101,13 +101,13 @@ void InnerProductLayer::Backward_cpu(const vector*>& top, const Dtype* bottom_data = bottom[0]->cpu_data(); // Gradient with respect to weight caffe_cpu_gemm(CblasTrans, CblasNoTrans, N_, K_, M_, (Dtype)1., - top_diff, bottom_data, (Dtype)1., this->blobs_[0]->mutable_cpu_diff()); + top_diff, bottom_data, (Dtype)0., this->blobs_[0]->mutable_cpu_diff()); } if (bias_term_ && this->param_propagate_down_[1]) { const Dtype* top_diff = top[0]->cpu_diff(); // Gradient with respect to bias caffe_cpu_gemv(CblasTrans, M_, N_, (Dtype)1., top_diff, - bias_multiplier_.cpu_data(), (Dtype)1., + bias_multiplier_.cpu_data(), (Dtype)0., this->blobs_[1]->mutable_cpu_diff()); } if (propagate_down[0]) { diff --git a/src/caffe/layers/inner_product_layer.cu b/src/caffe/layers/inner_product_layer.cu index dd90cac12a8..a9e1784a205 100644 --- a/src/caffe/layers/inner_product_layer.cu +++ b/src/caffe/layers/inner_product_layer.cu @@ -33,13 +33,13 @@ void InnerProductLayer::Backward_gpu(const vector*>& top, const Dtype* bottom_data = bottom[0]->gpu_data(); // Gradient with respect to weight caffe_gpu_gemm(CblasTrans, CblasNoTrans, N_, K_, M_, (Dtype)1., - top_diff, bottom_data, (Dtype)1., this->blobs_[0]->mutable_gpu_diff()); + top_diff, bottom_data, (Dtype)0., this->blobs_[0]->mutable_gpu_diff()); } if (bias_term_ && this->param_propagate_down_[1]) { const Dtype* top_diff = top[0]->gpu_diff(); // Gradient with respect to bias caffe_gpu_gemv(CblasTrans, M_, N_, (Dtype)1., top_diff, - bias_multiplier_.gpu_data(), (Dtype)1., + bias_multiplier_.gpu_data(), (Dtype)0., this->blobs_[1]->mutable_gpu_diff()); } if (propagate_down[0]) { diff --git a/src/caffe/layers/lrn_layer.cu b/src/caffe/layers/lrn_layer.cu index 001b3c34ac1..24aa6a30130 100644 --- a/src/caffe/layers/lrn_layer.cu +++ b/src/caffe/layers/lrn_layer.cu @@ -7,46 +7,44 @@ namespace caffe { template -__global__ void LRNFillScale(const int nthreads, const Dtype* const in, +__global__ void LRNFillScale(const int nthreads, const Dtype* in, const int num, const int channels, const int height, const int width, const int size, const Dtype alpha_over_size, - const Dtype k, Dtype* const scale) { + const Dtype k, Dtype* scale) { CUDA_KERNEL_LOOP(index, nthreads) { // find out the local offset - const int w = index % width; - const int h = (index / width) % height; - const int n = index / width / height; - const int offset = (n * channels * height + h) * width + w; - const int step = height * width; - const Dtype* const in_off = in + offset; - Dtype* const scale_off = scale + offset; + int w = index % width; + int h = (index / width) % height; + int n = index / width / height; + int offset = (n * channels * height + h) * width + w; + int step = height * width; + in += offset; + scale += offset; int head = 0; - const int pre_pad = (size - 1) / 2; - const int post_pad = size - pre_pad - 1; + int pre_pad = (size - 1) / 2; + int post_pad = size - pre_pad - 1; Dtype accum_scale = 0; // fill the scale at [n, :, h, w] // accumulate values while (head < post_pad && head < channels) { - accum_scale += in_off[head * step] * in_off[head * step]; + accum_scale += in[head * step] * in[head * step]; ++head; } // both add and subtract while (head < channels) { - accum_scale += in_off[head * step] * in_off[head * step]; + accum_scale += in[head * step] * in[head * step]; if (head - size >= 0) { - accum_scale -= in_off[(head - size) * step] - * in_off[(head - size) * step]; + accum_scale -= in[(head - size) * step] * in[(head - size) * step]; } - scale_off[(head - post_pad) * step] = k + accum_scale * alpha_over_size; + scale[(head - post_pad) * step] = k + accum_scale * alpha_over_size; ++head; } // subtract only while (head < channels + post_pad) { if (head - size >= 0) { - accum_scale -= in_off[(head - size) * step] - * in_off[(head - size) * step]; + accum_scale -= in[(head - size) * step] * in[(head - size) * step]; } - scale_off[(head - post_pad) * step] = k + accum_scale * alpha_over_size; + scale[(head - post_pad) * step] = k + accum_scale * alpha_over_size; ++head; } } @@ -70,8 +68,8 @@ void LRNLayer::Forward_gpu(const vector*>& bottom, // TODO: check if it would be faster to just put it into the previous kernel. template -__global__ void LRNComputeOutput(const int nthreads, const Dtype* const in, - const Dtype* const scale, const Dtype negative_beta, Dtype* const out) { +__global__ void LRNComputeOutput(const int nthreads, const Dtype* in, + const Dtype* scale, const Dtype negative_beta, Dtype* out) { CUDA_KERNEL_LOOP(index, nthreads) { out[index] = in[index] * pow(scale[index], negative_beta); } @@ -120,58 +118,56 @@ void LRNLayer::Backward_gpu(const vector*>& top, } template -__global__ void LRNComputeDiff(const int nthreads, - const Dtype* const bottom_data, const Dtype* const top_data, - const Dtype* const scale, const Dtype* const top_diff, +__global__ void LRNComputeDiff(const int nthreads, const Dtype* bottom_data, + const Dtype* top_data, const Dtype* scale, const Dtype* top_diff, const int num, const int channels, const int height, const int width, const int size, const Dtype negative_beta, - const Dtype cache_ratio, Dtype* const bottom_diff) { + const Dtype cache_ratio, + Dtype* bottom_diff) { CUDA_KERNEL_LOOP(index, nthreads) { // find out the local offset - const int w = index % width; - const int h = (index / width) % height; - const int n = index / width / height; - const int offset = (n * channels * height + h) * width + w; - const int step = height * width; - const Dtype* const bottom_off = bottom_data + offset; - const Dtype* const top_off = top_data + offset; - const Dtype* const scale_off = scale + offset; - const Dtype* const top_diff_off = top_diff + offset; - Dtype* const bottom_diff_off = bottom_diff + offset; + int w = index % width; + int h = (index / width) % height; + int n = index / width / height; + int offset = (n * channels * height + h) * width + w; + int step = height * width; + bottom_data += offset; + top_data += offset; + scale += offset; + top_diff += offset; + bottom_diff += offset; int head = 0; - const int pre_pad = size - (size + 1) / 2; - const int post_pad = size - pre_pad - 1; + int pre_pad = size - (size + 1) / 2; + int post_pad = size - pre_pad - 1; Dtype accum_ratio = 0; // accumulate values while (head < post_pad && head < channels) { - accum_ratio += top_diff_off[head * step] * top_off[head * step] / - scale_off[head * step]; + accum_ratio += top_diff[head * step] * top_data[head * step] / + scale[head * step]; ++head; } // both add and subtract while (head < channels) { - accum_ratio += top_diff_off[head * step] * top_off[head * step] / - scale_off[head * step]; + accum_ratio += top_diff[head * step] * top_data[head * step] / + scale[head * step]; if (head - size >= 0) { - accum_ratio -= top_diff_off[(head - size) * step] * - top_off[(head - size) * step] / scale_off[(head - size) * step]; + accum_ratio -= top_diff[(head - size) * step] * + top_data[(head - size) * step] / scale[(head - size) * step]; } - bottom_diff_off[(head - post_pad) * step] = - top_diff_off[(head - post_pad) * step] - * pow(scale_off[(head - post_pad) * step], negative_beta) - - cache_ratio * bottom_off[(head - post_pad) * step] * accum_ratio; + bottom_diff[(head - post_pad) * step] = top_diff[(head - post_pad) * step] + * pow(scale[(head - post_pad) * step], negative_beta) - cache_ratio * + bottom_data[(head - post_pad) * step] * accum_ratio; ++head; } // subtract only while (head < channels + post_pad) { if (head - size >= 0) { - accum_ratio -= top_diff_off[(head - size) * step] * - top_off[(head - size) * step] / scale_off[(head - size) * step]; + accum_ratio -= top_diff[(head - size) * step] * + top_data[(head - size) * step] / scale[(head - size) * step]; } - bottom_diff_off[(head - post_pad) * step] = - top_diff_off[(head - post_pad) * step] - * pow(scale_off[(head - post_pad) * step], negative_beta) - - cache_ratio * bottom_off[(head - post_pad) * step] * accum_ratio; + bottom_diff[(head - post_pad) * step] = top_diff[(head - post_pad) * step] + * pow(scale[(head - post_pad) * step], negative_beta) - cache_ratio * + bottom_data[(head - post_pad) * step] * accum_ratio; ++head; } } diff --git a/src/caffe/layers/mvn_layer.cpp b/src/caffe/layers/mvn_layer.cpp index 3e79bddcdde..b74d7b4f300 100644 --- a/src/caffe/layers/mvn_layer.cpp +++ b/src/caffe/layers/mvn_layer.cpp @@ -22,7 +22,6 @@ void MVNLayer::Reshape(const vector*>& bottom, bottom[0]->height(), bottom[0]->width()); Dtype* multiplier_data = sum_multiplier_.mutable_cpu_data(); caffe_set(sum_multiplier_.count(), Dtype(1), multiplier_data); - eps_ = this->layer_param_.mvn_param().eps(); } template @@ -37,6 +36,7 @@ void MVNLayer::Forward_cpu(const vector*>& bottom, num = bottom[0]->num() * bottom[0]->channels(); int dim = bottom[0]->count() / num; + Dtype eps = 1e-10; if (this->layer_param_.mvn_param().normalize_variance()) { // put the squares of bottom into temp_ @@ -66,7 +66,7 @@ void MVNLayer::Forward_cpu(const vector*>& bottom, caffe_powx(variance_.count(), variance_.cpu_data(), Dtype(0.5), variance_.mutable_cpu_data()); - caffe_add_scalar(variance_.count(), eps_, variance_.mutable_cpu_data()); + caffe_add_scalar(variance_.count(), eps, variance_.mutable_cpu_data()); caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, 1., variance_.cpu_data(), sum_multiplier_.cpu_data(), 0., @@ -102,6 +102,7 @@ void MVNLayer::Backward_cpu(const vector*>& top, num = bottom[0]->num() * bottom[0]->channels(); int dim = bottom[0]->count() / num; + Dtype eps = 1e-10; if (this->layer_param_.mvn_param().normalize_variance()) { caffe_mul(temp_.count(), top_data, top_diff, bottom_diff); @@ -124,6 +125,24 @@ void MVNLayer::Backward_cpu(const vector*>& top, // put the squares of bottom into temp_ caffe_powx(temp_.count(), bottom_data, Dtype(2), temp_.mutable_cpu_data()); + + // computes variance using var(X) = E(X^2) - (EX)^2 + caffe_cpu_gemv(CblasNoTrans, num, dim, 1. / dim, bottom_data, + sum_multiplier_.cpu_data(), 0., mean_.mutable_cpu_data()); // EX + caffe_cpu_gemv(CblasNoTrans, num, dim, 1. / dim, temp_.cpu_data(), + sum_multiplier_.cpu_data(), 0., + variance_.mutable_cpu_data()); // E(X^2) + caffe_powx(mean_.count(), mean_.cpu_data(), Dtype(2), + temp_.mutable_cpu_data()); // (EX)^2 + caffe_sub(mean_.count(), variance_.cpu_data(), temp_.cpu_data(), + variance_.mutable_cpu_data()); // variance + + // normalize variance + caffe_powx(variance_.count(), variance_.cpu_data(), Dtype(0.5), + variance_.mutable_cpu_data()); + + caffe_add_scalar(variance_.count(), eps, variance_.mutable_cpu_data()); + caffe_cpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, 1., variance_.cpu_data(), sum_multiplier_.cpu_data(), 0., temp_.mutable_cpu_data()); diff --git a/src/caffe/layers/mvn_layer.cu b/src/caffe/layers/mvn_layer.cu index 3888a0c7106..0667f50380f 100644 --- a/src/caffe/layers/mvn_layer.cu +++ b/src/caffe/layers/mvn_layer.cu @@ -36,6 +36,8 @@ void MVNLayer::Forward_gpu(const vector*>& bottom, caffe_gpu_sub(mean_.count(), variance_.gpu_data(), temp_.gpu_data(), variance_.mutable_gpu_data()); // variance + Dtype eps = 1e-10; + // do mean and variance normalization // subtract mean caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, -1., @@ -48,7 +50,7 @@ void MVNLayer::Forward_gpu(const vector*>& bottom, caffe_gpu_powx(variance_.count(), variance_.gpu_data(), Dtype(0.5), variance_.mutable_gpu_data()); - caffe_gpu_add_scalar(variance_.count(), eps_, variance_.mutable_gpu_data()); + caffe_gpu_add_scalar(variance_.count(), eps, variance_.mutable_gpu_data()); caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, 1., variance_.gpu_data(), sum_multiplier_.gpu_data(), 0., @@ -85,6 +87,8 @@ void MVNLayer::Backward_gpu(const vector*>& top, int dim = bottom[0]->count() / num; + Dtype eps = 1e-10; + if (this->layer_param_.mvn_param().normalize_variance()) { caffe_gpu_mul(temp_.count(), top_data, top_diff, bottom_diff); caffe_gpu_gemv(CblasNoTrans, num, dim, 1., bottom_diff, @@ -107,6 +111,23 @@ void MVNLayer::Backward_gpu(const vector*>& top, caffe_gpu_powx(temp_.count(), bottom_data, Dtype(2), temp_.mutable_gpu_data()); + // computes variance using var(X) = E(X^2) - (EX)^2 + caffe_gpu_gemv(CblasNoTrans, num, dim, 1. / dim, bottom_data, + sum_multiplier_.gpu_data(), 0., mean_.mutable_gpu_data()); // EX + caffe_gpu_gemv(CblasNoTrans, num, dim, 1. / dim, temp_.gpu_data(), + sum_multiplier_.gpu_data(), 0., + variance_.mutable_gpu_data()); // E(X^2) + caffe_gpu_powx(mean_.count(), mean_.gpu_data(), Dtype(2), + temp_.mutable_gpu_data()); // (EX)^2 + caffe_gpu_sub(mean_.count(), variance_.gpu_data(), temp_.gpu_data(), + variance_.mutable_gpu_data()); // variance + + // normalize variance + caffe_gpu_powx(variance_.count(), variance_.gpu_data(), Dtype(0.5), + variance_.mutable_gpu_data()); + + caffe_gpu_add_scalar(variance_.count(), eps, variance_.mutable_gpu_data()); + caffe_gpu_gemm(CblasNoTrans, CblasNoTrans, num, dim, 1, 1., variance_.gpu_data(), sum_multiplier_.gpu_data(), 0., temp_.mutable_gpu_data()); diff --git a/src/caffe/layers/pooling_layer.cu b/src/caffe/layers/pooling_layer.cu index ca4b13f7c41..d1d48501af3 100644 --- a/src/caffe/layers/pooling_layer.cu +++ b/src/caffe/layers/pooling_layer.cu @@ -9,32 +9,31 @@ namespace caffe { template -__global__ void MaxPoolForward(const int nthreads, - const Dtype* const bottom_data, const int num, const int channels, - const int height, const int width, const int pooled_height, - const int pooled_width, const int kernel_h, const int kernel_w, - const int stride_h, const int stride_w, const int pad_h, const int pad_w, - Dtype* const top_data, int* mask, Dtype* top_mask) { +__global__ void MaxPoolForward(const int nthreads, const Dtype* bottom_data, + const int num, const int channels, const int height, + const int width, const int pooled_height, const int pooled_width, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, Dtype* top_data, + int* mask, Dtype* top_mask) { CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int c = (index / pooled_width / pooled_height) % channels; - const int n = index / pooled_width / pooled_height / channels; + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; int hstart = ph * stride_h - pad_h; int wstart = pw * stride_w - pad_w; - const int hend = min(hstart + kernel_h, height); - const int wend = min(wstart + kernel_w, width); + int hend = min(hstart + kernel_h, height); + int wend = min(wstart + kernel_w, width); hstart = max(hstart, 0); wstart = max(wstart, 0); Dtype maxval = -FLT_MAX; int maxidx = -1; - const Dtype* const bottom_slice = - bottom_data + (n * channels + c) * height * width; + bottom_data += (n * channels + c) * height * width; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - if (bottom_slice[h * width + w] > maxval) { + if (bottom_data[h * width + w] > maxval) { maxidx = h * width + w; - maxval = bottom_slice[maxidx]; + maxval = bottom_data[maxidx]; } } } @@ -48,32 +47,30 @@ __global__ void MaxPoolForward(const int nthreads, } template -__global__ void AvePoolForward(const int nthreads, - const Dtype* const bottom_data, const int num, const int channels, - const int height, const int width, const int pooled_height, - const int pooled_width, const int kernel_h, const int kernel_w, - const int stride_h, const int stride_w, const int pad_h, const int pad_w, - Dtype* const top_data) { +__global__ void AvePoolForward(const int nthreads, const Dtype* bottom_data, + const int num, const int channels, const int height, + const int width, const int pooled_height, const int pooled_width, + const int kernel_h, const int kernel_w, const int stride_h, + const int stride_w, const int pad_h, const int pad_w, Dtype* top_data) { CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int c = (index / pooled_width / pooled_height) % channels; - const int n = index / pooled_width / pooled_height / channels; + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; int hstart = ph * stride_h - pad_h; int wstart = pw * stride_w - pad_w; int hend = min(hstart + kernel_h, height + pad_h); int wend = min(wstart + kernel_w, width + pad_w); - const int pool_size = (hend - hstart) * (wend - wstart); + int pool_size = (hend - hstart) * (wend - wstart); hstart = max(hstart, 0); wstart = max(wstart, 0); hend = min(hend, height); wend = min(wend, width); Dtype aveval = 0; - const Dtype* const bottom_slice = - bottom_data + (n * channels + c) * height * width; + bottom_data += (n * channels + c) * height * width; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - aveval += bottom_slice[h * width + w]; + aveval += bottom_data[h * width + w]; } } top_data[index] = aveval / pool_size; @@ -82,38 +79,37 @@ __global__ void AvePoolForward(const int nthreads, template __global__ void StoPoolForwardTrain(const int nthreads, - const Dtype* const bottom_data, + const Dtype* bottom_data, const int num, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const int kernel_h, const int kernel_w, const int stride_h, - const int stride_w, Dtype* const rand_idx, Dtype* const top_data) { + const int stride_w, Dtype* rand_idx, Dtype* top_data) { CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int c = (index / pooled_width / pooled_height) % channels; - const int n = index / pooled_width / pooled_height / channels; - const int hstart = ph * stride_h; - const int hend = min(hstart + kernel_h, height); - const int wstart = pw * stride_w; - const int wend = min(wstart + kernel_w, width); + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; + int hstart = ph * stride_h; + int hend = min(hstart + kernel_h, height); + int wstart = pw * stride_w; + int wend = min(wstart + kernel_w, width); Dtype cumsum = 0.; - const Dtype* const bottom_slice = - bottom_data + (n * channels + c) * height * width; + bottom_data += (n * channels + c) * height * width; // First pass: get sum for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - cumsum += bottom_slice[h * width + w]; + cumsum += bottom_data[h * width + w]; } } - const float thres = rand_idx[index] * cumsum; + float thres = rand_idx[index] * cumsum; // Second pass: get value, and set index. cumsum = 0; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - cumsum += bottom_slice[h * width + w]; + cumsum += bottom_data[h * width + w]; if (cumsum >= thres) { rand_idx[index] = ((n * channels + c) * height + h) * width + w; - top_data[index] = bottom_slice[h * width + w]; + top_data[index] = bottom_data[h * width + w]; return; } } @@ -124,30 +120,29 @@ __global__ void StoPoolForwardTrain(const int nthreads, template __global__ void StoPoolForwardTest(const int nthreads, - const Dtype* const bottom_data, + const Dtype* bottom_data, const int num, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const int kernel_h, const int kernel_w, const int stride_h, - const int stride_w, Dtype* const top_data) { + const int stride_w, Dtype* top_data) { CUDA_KERNEL_LOOP(index, nthreads) { - const int pw = index % pooled_width; - const int ph = (index / pooled_width) % pooled_height; - const int c = (index / pooled_width / pooled_height) % channels; - const int n = index / pooled_width / pooled_height / channels; - const int hstart = ph * stride_h; - const int hend = min(hstart + kernel_h, height); - const int wstart = pw * stride_w; - const int wend = min(wstart + kernel_w, width); + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; + int hstart = ph * stride_h; + int hend = min(hstart + kernel_h, height); + int wstart = pw * stride_w; + int wend = min(wstart + kernel_w, width); // We set cumsum to be 0 to avoid divide-by-zero problems Dtype cumsum = FLT_MIN; Dtype cumvalues = 0.; - const Dtype* const bottom_slice = - bottom_data + (n * channels + c) * height * width; + bottom_data += (n * channels + c) * height * width; // First pass: get sum for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - cumsum += bottom_slice[h * width + w]; - cumvalues += bottom_slice[h * width + w] * bottom_slice[h * width + w]; + cumsum += bottom_data[h * width + w]; + cumvalues += bottom_data[h * width + w] * bottom_data[h * width + w]; } } top_data[index] = cumvalues / cumsum; @@ -215,43 +210,43 @@ void PoolingLayer::Forward_gpu(const vector*>& bottom, template -__global__ void MaxPoolBackward(const int nthreads, const Dtype* const top_diff, - const int* const mask, const Dtype* const top_mask, const int num, - const int channels, const int height, const int width, - const int pooled_height, const int pooled_width, const int kernel_h, - const int kernel_w, const int stride_h, const int stride_w, const int pad_h, - const int pad_w, Dtype* const bottom_diff) { +__global__ void MaxPoolBackward(const int nthreads, const Dtype* top_diff, + const int* mask, const Dtype* top_mask, const int num, const int channels, + const int height, const int width, const int pooled_height, + const int pooled_width, const int kernel_h, const int kernel_w, + const int stride_h, const int stride_w, const int pad_h, const int pad_w, + Dtype* bottom_diff) { CUDA_KERNEL_LOOP(index, nthreads) { // find out the local index // find out the local offset - const int w = index % width; - const int h = (index / width) % height; - const int c = (index / width / height) % channels; - const int n = index / width / height / channels; - const int phstart = - (h + pad_h < kernel_h) ? 0 : (h + pad_h - kernel_h) / stride_h + 1; - const int phend = min((h + pad_h) / stride_h + 1, pooled_height); - const int pwstart = - (w + pad_w < kernel_w) ? 0 : (w + pad_w - kernel_w) / stride_w + 1; - const int pwend = min((w + pad_w) / stride_w + 1, pooled_width); + int w = index % width; + int h = (index / width) % height; + int c = (index / width / height) % channels; + int n = index / width / height / channels; + int phstart = + (h + pad_h < kernel_h) ? 0 : (h + pad_h - kernel_h) / stride_h + 1; + int phend = min((h + pad_h) / stride_h + 1, pooled_height); + int pwstart = + (w + pad_w < kernel_w) ? 0 : (w + pad_w - kernel_w) / stride_w + 1; + int pwend = min((w + pad_w) / stride_w + 1, pooled_width); Dtype gradient = 0; - const int offset = (n * channels + c) * pooled_height * pooled_width; - const Dtype* const top_diff_slice = top_diff + offset; + int offset = (n * channels + c) * pooled_height * pooled_width; + top_diff += offset; if (mask) { - const int* const mask_slice = mask + offset; + mask += offset; for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { - if (mask_slice[ph * pooled_width + pw] == h * width + w) { - gradient += top_diff_slice[ph * pooled_width + pw]; + if (mask[ph * pooled_width + pw] == h * width + w) { + gradient += top_diff[ph * pooled_width + pw]; } } } } else { - const Dtype* const top_mask_slice = top_mask + offset; + top_mask += offset; for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { - if (top_mask_slice[ph * pooled_width + pw] == h * width + w) { - gradient += top_diff_slice[ph * pooled_width + pw]; + if (top_mask[ph * pooled_width + pw] == h * width + w) { + gradient += top_diff[ph * pooled_width + pw]; } } } @@ -261,26 +256,25 @@ __global__ void MaxPoolBackward(const int nthreads, const Dtype* const top_diff, } template -__global__ void AvePoolBackward(const int nthreads, const Dtype* const top_diff, +__global__ void AvePoolBackward(const int nthreads, const Dtype* top_diff, const int num, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const int kernel_h, const int kernel_w, const int stride_h, const int stride_w, const int pad_h, const int pad_w, - Dtype* const bottom_diff) { + Dtype* bottom_diff) { CUDA_KERNEL_LOOP(index, nthreads) { // find out the local index // find out the local offset - const int w = index % width + pad_w; - const int h = (index / width) % height + pad_h; - const int c = (index / width / height) % channels; - const int n = index / width / height / channels; - const int phstart = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1; - const int phend = min(h / stride_h + 1, pooled_height); - const int pwstart = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1; - const int pwend = min(w / stride_w + 1, pooled_width); + int w = index % width + pad_w; + int h = (index / width) % height + pad_h; + int c = (index / width / height) % channels; + int n = index / width / height / channels; + int phstart = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1; + int phend = min(h / stride_h + 1, pooled_height); + int pwstart = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1; + int pwend = min(w / stride_w + 1, pooled_width); Dtype gradient = 0; - const Dtype* const top_diff_slice = - top_diff + (n * channels + c) * pooled_height * pooled_width; + top_diff += (n * channels + c) * pooled_height * pooled_width; for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { // figure out the pooling size @@ -289,7 +283,7 @@ __global__ void AvePoolBackward(const int nthreads, const Dtype* const top_diff, int hend = min(hstart + kernel_h, height + pad_h); int wend = min(wstart + kernel_w, width + pad_w); int pool_size = (hend - hstart) * (wend - wstart); - gradient += top_diff_slice[ph * pooled_width + pw] / pool_size; + gradient += top_diff[ph * pooled_width + pw] / pool_size; } } bottom_diff[index] = gradient; @@ -299,31 +293,29 @@ __global__ void AvePoolBackward(const int nthreads, const Dtype* const top_diff, template __global__ void StoPoolBackward(const int nthreads, - const Dtype* const rand_idx, const Dtype* const top_diff, + const Dtype* rand_idx, const Dtype* top_diff, const int num, const int channels, const int height, const int width, const int pooled_height, const int pooled_width, const int kernel_h, const int kernel_w, const int stride_h, - const int stride_w, Dtype* const bottom_diff) { + const int stride_w, Dtype* bottom_diff) { CUDA_KERNEL_LOOP(index, nthreads) { // find out the local index // find out the local offset - const int w = index % width; - const int h = (index / width) % height; - const int c = (index / width / height) % channels; - const int n = index / width / height / channels; - const int phstart = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1; - const int phend = min(h / stride_h + 1, pooled_height); - const int pwstart = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1; - const int pwend = min(w / stride_w + 1, pooled_width); + int w = index % width; + int h = (index / width) % height; + int c = (index / width / height) % channels; + int n = index / width / height / channels; + int phstart = (h < kernel_h) ? 0 : (h - kernel_h) / stride_h + 1; + int phend = min(h / stride_h + 1, pooled_height); + int pwstart = (w < kernel_w) ? 0 : (w - kernel_w) / stride_w + 1; + int pwend = min(w / stride_w + 1, pooled_width); Dtype gradient = 0; - const Dtype* const rand_idx_slice = - rand_idx + (n * channels + c) * pooled_height * pooled_width; - const Dtype* const top_diff_slice = - top_diff + (n * channels + c) * pooled_height * pooled_width; + rand_idx += (n * channels + c) * pooled_height * pooled_width; + top_diff += (n * channels + c) * pooled_height * pooled_width; for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { - gradient += top_diff_slice[ph * pooled_width + pw] * - (index == static_cast(rand_idx_slice[ph * pooled_width + pw])); + gradient += top_diff[ph * pooled_width + pw] * + (index == static_cast(rand_idx[ph * pooled_width + pw])); } } bottom_diff[index] = gradient; diff --git a/src/caffe/layers/prelu_layer.cpp b/src/caffe/layers/prelu_layer.cpp index 81831755512..7119a274dd3 100644 --- a/src/caffe/layers/prelu_layer.cpp +++ b/src/caffe/layers/prelu_layer.cpp @@ -45,8 +45,7 @@ void PReLULayer::LayerSetUp(const vector*>& bottom, // Propagate gradients to the parameters (as directed by backward pass). this->param_propagate_down_.resize(this->blobs_.size(), true); - multiplier_.Reshape(vector(1, bottom[0]->count(1))); - backward_buff_.Reshape(vector(1, bottom[0]->count(1))); + multiplier_.Reshape(vector(1, bottom[0]->count() / bottom[0]->num())); caffe_set(multiplier_.count(), Dtype(1), multiplier_.mutable_cpu_data()); } @@ -113,6 +112,7 @@ void PReLULayer::Backward_cpu(const vector*>& top, // keep top_diff unchanged. if (this->param_propagate_down_[0]) { Dtype* slope_diff = this->blobs_[0]->mutable_cpu_diff(); + caffe_set(this->blobs_[0]->count(), Dtype(0), slope_diff); for (int i = 0; i < count; ++i) { int c = (i / dim) % channels / div_factor; slope_diff[c] += top_diff[i] * bottom_data[i] * (bottom_data[i] <= 0); diff --git a/src/caffe/layers/prelu_layer.cu b/src/caffe/layers/prelu_layer.cu index e1f20048f60..fd0eda5d191 100644 --- a/src/caffe/layers/prelu_layer.cu +++ b/src/caffe/layers/prelu_layer.cu @@ -75,36 +75,38 @@ void PReLULayer::Backward_gpu(const vector*>& top, bottom_data = bottom_memory_.gpu_data(); } - // Propagate to param + // Propagte to param // Since to write bottom diff will affect top diff if top and bottom blobs // are identical (in-place computaion), we first compute param backward to // keep top_diff unchanged. if (this->param_propagate_down_[0]) { Dtype* slope_diff = this->blobs_[0]->mutable_gpu_diff(); + // slope_diff is set as 0, then accumulated over batches + caffe_gpu_set(this->blobs_[0]->count(), Dtype(0), slope_diff); int cdim = channels * dim; Dtype dsum = 0.; for (int n = 0; n < bottom[0]->num(); ++n) { + Dtype* temp_buff = multiplier_.mutable_gpu_diff(); // compute element-wise diff // NOLINT_NEXT_LINE(whitespace/operators) - PReLUParamBackward<<<<>>( cdim, top_diff + top[0]->offset(n), - bottom_data + bottom[0]->offset(n), - backward_buff_.mutable_gpu_diff()); + bottom_data + bottom[0]->offset(n), multiplier_.mutable_gpu_diff()); CUDA_POST_KERNEL_CHECK; if (channel_shared_) { Dtype d; - caffe_gpu_dot(channels * dim, backward_buff_.gpu_diff(), + caffe_gpu_dot(channels * dim, multiplier_.gpu_diff(), multiplier_.gpu_data(), &d); dsum += d; } else { caffe_gpu_gemv(CblasNoTrans, channels, dim, 1., - backward_buff_.gpu_diff(), multiplier_.gpu_data(), 1., + multiplier_.gpu_diff(), multiplier_.gpu_data(), 1., slope_diff); } } if (channel_shared_) { - caffe_gpu_add_scalar(this->blobs_[0]->count(), Dtype(dsum), slope_diff); + caffe_gpu_set(this->blobs_[0]->count(), Dtype(dsum), slope_diff); } } // Propagate to bottom diff --git a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp index cc236fe1e8e..077d949981c 100644 --- a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp +++ b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cpp @@ -71,7 +71,7 @@ void SigmoidCrossEntropyLossLayer::Backward_cpu( } #ifdef CPU_ONLY -STUB_GPU_BACKWARD(SigmoidCrossEntropyLossLayer, Backward); +STUB_GPU(SigmoidCrossEntropyLossLayer); #endif INSTANTIATE_CLASS(SigmoidCrossEntropyLossLayer); diff --git a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu index 547fa80c72f..08f7f492297 100644 --- a/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu +++ b/src/caffe/layers/sigmoid_cross_entropy_loss_layer.cu @@ -8,6 +8,26 @@ namespace caffe { +template +void SigmoidCrossEntropyLossLayer::Forward_gpu( + const vector*>& bottom, const vector*>& top) { + // The forward pass computes the sigmoid outputs. + sigmoid_bottom_vec_[0] = bottom[0]; + sigmoid_layer_->Forward(sigmoid_bottom_vec_, sigmoid_top_vec_); + // Compute the loss (negative log likelihood) + const int count = bottom[0]->count(); + const int num = bottom[0]->num(); + // Stable version of loss computation from input data + const Dtype* input_data = bottom[0]->cpu_data(); + const Dtype* target = bottom[1]->cpu_data(); + Dtype loss = 0; + for (int i = 0; i < count; ++i) { + loss -= input_data[i] * (target[i] - (input_data[i] >= 0)) - + log(1 + exp(input_data[i] - 2 * input_data[i] * (input_data[i] >= 0))); + } + top[0]->mutable_cpu_data()[0] = loss / num; +} + template void SigmoidCrossEntropyLossLayer::Backward_gpu( const vector*>& top, const vector& propagate_down, @@ -31,7 +51,7 @@ void SigmoidCrossEntropyLossLayer::Backward_gpu( } } -INSTANTIATE_LAYER_GPU_BACKWARD(SigmoidCrossEntropyLossLayer); +INSTANTIATE_LAYER_GPU_FUNCS(SigmoidCrossEntropyLossLayer); } // namespace caffe diff --git a/src/caffe/layers/slice_layer.cu b/src/caffe/layers/slice_layer.cu index 796841d3f52..e6e65677bd8 100644 --- a/src/caffe/layers/slice_layer.cu +++ b/src/caffe/layers/slice_layer.cu @@ -6,41 +6,22 @@ namespace caffe { -template -__global__ void Slice(const int nthreads, const Dtype* in_data, - const bool forward, const int num_slices, const int slice_size, - const int bottom_slice_axis, const int top_slice_axis, - const int offset_slice_axis, Dtype* out_data) { - CUDA_KERNEL_LOOP(index, nthreads) { - const int total_slice_size = slice_size * top_slice_axis; - const int slice_num = index / total_slice_size; - const int slice_index = index % total_slice_size; - const int bottom_index = slice_index + - (slice_num * bottom_slice_axis + offset_slice_axis) * slice_size; - if (forward) { - out_data[index] = in_data[bottom_index]; - } else { - out_data[bottom_index] = in_data[index]; - } - } -} - template void SliceLayer::Forward_gpu(const vector*>& bottom, const vector*>& top) { int offset_slice_axis = 0; const Dtype* bottom_data = bottom[0]->gpu_data(); const int bottom_slice_axis = bottom[0]->shape(slice_axis_); - const bool kForward = true; for (int i = 0; i < top.size(); ++i) { Dtype* top_data = top[i]->mutable_gpu_data(); const int top_slice_axis = top[i]->shape(slice_axis_); - const int top_slice_size = top_slice_axis * slice_size_; - const int nthreads = top_slice_size * num_slices_; - Slice // NOLINT_NEXT_LINE(whitespace/operators) - <<>>( - nthreads, bottom_data, kForward, num_slices_, slice_size_, - bottom_slice_axis, top_slice_axis, offset_slice_axis, top_data); + for (int n = 0; n < num_slices_; ++n) { + const int top_offset = n * top_slice_axis * slice_size_; + const int bottom_offset = + (n * bottom_slice_axis + offset_slice_axis) * slice_size_; + caffe_copy(top_slice_axis * slice_size_, + bottom_data + bottom_offset, top_data + top_offset); + } offset_slice_axis += top_slice_axis; } } @@ -52,16 +33,16 @@ void SliceLayer::Backward_gpu(const vector*>& top, int offset_slice_axis = 0; Dtype* bottom_diff = bottom[0]->mutable_gpu_diff(); const int bottom_slice_axis = bottom[0]->shape(slice_axis_); - const bool kForward = false; for (int i = 0; i < top.size(); ++i) { const Dtype* top_diff = top[i]->gpu_diff(); const int top_slice_axis = top[i]->shape(slice_axis_); - const int top_slice_size = top_slice_axis * slice_size_; - const int nthreads = top_slice_size * num_slices_; - Slice // NOLINT_NEXT_LINE(whitespace/operators) - <<>>( - nthreads, top_diff, kForward, num_slices_, slice_size_, - bottom_slice_axis, top_slice_axis, offset_slice_axis, bottom_diff); + for (int n = 0; n < num_slices_; ++n) { + const int top_offset = n * top_slice_axis * slice_size_; + const int bottom_offset = + (n * bottom_slice_axis + offset_slice_axis) * slice_size_; + caffe_copy(top_slice_axis * slice_size_, + top_diff + top_offset, bottom_diff + bottom_offset); + } offset_slice_axis += top_slice_axis; } } diff --git a/src/caffe/layers/triplet_loss_layer.cpp b/src/caffe/layers/triplet_loss_layer.cpp new file mode 100644 index 00000000000..fc8b9fe036f --- /dev/null +++ b/src/caffe/layers/triplet_loss_layer.cpp @@ -0,0 +1,124 @@ +#include +#include + +#include "caffe/layer.hpp" +#include "caffe/loss_layers.hpp" +#include "caffe/util/io.hpp" +#include "caffe/util/math_functions.hpp" + +namespace caffe { + +template +void TripletLossLayer::LayerSetUp( + const vector*>& bottom, const vector*>& top) { + LossLayer::LayerSetUp(bottom, top); + CHECK_EQ(bottom[0]->channels(), bottom[1]->channels()); + CHECK_EQ(bottom[0]->channels(), bottom[2]->channels()); + CHECK_EQ(bottom[0]->height(), 1); + CHECK_EQ(bottom[0]->width(), 1); + CHECK_EQ(bottom[1]->height(), 1); + CHECK_EQ(bottom[1]->width(), 1); + CHECK_EQ(bottom[2]->height(), 1); + CHECK_EQ(bottom[2]->width(), 1); + CHECK_EQ(bottom[3]->channels(), 1); + CHECK_EQ(bottom[3]->height(), 1); + CHECK_EQ(bottom[3]->width(), 1); + diff_pos.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); + diff_neg.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); + diff_sq_pos.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); + diff_sq_neg.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); + dist_sq_.Reshape(bottom[0]->num(), 1, 1, 1); + dist_sq_pos.Reshape(bottom[0]->num(), 1, 1, 1); + dist_sq_neg.Reshape(bottom[0]->num(), 1, 1, 1); + // vector of ones used to sum along channels + summer_vec_.Reshape(bottom[0]->channels(), 1, 1, 1); + for (int i = 0; i < bottom[0]->channels(); ++i) + summer_vec_.mutable_cpu_data()[i] = Dtype(1); +} + +template +void TripletLossLayer::Forward_cpu( + const vector*>& bottom, + const vector*>& top) { + int count = bottom[0]->count(); + caffe_sub( + count, + bottom[0]->cpu_data(), // a + bottom[1]->cpu_data(), // b + diff_pos.mutable_cpu_data()); // a_i-b_i + caffe_sub( + count, + bottom[0]->cpu_data(), // a + bottom[2]->cpu_data(), // c + diff_neg.mutable_cpu_data()); // a_i-c_i + const int channels = bottom[0]->channels(); + Dtype margin = this->layer_param_.triplet_loss_param().margin(); + Dtype loss(0.0); + + //Loss component calculated from ab + for (int i = 0; i < bottom[0]->num(); ++i) { + dist_sq_pos.mutable_cpu_data()[i] = caffe_cpu_dot(channels, + diff_pos.cpu_data() + (i*channels), diff_pos.cpu_data() + (i*channels)); + //ab is a similar pair + dist_sq_ += dist_sq_pos.cpu_data()[i]; + + + //Loss component calculated from ac + dist_sq_neg.mutable_cpu_data()[i] = caffe_cpu_dot(channels, + diff_neg.cpu_data() + (i*channels), diff_neg.cpu_data() + (i*channels)); + //ac is a dissimilar pair + dist_sq_ -= dist_sq_neg.cpu_data()[i]; + loss += std::max(margin + dist_sq_.cpu_data()[i], Dtype(0.0)); + + } + loss = loss / static_cast(bottom[0]->num()) / Dtype(2); + top[0]->mutable_cpu_data()[0] = loss; +} + +template +void TripletLossLayer::Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom) { + Dtype margin = this->layer_param_.triplet_loss_param().margin(); + for (int i = 0; i < 3; ++i) { + if (propagate_down[i]) { + const Dtype sign = (i == 0) ? 1 : -1; + const Dtype alpha = sign * top[0]->cpu_diff()[0] / + static_cast(bottom[i]->num()); + int num = bottom[i]->num(); + int channels = bottom[i]->channels(); + for (int j = 0; j < num; ++j) { + Dtype* bout = bottom[i]->mutable_cpu_diff(); + if ((margin-dist_sq_.cpu_data()[j]) > Dtype(0.0)) { + // similar pairs + caffe_cpu_axpby( + channels, + alpha, + diff_pos.cpu_data() + (j*channels), + Dtype(0.0), + bout + (j*channels)); + + // dissimilar pairs + + caffe_cpu_axpby( + channels, + -alpha, + diff_neg.cpu_data() + (j*channels), + Dtype(0.0), + bout + (j*channels)); + + } else { + caffe_set(channels, Dtype(0), bout + (j*channels)); + } + } + } + } +} + +#ifdef CPU_ONLY +STUB_GPU(TripletLossLayer); +#endif + +INSTANTIATE_CLASS(TripletLossLayer); +REGISTER_LAYER_CLASS(TripletLoss); + +} // namespace caffe diff --git a/src/caffe/net.cpp b/src/caffe/net.cpp index a18ee63818e..888eec1d501 100644 --- a/src/caffe/net.cpp +++ b/src/caffe/net.cpp @@ -79,17 +79,10 @@ void Net::Init(const NetParameter& in_param) { } // Setup layer. const LayerParameter& layer_param = param.layer(layer_id); - if (layer_param.propagate_down_size() > 0) { - CHECK_EQ(layer_param.propagate_down_size(), - layer_param.bottom_size()) - << "propagate_down param must be specified " - << "either 0 or bottom_size times "; - } layers_.push_back(LayerRegistry::CreateLayer(layer_param)); layer_names_.push_back(layer_param.name()); LOG(INFO) << "Creating Layer " << layer_param.name(); bool need_backward = false; - // Figure out this layer's input and output for (int bottom_id = 0; bottom_id < layer_param.bottom_size(); ++bottom_id) { @@ -158,33 +151,15 @@ void Net::Init(const NetParameter& in_param) { // Go through the net backwards to determine which blobs contribute to the // loss. We can skip backward computation for blobs that don't contribute // to the loss. - // Also checks if all bottom blobs don't need backward computation (possible - // because the skip_propagate_down param) and so we can skip bacward - // computation for the entire layer set blobs_under_loss; - set blobs_skip_backp; for (int layer_id = layers_.size() - 1; layer_id >= 0; --layer_id) { bool layer_contributes_loss = false; - bool layer_skip_propagate_down = true; for (int top_id = 0; top_id < top_vecs_[layer_id].size(); ++top_id) { const string& blob_name = blob_names_[top_id_vecs_[layer_id][top_id]]; if (layers_[layer_id]->loss(top_id) || (blobs_under_loss.find(blob_name) != blobs_under_loss.end())) { layer_contributes_loss = true; - } - if (blobs_skip_backp.find(blob_name) == blobs_skip_backp.end()) { - layer_skip_propagate_down = false; - } - if (layer_contributes_loss && !layer_skip_propagate_down) break; - } - // If this layer can skip backward computation, also all his bottom blobs - // don't need backpropagation - if (layer_need_backward_[layer_id] && layer_skip_propagate_down) { - layer_need_backward_[layer_id] = false; - for (int bottom_id = 0; bottom_id < bottom_vecs_[layer_id].size(); - ++bottom_id) { - bottom_need_backward_[layer_id][bottom_id] = false; } } if (!layer_contributes_loss) { layer_need_backward_[layer_id] = false; } @@ -203,11 +178,6 @@ void Net::Init(const NetParameter& in_param) { } else { bottom_need_backward_[layer_id][bottom_id] = false; } - if (!bottom_need_backward_[layer_id][bottom_id]) { - const string& blob_name = - blob_names_[bottom_id_vecs_[layer_id][bottom_id]]; - blobs_skip_backp.insert(blob_name); - } } } // Handle force_backward if needed. @@ -397,9 +367,9 @@ void Net::AppendTop(const NetParameter& param, const int layer_id, // Helper for Net::Init: add a new bottom blob to the net. template -int Net::AppendBottom(const NetParameter& param, const int layer_id, - const int bottom_id, set* available_blobs, - map* blob_name_to_idx) { +int Net::AppendBottom(const NetParameter& param, + const int layer_id, const int bottom_id, + set* available_blobs, map* blob_name_to_idx) { const LayerParameter& layer_param = param.layer(layer_id); const string& blob_name = layer_param.bottom(bottom_id); if (available_blobs->find(blob_name) == available_blobs->end()) { @@ -411,12 +381,7 @@ int Net::AppendBottom(const NetParameter& param, const int layer_id, bottom_vecs_[layer_id].push_back(blobs_[blob_id].get()); bottom_id_vecs_[layer_id].push_back(blob_id); available_blobs->erase(blob_name); - bool propagate_down = true; - // Check if the backpropagation on bottom_id should be skipped - if (layer_param.propagate_down_size() > 0) - propagate_down = layer_param.propagate_down(bottom_id); - const bool need_backward = blob_need_backward_[blob_id] && - propagate_down; + const bool need_backward = blob_need_backward_[blob_id]; bottom_need_backward_[layer_id].push_back(need_backward); return blob_id; } @@ -445,7 +410,7 @@ void Net::AppendParam(const NetParameter& param, const int layer_id, // (i.e., not given a param_name) or explicitly given a name that we // haven't already seen. param_owners_.push_back(-1); - if (param_name.size()) { + if (param_size) { param_names_index_[param_name] = net_param_id; } } else { @@ -505,6 +470,7 @@ Dtype Net::ForwardFromTo(int start, int end) { } for (int i = start; i <= end; ++i) { // LOG(ERROR) << "Forwarding " << layer_names_[i]; + layers_[i]->Reshape(bottom_vecs_[i], top_vecs_[i]); Dtype layer_loss = layers_[i]->Forward(bottom_vecs_[i], top_vecs_[i]); loss += layer_loss; if (debug_info_) { ForwardDebugInfo(i); } diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto index 81a8c69d88e..edf7ae81d58 100644 --- a/src/caffe/proto/caffe.proto +++ b/src/caffe/proto/caffe.proto @@ -49,14 +49,6 @@ message FillerParameter { // The expected number of non-zero output weights for a given input in // Gaussian filler -- the default -1 means don't perform sparsification. optional int32 sparse = 7 [default = -1]; - // Normalize the filler variance by fan_in, fan_out, or their average. - // Applies to 'xavier' and 'msra' fillers. - enum VarianceNorm { - FAN_IN = 0; - FAN_OUT = 1; - AVERAGE = 2; - } - optional VarianceNorm variance_norm = 8 [default = FAN_IN]; } message NetParameter { @@ -96,7 +88,7 @@ message NetParameter { // NOTE // Update the next available ID when you add a new SolverParameter field. // -// SolverParameter next available ID: 37 (last added: iter_size) +// SolverParameter next available ID: 36 (last added: clip_gradients) message SolverParameter { ////////////////////////////////////////////////////////////////////////////// // Specifying the train and test networks @@ -149,8 +141,6 @@ message SolverParameter { // Display the loss averaged over the last average_loss iterations optional int32 average_loss = 33 [default = 1]; optional int32 max_iter = 7; // the maximum number of iterations - // accumulate gradients over `iter_size` x `batch_size` instances - optional int32 iter_size = 36 [default = 1]; optional string lr_policy = 8; // The learning rate decay policy. optional float gamma = 9; // The parameter to compute the learning rate. optional float power = 10; // The parameter to compute the learning rate. @@ -269,7 +259,7 @@ message ParamSpec { // NOTE // Update the next available ID when you add a new LayerParameter field. // -// LayerParameter next available layer-specific ID: 137 (last added: reduction_param) +// LayerParameter next available layer-specific ID: 132 (last added: prelu_param) message LayerParameter { optional string name = 1; // the layer name optional string type = 2; // the layer type @@ -290,10 +280,6 @@ message LayerParameter { // The blobs containing the numeric parameters of the layer. repeated BlobProto blobs = 7; - - // Specifies on which bottoms the backpropagation should be skipped. - // The size must be either 0 or equal to the number of bottoms. - repeated bool propagate_down = 11; // Rules controlling whether and when a layer is included in the network, // based on the current NetState. You may specify a non-zero number of rules @@ -326,14 +312,12 @@ message LayerParameter { optional DummyDataParameter dummy_data_param = 109; optional EltwiseParameter eltwise_param = 110; optional ExpParameter exp_param = 111; - optional FlattenParameter flatten_param = 135; optional HDF5DataParameter hdf5_data_param = 112; optional HDF5OutputParameter hdf5_output_param = 113; optional HingeLossParameter hinge_loss_param = 114; optional ImageDataParameter image_data_param = 115; optional InfogainLossParameter infogain_loss_param = 116; optional InnerProductParameter inner_product_param = 117; - optional LogParameter log_param = 134; optional LRNParameter lrn_param = 118; optional MemoryDataParameter memory_data_param = 119; optional MVNParameter mvn_param = 120; @@ -341,16 +325,14 @@ message LayerParameter { optional PowerParameter power_param = 122; optional PReLUParameter prelu_param = 131; optional PythonParameter python_param = 130; - optional ReductionParameter reduction_param = 136; optional ReLUParameter relu_param = 123; - optional ReshapeParameter reshape_param = 133; optional SigmoidParameter sigmoid_param = 124; optional SoftmaxParameter softmax_param = 125; - optional SPPParameter spp_param = 132; optional SliceParameter slice_param = 126; optional TanHParameter tanh_param = 127; optional ThresholdParameter threshold_param = 128; optional WindowDataParameter window_data_param = 129; + optional TripletLossParameter triplet_loss_param = 132; } // Message that stores parameters used to apply transformation @@ -370,10 +352,6 @@ message TransformationParameter { // or can be repeated the same number of times as channels // (would subtract them from the corresponding channel) repeated float mean_value = 5; - // Force the decoded image to have 3 color channels. - optional bool force_color = 6 [default = false]; - // Force the decoded image to have 1 color channels. - optional bool force_gray = 7 [default = false]; } // Message that stores parameters shared by loss layers @@ -385,9 +363,7 @@ message LossParameter { optional bool normalize = 2 [default = true]; } -// Messages that store parameters used by individual layer types follow, in -// alphabetical order. - +// Message that stores parameters used by AccuracyLayer message AccuracyParameter { // When computing accuracy, count as correct by comparing the true label to // the top k scoring classes. By default, only compare to the top scoring @@ -405,12 +381,14 @@ message AccuracyParameter { optional int32 ignore_label = 3; } +// Message that stores parameters used by ArgMaxLayer message ArgMaxParameter { // If true produce pairs (argmax, maxval) optional bool out_max_val = 1 [default = false]; optional uint32 top_k = 2 [default = 1]; } +// Message that stores parameters used by ConcatLayer message ConcatParameter { // The axis along which to concatenate -- may be negative to index from the // end (e.g., -1 for the last axis). Other axes must have the @@ -422,18 +400,16 @@ message ConcatParameter { optional uint32 concat_dim = 1 [default = 1]; } +// Message that stores parameters used by ContrastiveLossLayer message ContrastiveLossParameter { - // margin for dissimilar pair + //margin for dissimilar pair optional float margin = 1 [default = 1.0]; - // The first implementation of this cost did not exactly match the cost of - // Hadsell et al 2006 -- using (margin - d^2) instead of (margin - d)^2. - // legacy_version = false (the default) uses (margin - d)^2 as proposed in the - // Hadsell paper. New models should probably use this version. - // legacy_version = true uses (margin - d^2). This is kept to support / - // reproduce existing models and results - optional bool legacy_version = 2 [default = false]; } - +message TripletLossParameter { + //margin for negative triplet + optional float margin = 1 [default = 1.0]; +} +// Message that stores parameters used by ConvolutionLayer message ConvolutionParameter { optional uint32 num_output = 1; // The number of outputs for the layer optional bool bias_term = 2 [default = true]; // whether to have bias terms @@ -459,6 +435,7 @@ message ConvolutionParameter { optional Engine engine = 15 [default = DEFAULT]; } +// Message that stores parameters used by DataLayer message DataParameter { enum DB { LEVELDB = 0; @@ -489,10 +466,12 @@ message DataParameter { optional bool force_encoded_color = 9 [default = false]; } +// Message that stores parameters used by DropoutLayer message DropoutParameter { optional float dropout_ratio = 1 [default = 0.5]; // dropout ratio } +// Message that stores parameters used by DummyDataLayer. // DummyDataLayer fills any number of arbitrarily shaped blobs with random // (or constant) data generated by "Fillers" (see "message FillerParameter"). message DummyDataParameter { @@ -512,6 +491,7 @@ message DummyDataParameter { repeated uint32 width = 5; } +// Message that stores parameters used by EltwiseLayer message EltwiseParameter { enum EltwiseOp { PROD = 0; @@ -526,6 +506,7 @@ message EltwiseParameter { optional bool stable_prod_grad = 3 [default = true]; } +// Message that stores parameters used by ExpLayer message ExpParameter { // ExpLayer computes outputs y = base ^ (shift + scale * x), for base > 0. // Or if base is set to the default (-1), base is set to e, @@ -535,18 +516,6 @@ message ExpParameter { optional float shift = 3 [default = 0.0]; } -/// Message that stores parameters used by FlattenLayer -message FlattenParameter { - // The first axis to flatten: all preceding axes are retained in the output. - // May be negative to index from the end (e.g., -1 for the last axis). - optional int32 axis = 1 [default = 1]; - - // The last axis to flatten: all following axes are retained in the output. - // May be negative to index from the end (e.g., the default -1 for the last - // axis). - optional int32 end_axis = 2 [default = -1]; -} - // Message that stores parameters used by HDF5DataLayer message HDF5DataParameter { // Specify the data source. @@ -562,6 +531,7 @@ message HDF5DataParameter { optional bool shuffle = 3 [default = false]; } +// Message that stores parameters used by HDF5OutputLayer message HDF5OutputParameter { optional string file_name = 1; } @@ -575,6 +545,7 @@ message HingeLossParameter { optional Norm norm = 1 [default = L1]; } +// Message that stores parameters used by ImageDataLayer message ImageDataParameter { // Specify the data source. optional string source = 1; @@ -606,11 +577,13 @@ message ImageDataParameter { optional string root_folder = 12 [default = ""]; } +// Message that stores parameters InfogainLossLayer message InfogainLossParameter { // Specify the infogain matrix source. optional string source = 1; } +// Message that stores parameters used by InnerProductLayer message InnerProductParameter { optional uint32 num_output = 1; // The number of outputs for the layer optional bool bias_term = 2 [default = true]; // whether to have bias terms @@ -623,16 +596,6 @@ message InnerProductParameter { optional int32 axis = 5 [default = 1]; } -// Message that stores parameters used by LogLayer -message LogParameter { - // LogLayer computes outputs y = log_base(shift + scale * x), for base > 0. - // Or if base is set to the default (-1), base is set to e, - // so y = ln(shift + scale * x) = log_e(shift + scale * x) - optional float base = 1 [default = -1.0]; - optional float scale = 2 [default = 1.0]; - optional float shift = 3 [default = 0.0]; -} - // Message that stores parameters used by LRNLayer message LRNParameter { optional uint32 local_size = 1 [default = 5]; @@ -646,6 +609,7 @@ message LRNParameter { optional float k = 5 [default = 1.]; } +// Message that stores parameters used by MemoryDataLayer message MemoryDataParameter { optional uint32 batch_size = 1; optional uint32 channels = 2; @@ -653,17 +617,16 @@ message MemoryDataParameter { optional uint32 width = 4; } +// Message that stores parameters used by MVNLayer message MVNParameter { // This parameter can be set to false to normalize mean only optional bool normalize_variance = 1 [default = true]; // This parameter can be set to true to perform DNN-like MVN optional bool across_channels = 2 [default = false]; - - // Epsilon for not dividing by zero while normalizing variance - optional float eps = 3 [default = 1e-9]; } +// Message that stores parameters used by PoolingLayer message PoolingParameter { enum PoolMethod { MAX = 0; @@ -693,6 +656,7 @@ message PoolingParameter { optional bool global_pooling = 12 [default = false]; } +// Message that stores parameters used by PowerLayer message PowerParameter { // PowerLayer computes outputs y = (shift + scale * x) ^ power. optional float power = 1 [default = 1.0]; @@ -700,40 +664,12 @@ message PowerParameter { optional float shift = 3 [default = 0.0]; } +// Message that stores parameters used by PythonLayer message PythonParameter { optional string module = 1; optional string layer = 2; } -// Message that stores parameters used by ReductionLayer -message ReductionParameter { - enum ReductionOp { - SUM = 1; - ASUM = 2; - SUMSQ = 3; - MEAN = 4; - } - - optional ReductionOp operation = 1 [default = SUM]; // reduction operation - - // The first axis to reduce to a scalar -- may be negative to index from the - // end (e.g., -1 for the last axis). - // (Currently, only reduction along ALL "tail" axes is supported; reduction - // of axis M through N, where N < num_axes - 1, is unsupported.) - // Suppose we have an n-axis bottom Blob with shape: - // (d0, d1, d2, ..., d(m-1), dm, d(m+1), ..., d(n-1)). - // If axis == m, the output Blob will have shape - // (d0, d1, d2, ..., d(m-1)), - // and the ReductionOp operation is performed (d0 * d1 * d2 * ... * d(m-1)) - // times, each including (dm * d(m+1) * ... * d(n-1)) individual data. - // If axis == 0 (the default), the output Blob always has the empty shape - // (count 1), performing reduction across the entire input -- - // often useful for creating new loss functions. - optional int32 axis = 2 [default = 0]; - - optional float coeff = 3 [default = 1.0]; // coefficient for output -} - // Message that stores parameters used by ReLULayer message ReLUParameter { // Allow non-zero slope for negative inputs to speed up optimization @@ -750,70 +686,7 @@ message ReLUParameter { optional Engine engine = 2 [default = DEFAULT]; } -message ReshapeParameter { - // Specify the output dimensions. If some of the dimensions are set to 0, - // the corresponding dimension from the bottom layer is used (unchanged). - // Exactly one dimension may be set to -1, in which case its value is - // inferred from the count of the bottom blob and the remaining dimensions. - // For example, suppose we want to reshape a 2D blob "input" with shape 2 x 8: - // - // layer { - // type: "Reshape" bottom: "input" top: "output" - // reshape_param { ... } - // } - // - // If "input" is 2D with shape 2 x 8, then the following reshape_param - // specifications are all equivalent, producing a 3D blob "output" with shape - // 2 x 2 x 4: - // - // reshape_param { shape { dim: 2 dim: 2 dim: 4 } } - // reshape_param { shape { dim: 0 dim: 2 dim: 4 } } - // reshape_param { shape { dim: 0 dim: 2 dim: -1 } } - // reshape_param { shape { dim: -1 dim: 0 dim: 2 } } - // - optional BlobShape shape = 1; - - // axis and num_axes control the portion of the bottom blob's shape that are - // replaced by (included in) the reshape. By default (axis == 0 and - // num_axes == -1), the entire bottom blob shape is included in the reshape, - // and hence the shape field must specify the entire output shape. - // - // axis may be non-zero to retain some portion of the beginning of the input - // shape (and may be negative to index from the end; e.g., -1 to begin the - // reshape after the last axis, including nothing in the reshape, - // -2 to include only the last axis, etc.). - // - // For example, suppose "input" is a 2D blob with shape 2 x 8. - // Then the following ReshapeLayer specifications are all equivalent, - // producing a blob "output" with shape 2 x 2 x 4: - // - // reshape_param { shape { dim: 2 dim: 2 dim: 4 } } - // reshape_param { shape { dim: 2 dim: 4 } axis: 1 } - // reshape_param { shape { dim: 2 dim: 4 } axis: -3 } - // - // num_axes specifies the extent of the reshape. - // If num_axes >= 0 (and axis >= 0), the reshape will be performed only on - // input axes in the range [axis, axis+num_axes]. - // num_axes may also be -1, the default, to include all remaining axes - // (starting from axis). - // - // For example, suppose "input" is a 2D blob with shape 2 x 8. - // Then the following ReshapeLayer specifications are equivalent, - // producing a blob "output" with shape 1 x 2 x 8. - // - // reshape_param { shape { dim: 1 dim: 2 dim: 8 } } - // reshape_param { shape { dim: 1 dim: 2 } num_axes: 1 } - // reshape_param { shape { dim: 1 } num_axes: 0 } - // - // On the other hand, these would produce output blob shape 2 x 1 x 8: - // - // reshape_param { shape { dim: 2 dim: 1 dim: 8 } } - // reshape_param { shape { dim: 1 } axis: 1 num_axes: 0 } - // - optional int32 axis = 2 [default = 0]; - optional int32 num_axes = 3 [default = -1]; -} - +// Message that stores parameters used by SigmoidLayer message SigmoidParameter { enum Engine { DEFAULT = 0; @@ -823,6 +696,7 @@ message SigmoidParameter { optional Engine engine = 1 [default = DEFAULT]; } +// Message that stores parameters used by SliceLayer message SliceParameter { // The axis along which to slice -- may be negative to index from the end // (e.g., -1 for the last axis). @@ -849,6 +723,7 @@ message SoftmaxParameter { optional int32 axis = 2 [default = 1]; } +// Message that stores parameters used by TanHLayer message TanHParameter { enum Engine { DEFAULT = 0; @@ -858,10 +733,12 @@ message TanHParameter { optional Engine engine = 1 [default = DEFAULT]; } +// Message that stores parameters used by ThresholdLayer message ThresholdParameter { optional float threshold = 1 [default = 0]; // Strictly positive values } +// Message that stores parameters used by WindowDataLayer message WindowDataParameter { // Specify the data source. optional string source = 1; @@ -895,22 +772,6 @@ message WindowDataParameter { optional string root_folder = 13 [default = ""]; } -message SPPParameter { - enum PoolMethod { - MAX = 0; - AVE = 1; - STOCHASTIC = 2; - } - optional uint32 pyramid_height = 1; - optional PoolMethod pool = 2 [default = MAX]; // The pooling method - enum Engine { - DEFAULT = 0; - CAFFE = 1; - CUDNN = 2; - } - optional Engine engine = 6 [default = DEFAULT]; -} - // DEPRECATED: use LayerParameter. message V1LayerParameter { repeated string bottom = 2; @@ -957,6 +818,7 @@ message V1LayerParameter { SPLIT = 22; SLICE = 33; TANH = 23; + TRIPLET_LOSS = 40; WINDOW_DATA = 24; THRESHOLD = 31; } @@ -1002,6 +864,7 @@ message V1LayerParameter { optional TransformationParameter transform_param = 36; optional LossParameter loss_param = 42; optional V0LayerParameter layer = 1; + optional TripletLossParameter triplet_loss_param = 43; } // DEPRECATED: V0LayerParameter is the old way of specifying layer parameters @@ -1098,6 +961,7 @@ message V0LayerParameter { optional HDF5OutputParameter hdf5_output_param = 1001; } +// Message that stores parameters used by PReLULayer message PReLUParameter { // Parametric ReLU described in K. He et al, Delving Deep into Rectifiers: // Surpassing Human-Level Performance on ImageNet Classification, 2015. diff --git a/src/caffe/solver.cpp b/src/caffe/solver.cpp index aabe0edec80..096980dd7af 100644 --- a/src/caffe/solver.cpp +++ b/src/caffe/solver.cpp @@ -167,26 +167,7 @@ void Solver::Step(int iters) { vector losses; Dtype smoothed_loss = 0; - while (iter_ < stop_iter) { - // zero-init the params - for (int i = 0; i < net_->params().size(); ++i) { - shared_ptr > blob = net_->params()[i]; - switch (Caffe::mode()) { - case Caffe::CPU: - caffe_set(blob->count(), static_cast(0), - blob->mutable_cpu_diff()); - break; - case Caffe::GPU: -#ifndef CPU_ONLY - caffe_gpu_set(blob->count(), static_cast(0), - blob->mutable_gpu_diff()); -#else - NO_GPU; -#endif - break; - } - } - + for (; iter_ < stop_iter; ++iter_) { if (param_.test_interval() && iter_ % param_.test_interval() == 0 && (iter_ > 0 || param_.test_initialization())) { TestAll(); @@ -194,13 +175,7 @@ void Solver::Step(int iters) { const bool display = param_.display() && iter_ % param_.display() == 0; net_->set_debug_info(display && param_.debug_info()); - // accumulate the loss and gradient - Dtype loss = 0; - for (int i = 0; i < param_.iter_size(); ++i) { - loss += net_->ForwardBackward(bottom_vec); - } - loss /= param_.iter_size(); - // average the loss across iterations for smoothed reporting + Dtype loss = net_->ForwardBackward(bottom_vec); if (losses.size() < average_loss) { losses.push_back(loss); int size = losses.size(); @@ -232,14 +207,11 @@ void Solver::Step(int iters) { } } } - ApplyUpdate(); - - // Increment the internal iter_ counter -- its value should always indicate - // the number of times the weights have been updated. - ++iter_; + ComputeUpdateValue(); + net_->Update(); // Save a snapshot if needed. - if (param_.snapshot() && iter_ % param_.snapshot() == 0) { + if (param_.snapshot() && (iter_ + 1) % param_.snapshot() == 0) { Snapshot(); } } @@ -355,14 +327,15 @@ void Solver::Snapshot() { string model_filename, snapshot_filename; const int kBufferSize = 20; char iter_str_buffer[kBufferSize]; - snprintf(iter_str_buffer, kBufferSize, "_iter_%d", iter_); + // Add one to iter_ to get the number of iterations that have completed. + snprintf(iter_str_buffer, kBufferSize, "_iter_%d", iter_ + 1); filename += iter_str_buffer; model_filename = filename + ".caffemodel"; LOG(INFO) << "Snapshotting to " << model_filename; WriteProtoToBinaryFile(net_param, model_filename.c_str()); SolverState state; SnapshotSolverState(&state); - state.set_iter(iter_); + state.set_iter(iter_ + 1); state.set_learned_net(model_filename); state.set_current_step(current_step_); snapshot_filename = filename + ".solverstate"; @@ -480,138 +453,95 @@ void SGDSolver::ClipGradients() { } template -void SGDSolver::ApplyUpdate() { +void SGDSolver::ComputeUpdateValue() { + const vector > >& net_params = this->net_->params(); + const vector& net_params_lr = this->net_->params_lr(); + const vector& net_params_weight_decay = + this->net_->params_weight_decay(); + // get the learning rate Dtype rate = GetLearningRate(); if (this->param_.display() && this->iter_ % this->param_.display() == 0) { LOG(INFO) << "Iteration " << this->iter_ << ", lr = " << rate; } ClipGradients(); - for (int param_id = 0; param_id < this->net_->params().size(); ++param_id) { - Normalize(param_id); - Regularize(param_id); - ComputeUpdateValue(param_id, rate); - } - this->net_->Update(); -} - -template -void SGDSolver::Normalize(int param_id) { - if (this->param_.iter_size() == 1) { return; } - // Scale gradient to counterbalance accumulation. - const vector > >& net_params = this->net_->params(); - const Dtype accum_normalization = Dtype(1.) / this->param_.iter_size(); - switch (Caffe::mode()) { - case Caffe::CPU: { - caffe_scal(net_params[param_id]->count(), accum_normalization, - net_params[param_id]->mutable_cpu_diff()); - break; - } - case Caffe::GPU: { -#ifndef CPU_ONLY - caffe_gpu_scal(net_params[param_id]->count(), accum_normalization, - net_params[param_id]->mutable_gpu_diff()); -#else - NO_GPU; -#endif - break; - } - default: - LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode(); - } -} - -template -void SGDSolver::Regularize(int param_id) { - const vector > >& net_params = this->net_->params(); - const vector& net_params_weight_decay = - this->net_->params_weight_decay(); + Dtype momentum = this->param_.momentum(); Dtype weight_decay = this->param_.weight_decay(); string regularization_type = this->param_.regularization_type(); - Dtype local_decay = weight_decay * net_params_weight_decay[param_id]; switch (Caffe::mode()) { - case Caffe::CPU: { - if (local_decay) { - if (regularization_type == "L2") { - // add weight decay - caffe_axpy(net_params[param_id]->count(), - local_decay, - net_params[param_id]->cpu_data(), - net_params[param_id]->mutable_cpu_diff()); - } else if (regularization_type == "L1") { - caffe_cpu_sign(net_params[param_id]->count(), - net_params[param_id]->cpu_data(), - temp_[param_id]->mutable_cpu_data()); - caffe_axpy(net_params[param_id]->count(), - local_decay, - temp_[param_id]->cpu_data(), - net_params[param_id]->mutable_cpu_diff()); - } else { - LOG(FATAL) << "Unknown regularization type: " << regularization_type; + case Caffe::CPU: + for (int param_id = 0; param_id < net_params.size(); ++param_id) { + // Compute the value to history, and then copy them to the blob's diff. + Dtype local_rate = rate * net_params_lr[param_id]; + Dtype local_decay = weight_decay * net_params_weight_decay[param_id]; + + if (local_decay) { + if (regularization_type == "L2") { + // add weight decay + caffe_axpy(net_params[param_id]->count(), + local_decay, + net_params[param_id]->cpu_data(), + net_params[param_id]->mutable_cpu_diff()); + } else if (regularization_type == "L1") { + caffe_cpu_sign(net_params[param_id]->count(), + net_params[param_id]->cpu_data(), + temp_[param_id]->mutable_cpu_data()); + caffe_axpy(net_params[param_id]->count(), + local_decay, + temp_[param_id]->cpu_data(), + net_params[param_id]->mutable_cpu_diff()); + } else { + LOG(FATAL) << "Unknown regularization type: " << regularization_type; + } } + + caffe_cpu_axpby(net_params[param_id]->count(), local_rate, + net_params[param_id]->cpu_diff(), momentum, + history_[param_id]->mutable_cpu_data()); + // copy + caffe_copy(net_params[param_id]->count(), + history_[param_id]->cpu_data(), + net_params[param_id]->mutable_cpu_diff()); } break; - } - case Caffe::GPU: { + case Caffe::GPU: #ifndef CPU_ONLY - if (local_decay) { - if (regularization_type == "L2") { - // add weight decay - caffe_gpu_axpy(net_params[param_id]->count(), - local_decay, - net_params[param_id]->gpu_data(), - net_params[param_id]->mutable_gpu_diff()); - } else if (regularization_type == "L1") { - caffe_gpu_sign(net_params[param_id]->count(), - net_params[param_id]->gpu_data(), - temp_[param_id]->mutable_gpu_data()); - caffe_gpu_axpy(net_params[param_id]->count(), - local_decay, - temp_[param_id]->gpu_data(), - net_params[param_id]->mutable_gpu_diff()); - } else { - LOG(FATAL) << "Unknown regularization type: " << regularization_type; + for (int param_id = 0; param_id < net_params.size(); ++param_id) { + // Compute the value to history, and then copy them to the blob's diff. + Dtype local_rate = rate * net_params_lr[param_id]; + Dtype local_decay = weight_decay * net_params_weight_decay[param_id]; + + if (local_decay) { + if (regularization_type == "L2") { + // add weight decay + caffe_gpu_axpy(net_params[param_id]->count(), + local_decay, + net_params[param_id]->gpu_data(), + net_params[param_id]->mutable_gpu_diff()); + } else if (regularization_type == "L1") { + caffe_gpu_sign(net_params[param_id]->count(), + net_params[param_id]->gpu_data(), + temp_[param_id]->mutable_gpu_data()); + caffe_gpu_axpy(net_params[param_id]->count(), + local_decay, + temp_[param_id]->gpu_data(), + net_params[param_id]->mutable_gpu_diff()); + } else { + LOG(FATAL) << "Unknown regularization type: " << regularization_type; + } } - } -#else - NO_GPU; -#endif - break; - } - default: - LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode(); - } -} -template -void SGDSolver::ComputeUpdateValue(int param_id, Dtype rate) { - const vector > >& net_params = this->net_->params(); - const vector& net_params_lr = this->net_->params_lr(); - Dtype momentum = this->param_.momentum(); - Dtype local_rate = rate * net_params_lr[param_id]; - // Compute the update to history, then copy it to the parameter diff. - switch (Caffe::mode()) { - case Caffe::CPU: { - caffe_cpu_axpby(net_params[param_id]->count(), local_rate, - net_params[param_id]->cpu_diff(), momentum, - history_[param_id]->mutable_cpu_data()); - caffe_copy(net_params[param_id]->count(), - history_[param_id]->cpu_data(), - net_params[param_id]->mutable_cpu_diff()); - break; - } - case Caffe::GPU: { -#ifndef CPU_ONLY - caffe_gpu_axpby(net_params[param_id]->count(), local_rate, - net_params[param_id]->gpu_diff(), momentum, - history_[param_id]->mutable_gpu_data()); - caffe_copy(net_params[param_id]->count(), - history_[param_id]->gpu_data(), - net_params[param_id]->mutable_gpu_diff()); + caffe_gpu_axpby(net_params[param_id]->count(), local_rate, + net_params[param_id]->gpu_diff(), momentum, + history_[param_id]->mutable_gpu_data()); + // copy + caffe_copy(net_params[param_id]->count(), + history_[param_id]->gpu_data(), + net_params[param_id]->mutable_gpu_diff()); + } #else NO_GPU; #endif break; - } default: LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode(); } @@ -638,138 +568,252 @@ void SGDSolver::RestoreSolverState(const SolverState& state) { } template -void NesterovSolver::ComputeUpdateValue(int param_id, Dtype rate) { +void NesterovSolver::ComputeUpdateValue() { const vector > >& net_params = this->net_->params(); const vector& net_params_lr = this->net_->params_lr(); + const vector& net_params_weight_decay = + this->net_->params_weight_decay(); + // get the learning rate + Dtype rate = this->GetLearningRate(); + if (this->param_.display() && this->iter_ % this->param_.display() == 0) { + LOG(INFO) << "Iteration " << this->iter_ << ", lr = " << rate; + } + SGDSolver::ClipGradients(); Dtype momentum = this->param_.momentum(); - Dtype local_rate = rate * net_params_lr[param_id]; + Dtype weight_decay = this->param_.weight_decay(); + string regularization_type = this->param_.regularization_type(); switch (Caffe::mode()) { - case Caffe::CPU: { - // save history momentum for stepping back - caffe_copy(net_params[param_id]->count(), - this->history_[param_id]->cpu_data(), - this->update_[param_id]->mutable_cpu_data()); - - // update history - caffe_cpu_axpby(net_params[param_id]->count(), local_rate, - net_params[param_id]->cpu_diff(), momentum, - this->history_[param_id]->mutable_cpu_data()); - - // compute update: step back then over step - caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) + momentum, - this->history_[param_id]->cpu_data(), -momentum, - this->update_[param_id]->mutable_cpu_data()); - - // copy - caffe_copy(net_params[param_id]->count(), - this->update_[param_id]->cpu_data(), - net_params[param_id]->mutable_cpu_diff()); + case Caffe::CPU: + for (int param_id = 0; param_id < net_params.size(); ++param_id) { + // save history momentum for stepping back + caffe_copy(net_params[param_id]->count(), + this->history_[param_id]->cpu_data(), + this->update_[param_id]->mutable_cpu_data()); + + Dtype local_rate = rate * net_params_lr[param_id]; + Dtype local_decay = weight_decay * net_params_weight_decay[param_id]; + + if (local_decay) { + if (regularization_type == "L2") { + // add weight decay + caffe_axpy(net_params[param_id]->count(), + local_decay, + net_params[param_id]->cpu_data(), + net_params[param_id]->mutable_cpu_diff()); + } else if (regularization_type == "L1") { + caffe_cpu_sign(net_params[param_id]->count(), + net_params[param_id]->cpu_data(), + this->temp_[param_id]->mutable_cpu_data()); + caffe_axpy(net_params[param_id]->count(), + local_decay, + this->temp_[param_id]->cpu_data(), + net_params[param_id]->mutable_cpu_diff()); + } else { + LOG(FATAL) << "Unknown regularization type: " << regularization_type; + } + } + + // update history + caffe_cpu_axpby(net_params[param_id]->count(), local_rate, + net_params[param_id]->cpu_diff(), momentum, + this->history_[param_id]->mutable_cpu_data()); + + // compute udpate: step back then over step + caffe_cpu_axpby(net_params[param_id]->count(), Dtype(1) + momentum, + this->history_[param_id]->cpu_data(), -momentum, + this->update_[param_id]->mutable_cpu_data()); + + // copy + caffe_copy(net_params[param_id]->count(), + this->update_[param_id]->cpu_data(), + net_params[param_id]->mutable_cpu_diff()); + } break; - } - case Caffe::GPU: { + case Caffe::GPU: #ifndef CPU_ONLY - // save history momentum for stepping back - caffe_copy(net_params[param_id]->count(), - this->history_[param_id]->gpu_data(), - this->update_[param_id]->mutable_gpu_data()); - - // update history - caffe_gpu_axpby(net_params[param_id]->count(), local_rate, - net_params[param_id]->gpu_diff(), momentum, - this->history_[param_id]->mutable_gpu_data()); - - // compute update: step back then over step - caffe_gpu_axpby(net_params[param_id]->count(), Dtype(1) + momentum, - this->history_[param_id]->gpu_data(), -momentum, - this->update_[param_id]->mutable_gpu_data()); - - // copy - caffe_copy(net_params[param_id]->count(), - this->update_[param_id]->gpu_data(), - net_params[param_id]->mutable_gpu_diff()); + for (int param_id = 0; param_id < net_params.size(); ++param_id) { + // save history momentum for stepping back + caffe_copy(net_params[param_id]->count(), + this->history_[param_id]->gpu_data(), + this->update_[param_id]->mutable_gpu_data()); + + Dtype local_rate = rate * net_params_lr[param_id]; + Dtype local_decay = weight_decay * net_params_weight_decay[param_id]; + + if (local_decay) { + if (regularization_type == "L2") { + // add weight decay + caffe_gpu_axpy(net_params[param_id]->count(), + local_decay, + net_params[param_id]->gpu_data(), + net_params[param_id]->mutable_gpu_diff()); + } else if (regularization_type == "L1") { + caffe_gpu_sign(net_params[param_id]->count(), + net_params[param_id]->gpu_data(), + this->temp_[param_id]->mutable_gpu_data()); + caffe_gpu_axpy(net_params[param_id]->count(), + local_decay, + this->temp_[param_id]->gpu_data(), + net_params[param_id]->mutable_gpu_diff()); + } else { + LOG(FATAL) << "Unknown regularization type: " << regularization_type; + } + } + + // update history + caffe_gpu_axpby(net_params[param_id]->count(), local_rate, + net_params[param_id]->gpu_diff(), momentum, + this->history_[param_id]->mutable_gpu_data()); + + // compute udpate: step back then over step + caffe_gpu_axpby(net_params[param_id]->count(), Dtype(1) + momentum, + this->history_[param_id]->gpu_data(), -momentum, + this->update_[param_id]->mutable_gpu_data()); + + // copy + caffe_copy(net_params[param_id]->count(), + this->update_[param_id]->gpu_data(), + net_params[param_id]->mutable_gpu_diff()); + } #else NO_GPU; #endif break; - } default: LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode(); } } template -void AdaGradSolver::ComputeUpdateValue(int param_id, Dtype rate) { +void AdaGradSolver::ComputeUpdateValue() { const vector > >& net_params = this->net_->params(); const vector& net_params_lr = this->net_->params_lr(); + const vector& net_params_weight_decay = + this->net_->params_weight_decay(); + // get the learning rate + Dtype rate = this->GetLearningRate(); Dtype delta = this->param_.delta(); - Dtype local_rate = rate * net_params_lr[param_id]; + if (this->param_.display() && this->iter_ % this->param_.display() == 0) { + LOG(INFO) << "Iteration " << this->iter_ << ", lr = " << rate; + } + SGDSolver::ClipGradients(); + Dtype weight_decay = this->param_.weight_decay(); + string regularization_type = this->param_.regularization_type(); switch (Caffe::mode()) { - case Caffe::CPU: { - // compute square of gradient in update - caffe_powx(net_params[param_id]->count(), - net_params[param_id]->cpu_diff(), Dtype(2), - this->update_[param_id]->mutable_cpu_data()); - - // update history - caffe_add(net_params[param_id]->count(), - this->update_[param_id]->cpu_data(), - this->history_[param_id]->cpu_data(), - this->history_[param_id]->mutable_cpu_data()); - - // prepare update - caffe_powx(net_params[param_id]->count(), - this->history_[param_id]->cpu_data(), Dtype(0.5), - this->update_[param_id]->mutable_cpu_data()); - - caffe_add_scalar(net_params[param_id]->count(), - delta, this->update_[param_id]->mutable_cpu_data()); - - caffe_div(net_params[param_id]->count(), - net_params[param_id]->cpu_diff(), - this->update_[param_id]->cpu_data(), - this->update_[param_id]->mutable_cpu_data()); - - // scale and copy - caffe_cpu_axpby(net_params[param_id]->count(), local_rate, - this->update_[param_id]->cpu_data(), Dtype(0), - net_params[param_id]->mutable_cpu_diff()); + case Caffe::CPU: + for (int param_id = 0; param_id < net_params.size(); ++param_id) { + Dtype local_rate = rate * net_params_lr[param_id]; + Dtype local_decay = weight_decay * net_params_weight_decay[param_id]; + + if (local_decay) { + if (regularization_type == "L2") { + // add weight decay + caffe_axpy(net_params[param_id]->count(), + local_decay, + net_params[param_id]->cpu_data(), + net_params[param_id]->mutable_cpu_diff()); + } else if (regularization_type == "L1") { + caffe_cpu_sign(net_params[param_id]->count(), + net_params[param_id]->cpu_data(), + this->temp_[param_id]->mutable_cpu_data()); + caffe_axpy(net_params[param_id]->count(), + local_decay, + this->temp_[param_id]->cpu_data(), + net_params[param_id]->mutable_cpu_diff()); + } else { + LOG(FATAL) << "Unknown regularization type: " << regularization_type; + } + } + + // compute square of gradient in update + caffe_powx(net_params[param_id]->count(), + net_params[param_id]->cpu_diff(), Dtype(2), + this->update_[param_id]->mutable_cpu_data()); + + // update history + caffe_add(net_params[param_id]->count(), + this->update_[param_id]->cpu_data(), + this->history_[param_id]->cpu_data(), + this->history_[param_id]->mutable_cpu_data()); + + // prepare update + caffe_powx(net_params[param_id]->count(), + this->history_[param_id]->cpu_data(), Dtype(0.5), + this->update_[param_id]->mutable_cpu_data()); + + caffe_add_scalar(net_params[param_id]->count(), + delta, this->update_[param_id]->mutable_cpu_data()); + + caffe_div(net_params[param_id]->count(), + net_params[param_id]->cpu_diff(), + this->update_[param_id]->cpu_data(), + this->update_[param_id]->mutable_cpu_data()); + + // scale and copy + caffe_cpu_axpby(net_params[param_id]->count(), local_rate, + this->update_[param_id]->cpu_data(), Dtype(0), + net_params[param_id]->mutable_cpu_diff()); + } break; - } - case Caffe::GPU: { + case Caffe::GPU: #ifndef CPU_ONLY - // compute square of gradient in update - caffe_gpu_powx(net_params[param_id]->count(), - net_params[param_id]->gpu_diff(), Dtype(2), - this->update_[param_id]->mutable_gpu_data()); - - // update history - caffe_gpu_add(net_params[param_id]->count(), - this->update_[param_id]->gpu_data(), - this->history_[param_id]->gpu_data(), - this->history_[param_id]->mutable_gpu_data()); - - // prepare update - caffe_gpu_powx(net_params[param_id]->count(), - this->history_[param_id]->gpu_data(), Dtype(0.5), - this->update_[param_id]->mutable_gpu_data()); - - caffe_gpu_add_scalar(net_params[param_id]->count(), - delta, this->update_[param_id]->mutable_gpu_data()); - - caffe_gpu_div(net_params[param_id]->count(), - net_params[param_id]->gpu_diff(), - this->update_[param_id]->gpu_data(), - this->update_[param_id]->mutable_gpu_data()); - - // scale and copy - caffe_gpu_axpby(net_params[param_id]->count(), local_rate, - this->update_[param_id]->gpu_data(), Dtype(0), - net_params[param_id]->mutable_gpu_diff()); + for (int param_id = 0; param_id < net_params.size(); ++param_id) { + Dtype local_rate = rate * net_params_lr[param_id]; + Dtype local_decay = weight_decay * net_params_weight_decay[param_id]; + + if (local_decay) { + if (regularization_type == "L2") { + // add weight decay + caffe_gpu_axpy(net_params[param_id]->count(), + local_decay, + net_params[param_id]->gpu_data(), + net_params[param_id]->mutable_gpu_diff()); + } else if (regularization_type == "L1") { + caffe_gpu_sign(net_params[param_id]->count(), + net_params[param_id]->gpu_data(), + this->temp_[param_id]->mutable_gpu_data()); + caffe_gpu_axpy(net_params[param_id]->count(), + local_decay, + this->temp_[param_id]->gpu_data(), + net_params[param_id]->mutable_gpu_diff()); + } else { + LOG(FATAL) << "Unknown regularization type: " << regularization_type; + } + } + + // compute square of gradient in update + caffe_gpu_powx(net_params[param_id]->count(), + net_params[param_id]->gpu_diff(), Dtype(2), + this->update_[param_id]->mutable_gpu_data()); + + // update history + caffe_gpu_add(net_params[param_id]->count(), + this->update_[param_id]->gpu_data(), + this->history_[param_id]->gpu_data(), + this->history_[param_id]->mutable_gpu_data()); + + // prepare update + caffe_gpu_powx(net_params[param_id]->count(), + this->history_[param_id]->gpu_data(), Dtype(0.5), + this->update_[param_id]->mutable_gpu_data()); + + caffe_gpu_add_scalar(net_params[param_id]->count(), + delta, this->update_[param_id]->mutable_gpu_data()); + + caffe_gpu_div(net_params[param_id]->count(), + net_params[param_id]->gpu_diff(), + this->update_[param_id]->gpu_data(), + this->update_[param_id]->mutable_gpu_data()); + + // scale and copy + caffe_gpu_axpby(net_params[param_id]->count(), local_rate, + this->update_[param_id]->gpu_data(), Dtype(0), + net_params[param_id]->mutable_gpu_diff()); + } #else NO_GPU; #endif break; - } default: LOG(FATAL) << "Unknown caffe mode: " << Caffe::mode(); } diff --git a/src/caffe/test/test_accuracy_layer.cpp b/src/caffe/test/test_accuracy_layer.cpp index c14b67cc0e9..6cbf51df45e 100644 --- a/src/caffe/test/test_accuracy_layer.cpp +++ b/src/caffe/test/test_accuracy_layer.cpp @@ -16,7 +16,7 @@ namespace caffe { template -class AccuracyLayerTest : public CPUDeviceTest { +class AccuracyLayerTest : public ::testing::Test { protected: AccuracyLayerTest() : blob_bottom_data_(new Blob()), @@ -92,6 +92,7 @@ TYPED_TEST(AccuracyLayerTest, TestSetupTopK) { TYPED_TEST(AccuracyLayerTest, TestForwardCPU) { LayerParameter layer_param; + Caffe::set_mode(Caffe::CPU); AccuracyLayer layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); @@ -117,6 +118,7 @@ TYPED_TEST(AccuracyLayerTest, TestForwardCPU) { } TYPED_TEST(AccuracyLayerTest, TestForwardWithSpatialAxes) { + Caffe::set_mode(Caffe::CPU); this->blob_bottom_data_->Reshape(2, 10, 4, 5); vector label_shape(3); label_shape[0] = 2; label_shape[1] = 4; label_shape[2] = 5; @@ -160,6 +162,7 @@ TYPED_TEST(AccuracyLayerTest, TestForwardWithSpatialAxes) { } TYPED_TEST(AccuracyLayerTest, TestForwardIgnoreLabel) { + Caffe::set_mode(Caffe::CPU); LayerParameter layer_param; const TypeParam kIgnoreLabelValue = -1; layer_param.mutable_accuracy_param()->set_ignore_label(kIgnoreLabelValue); diff --git a/src/caffe/test/test_argmax_layer.cpp b/src/caffe/test/test_argmax_layer.cpp index 895c3d372ff..3487d42f21e 100644 --- a/src/caffe/test/test_argmax_layer.cpp +++ b/src/caffe/test/test_argmax_layer.cpp @@ -13,12 +13,13 @@ namespace caffe { template -class ArgMaxLayerTest : public CPUDeviceTest { +class ArgMaxLayerTest : public ::testing::Test { protected: ArgMaxLayerTest() : blob_bottom_(new Blob(10, 20, 1, 1)), blob_top_(new Blob()), top_k_(5) { + Caffe::set_mode(Caffe::CPU); Caffe::set_random_seed(1701); // fill the values FillerParameter filler_param; diff --git a/src/caffe/test/test_contrastive_loss_layer.cpp b/src/caffe/test/test_contrastive_loss_layer.cpp index 1e9447cbc51..d269fbc26f2 100644 --- a/src/caffe/test/test_contrastive_loss_layer.cpp +++ b/src/caffe/test/test_contrastive_loss_layer.cpp @@ -22,15 +22,15 @@ class ContrastiveLossLayerTest : public MultiDeviceTest { protected: ContrastiveLossLayerTest() - : blob_bottom_data_i_(new Blob(512, 2, 1, 1)), - blob_bottom_data_j_(new Blob(512, 2, 1, 1)), - blob_bottom_y_(new Blob(512, 1, 1, 1)), + : blob_bottom_data_i_(new Blob(128, 10, 1, 1)), + blob_bottom_data_j_(new Blob(128, 10, 1, 1)), + blob_bottom_y_(new Blob(128, 1, 1, 1)), blob_top_loss_(new Blob()) { // fill the values FillerParameter filler_param; - filler_param.set_min(-1.0); - filler_param.set_max(1.0); // distances~=1.0 to test both sides of margin - UniformFiller filler(filler_param); + filler_param.set_mean(0.0); + filler_param.set_std(0.3); // distances~=1.0 to test both sides of margin + GaussianFiller filler(filler_param); filler.Fill(this->blob_bottom_data_i_); blob_bottom_vec_.push_back(blob_bottom_data_i_); filler.Fill(this->blob_bottom_data_j_); @@ -79,8 +79,7 @@ TYPED_TEST(ContrastiveLossLayerTest, TestForward) { if (this->blob_bottom_y_->cpu_data()[i]) { // similar pairs loss += dist_sq; } else { - Dtype dist = std::max(margin - sqrt(dist_sq), 0.0); - loss += dist*dist; + loss += std::max(margin-dist_sq, Dtype(0)); } } loss /= static_cast(num) * Dtype(2); @@ -100,47 +99,4 @@ TYPED_TEST(ContrastiveLossLayerTest, TestGradient) { this->blob_top_vec_, 1); } -TYPED_TEST(ContrastiveLossLayerTest, TestForwardLegacy) { - typedef typename TypeParam::Dtype Dtype; - LayerParameter layer_param; - layer_param.mutable_contrastive_loss_param()->set_legacy_version(true); - ContrastiveLossLayer layer(layer_param); - layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); - layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); - // manually compute to compare - const Dtype margin = layer_param.contrastive_loss_param().margin(); - const int num = this->blob_bottom_data_i_->num(); - const int channels = this->blob_bottom_data_i_->channels(); - Dtype loss(0); - for (int i = 0; i < num; ++i) { - Dtype dist_sq(0); - for (int j = 0; j < channels; ++j) { - Dtype diff = this->blob_bottom_data_i_->cpu_data()[i*channels+j] - - this->blob_bottom_data_j_->cpu_data()[i*channels+j]; - dist_sq += diff*diff; - } - if (this->blob_bottom_y_->cpu_data()[i]) { // similar pairs - loss += dist_sq; - } else { - loss += std::max(margin - dist_sq, Dtype(0.0)); - } - } - loss /= static_cast(num) * Dtype(2); - EXPECT_NEAR(this->blob_top_loss_->cpu_data()[0], loss, 1e-6); -} - -TYPED_TEST(ContrastiveLossLayerTest, TestGradientLegacy) { - typedef typename TypeParam::Dtype Dtype; - LayerParameter layer_param; - layer_param.mutable_contrastive_loss_param()->set_legacy_version(true); - ContrastiveLossLayer layer(layer_param); - layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); - GradientChecker checker(1e-2, 1e-2, 1701); - // check the gradient for the first two bottom layers - checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, - this->blob_top_vec_, 0); - checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, - this->blob_top_vec_, 1); -} - } // namespace caffe diff --git a/src/caffe/test/test_convolution_layer.cpp b/src/caffe/test/test_convolution_layer.cpp index 67d41fff844..c1fe3b58c58 100644 --- a/src/caffe/test/test_convolution_layer.cpp +++ b/src/caffe/test/test_convolution_layer.cpp @@ -424,7 +424,7 @@ TYPED_TEST(ConvolutionLayerTest, TestGradientGroup) { #ifdef USE_CUDNN template -class CuDNNConvolutionLayerTest : public GPUDeviceTest { +class CuDNNConvolutionLayerTest : public ::testing::Test { protected: CuDNNConvolutionLayerTest() : blob_bottom_(new Blob(2, 3, 6, 4)), @@ -467,6 +467,7 @@ class CuDNNConvolutionLayerTest : public GPUDeviceTest { TYPED_TEST_CASE(CuDNNConvolutionLayerTest, TestDtypes); TYPED_TEST(CuDNNConvolutionLayerTest, TestSetupCuDNN) { + Caffe::set_mode(Caffe::GPU); this->blob_bottom_vec_.push_back(this->blob_bottom_2_); this->blob_top_vec_.push_back(this->blob_top_2_); LayerParameter layer_param; @@ -504,6 +505,7 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestSetupCuDNN) { } TYPED_TEST(CuDNNConvolutionLayerTest, TestSimpleConvolutionCuDNN) { + Caffe::set_mode(Caffe::GPU); this->blob_bottom_vec_.push_back(this->blob_bottom_2_); this->blob_top_vec_.push_back(this->blob_top_2_); LayerParameter layer_param; @@ -539,6 +541,7 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestSimpleConvolutionCuDNN) { } TYPED_TEST(CuDNNConvolutionLayerTest, TestSimpleConvolutionGroupCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; ConvolutionParameter* convolution_param = layer_param.mutable_convolution_param(); @@ -569,7 +572,7 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestSobelConvolutionCuDNN) { // Test separable convolution by computing the Sobel operator // as a single filter then comparing the result // as the convolution of two rectangular filters. - + Caffe::set_mode(Caffe::GPU); // Fill bottoms with identical Gaussian noise. shared_ptr > filler; FillerParameter filler_param; @@ -662,6 +665,7 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestSobelConvolutionCuDNN) { } TYPED_TEST(CuDNNConvolutionLayerTest, TestGradientCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; ConvolutionParameter* convolution_param = layer_param.mutable_convolution_param(); @@ -679,6 +683,7 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestGradientCuDNN) { } TYPED_TEST(CuDNNConvolutionLayerTest, TestGradientGroupCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; ConvolutionParameter* convolution_param = layer_param.mutable_convolution_param(); diff --git a/src/caffe/test/test_data/generate_sample_data.py b/src/caffe/test/test_data/generate_sample_data.py index ab5572685cb..e5dbc3406d8 100644 --- a/src/caffe/test/test_data/generate_sample_data.py +++ b/src/caffe/test/test_data/generate_sample_data.py @@ -5,8 +5,6 @@ import numpy as np import h5py -script_dir = os.path.dirname(os.path.abspath(__file__)) - num_cols = 8 num_rows = 10 height = 6 @@ -29,12 +27,12 @@ print data print label -with h5py.File(script_dir + '/sample_data.h5', 'w') as f: +with h5py.File(os.path.dirname(__file__) + '/sample_data.h5', 'w') as f: f['data'] = data f['label'] = label f['label2'] = label2 -with h5py.File(script_dir + '/sample_data_2_gzip.h5', 'w') as f: +with h5py.File(os.path.dirname(__file__) + '/sample_data_2_gzip.h5', 'w') as f: f.create_dataset( 'data', data=data + total_size, compression='gzip', compression_opts=1 @@ -48,6 +46,6 @@ compression='gzip', compression_opts=1 ) -with open(script_dir + '/sample_data_list.txt', 'w') as f: - f.write(script_dir + '/sample_data.h5\n') - f.write(script_dir + '/sample_data_2_gzip.h5\n') +with open(os.path.dirname(__file__) + '/sample_data_list.txt', 'w') as f: + f.write(os.path.dirname(__file__) + '/sample_data.h5\n') + f.write(os.path.dirname(__file__) + '/sample_data_2_gzip.h5\n') diff --git a/src/caffe/test/test_dummy_data_layer.cpp b/src/caffe/test/test_dummy_data_layer.cpp index c9ed38db3a5..99548352746 100644 --- a/src/caffe/test/test_dummy_data_layer.cpp +++ b/src/caffe/test/test_dummy_data_layer.cpp @@ -13,7 +13,7 @@ namespace caffe { template -class DummyDataLayerTest : public CPUDeviceTest { +class DummyDataLayerTest : public ::testing::Test { protected: DummyDataLayerTest() : blob_top_a_(new Blob()), @@ -44,6 +44,7 @@ class DummyDataLayerTest : public CPUDeviceTest { TYPED_TEST_CASE(DummyDataLayerTest, TestDtypes); TYPED_TEST(DummyDataLayerTest, TestOneTopConstant) { + Caffe::set_mode(Caffe::CPU); LayerParameter param; DummyDataParameter* dummy_data_param = param.mutable_dummy_data_param(); dummy_data_param->add_num(5); @@ -73,6 +74,7 @@ TYPED_TEST(DummyDataLayerTest, TestOneTopConstant) { } TYPED_TEST(DummyDataLayerTest, TestTwoTopConstant) { + Caffe::set_mode(Caffe::CPU); LayerParameter param; DummyDataParameter* dummy_data_param = param.mutable_dummy_data_param(); dummy_data_param->add_num(5); @@ -111,6 +113,7 @@ TYPED_TEST(DummyDataLayerTest, TestTwoTopConstant) { } TYPED_TEST(DummyDataLayerTest, TestThreeTopConstantGaussianConstant) { + Caffe::set_mode(Caffe::CPU); LayerParameter param; DummyDataParameter* dummy_data_param = param.mutable_dummy_data_param(); dummy_data_param->add_num(5); diff --git a/src/caffe/test/test_filler.cpp b/src/caffe/test/test_filler.cpp index 728b8dc5f0d..e04b0fd22af 100644 --- a/src/caffe/test/test_filler.cpp +++ b/src/caffe/test/test_filler.cpp @@ -142,102 +142,4 @@ TYPED_TEST(GaussianFillerTest, TestFill) { EXPECT_LE(var, target_var * 5.); } -template -class XavierFillerTest : public ::testing::Test { - protected: - XavierFillerTest() - : blob_(new Blob(1000, 2, 4, 5)), - filler_param_() { - } - virtual void test_params(FillerParameter_VarianceNorm variance_norm, - Dtype n) { - this->filler_param_.set_variance_norm(variance_norm); - this->filler_.reset(new XavierFiller(this->filler_param_)); - this->filler_->Fill(blob_); - EXPECT_TRUE(this->blob_); - const int count = this->blob_->count(); - const Dtype* data = this->blob_->cpu_data(); - Dtype mean = 0.; - Dtype ex2 = 0.; - for (int i = 0; i < count; ++i) { - mean += data[i]; - ex2 += data[i] * data[i]; - } - mean /= count; - ex2 /= count; - Dtype std = sqrt(ex2 - mean*mean); - Dtype target_std = sqrt(2.0 / n); - EXPECT_NEAR(mean, 0.0, 0.1); - EXPECT_NEAR(std, target_std, 0.1); - } - virtual ~XavierFillerTest() { delete blob_; } - Blob* const blob_; - FillerParameter filler_param_; - shared_ptr > filler_; -}; - -TYPED_TEST_CASE(XavierFillerTest, TestDtypes); - -TYPED_TEST(XavierFillerTest, TestFillFanIn) { - TypeParam n = 2*4*5; - this->test_params(FillerParameter_VarianceNorm_FAN_IN, n); -} -TYPED_TEST(XavierFillerTest, TestFillFanOut) { - TypeParam n = 1000*4*5; - this->test_params(FillerParameter_VarianceNorm_FAN_OUT, n); -} -TYPED_TEST(XavierFillerTest, TestFillAverage) { - TypeParam n = (2*4*5 + 1000*4*5) / 2.0; - this->test_params(FillerParameter_VarianceNorm_AVERAGE, n); -} - -template -class MSRAFillerTest : public ::testing::Test { - protected: - MSRAFillerTest() - : blob_(new Blob(1000, 2, 4, 5)), - filler_param_() { - } - virtual void test_params(FillerParameter_VarianceNorm variance_norm, - Dtype n) { - this->filler_param_.set_variance_norm(variance_norm); - this->filler_.reset(new MSRAFiller(this->filler_param_)); - this->filler_->Fill(blob_); - EXPECT_TRUE(this->blob_); - const int count = this->blob_->count(); - const Dtype* data = this->blob_->cpu_data(); - Dtype mean = 0.; - Dtype ex2 = 0.; - for (int i = 0; i < count; ++i) { - mean += data[i]; - ex2 += data[i] * data[i]; - } - mean /= count; - ex2 /= count; - Dtype std = sqrt(ex2 - mean*mean); - Dtype target_std = sqrt(2.0 / n); - EXPECT_NEAR(mean, 0.0, 0.1); - EXPECT_NEAR(std, target_std, 0.1); - } - virtual ~MSRAFillerTest() { delete blob_; } - Blob* const blob_; - FillerParameter filler_param_; - shared_ptr > filler_; -}; - -TYPED_TEST_CASE(MSRAFillerTest, TestDtypes); - -TYPED_TEST(MSRAFillerTest, TestFillFanIn) { - TypeParam n = 2*4*5; - this->test_params(FillerParameter_VarianceNorm_FAN_IN, n); -} -TYPED_TEST(MSRAFillerTest, TestFillFanOut) { - TypeParam n = 1000*4*5; - this->test_params(FillerParameter_VarianceNorm_FAN_OUT, n); -} -TYPED_TEST(MSRAFillerTest, TestFillAverage) { - TypeParam n = (2*4*5 + 1000*4*5) / 2.0; - this->test_params(FillerParameter_VarianceNorm_AVERAGE, n); -} - } // namespace caffe diff --git a/src/caffe/test/test_flatten_layer.cpp b/src/caffe/test/test_flatten_layer.cpp index 7b6757cba32..3042d293cf7 100644 --- a/src/caffe/test/test_flatten_layer.cpp +++ b/src/caffe/test/test_flatten_layer.cpp @@ -42,48 +42,13 @@ TYPED_TEST(FlattenLayerTest, TestSetup) { LayerParameter layer_param; FlattenLayer layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); - ASSERT_EQ(this->blob_top_->num_axes(), 2); - EXPECT_EQ(this->blob_top_->shape(0), 2); - EXPECT_EQ(this->blob_top_->shape(1), 3 * 6 * 5); + EXPECT_EQ(this->blob_top_->num(), 2); + EXPECT_EQ(this->blob_top_->channels(), 3 * 6 * 5); + EXPECT_EQ(this->blob_top_->height(), 1); + EXPECT_EQ(this->blob_top_->width(), 1); } -TYPED_TEST(FlattenLayerTest, TestSetupWithAxis) { - typedef typename TypeParam::Dtype Dtype; - LayerParameter layer_param; - layer_param.mutable_flatten_param()->set_axis(2); - FlattenLayer layer(layer_param); - layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); - ASSERT_EQ(this->blob_top_->num_axes(), 3); - EXPECT_EQ(this->blob_top_->shape(0), 2); - EXPECT_EQ(this->blob_top_->shape(1), 3); - EXPECT_EQ(this->blob_top_->shape(2), 6 * 5); -} - -TYPED_TEST(FlattenLayerTest, TestSetupWithEndAxis) { - typedef typename TypeParam::Dtype Dtype; - LayerParameter layer_param; - layer_param.mutable_flatten_param()->set_end_axis(-2); - FlattenLayer layer(layer_param); - layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); - ASSERT_EQ(this->blob_top_->num_axes(), 3); - EXPECT_EQ(this->blob_top_->shape(0), 2); - EXPECT_EQ(this->blob_top_->shape(1), 3 * 6); - EXPECT_EQ(this->blob_top_->shape(2), 5); -} - -TYPED_TEST(FlattenLayerTest, TestSetupWithStartAndEndAxis) { - typedef typename TypeParam::Dtype Dtype; - LayerParameter layer_param; - layer_param.mutable_flatten_param()->set_axis(0); - layer_param.mutable_flatten_param()->set_end_axis(-2); - FlattenLayer layer(layer_param); - layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); - ASSERT_EQ(this->blob_top_->num_axes(), 2); - EXPECT_EQ(this->blob_top_->shape(0), 2 * 3 * 6); - EXPECT_EQ(this->blob_top_->shape(1), 5); -} - -TYPED_TEST(FlattenLayerTest, TestForward) { +TYPED_TEST(FlattenLayerTest, Test) { typedef typename TypeParam::Dtype Dtype; LayerParameter layer_param; FlattenLayer layer(layer_param); @@ -106,4 +71,5 @@ TYPED_TEST(FlattenLayerTest, TestGradient) { this->blob_top_vec_); } + } // namespace caffe diff --git a/src/caffe/test/test_gradient_based_solver.cpp b/src/caffe/test/test_gradient_based_solver.cpp index c9135d64e70..eb2569c04f2 100644 --- a/src/caffe/test/test_gradient_based_solver.cpp +++ b/src/caffe/test/test_gradient_based_solver.cpp @@ -23,7 +23,7 @@ class GradientBasedSolverTest : public MultiDeviceTest { protected: GradientBasedSolverTest() : - seed_(1701), num_(4), channels_(3), height_(10), width_(10) {} + seed_(1701), num_(5), channels_(3), height_(10), width_(10) {} shared_ptr > solver_; int seed_; @@ -56,21 +56,19 @@ class GradientBasedSolverTest : public MultiDeviceTest { } void RunLeastSquaresSolver(const Dtype learning_rate, - const Dtype weight_decay, const Dtype momentum, const int num_iters, - const int iter_size = 1) { + const Dtype weight_decay, const Dtype momentum, const int num_iters) { ostringstream proto; proto << "max_iter: " << num_iters << " " "base_lr: " << learning_rate << " " "lr_policy: 'fixed' " - "iter_size: " << iter_size << " " "net_param { " " name: 'TestNetwork' " " layer { " " name: 'data' " " type: 'DummyData' " " dummy_data_param { " - " num: " << num_ / iter_size << " " + " num: " << num_ << " " " channels: " << channels_ << " " " height: " << height_ << " " " width: " << width_ << " " @@ -78,10 +76,6 @@ class GradientBasedSolverTest : public MultiDeviceTest { " height: 1 " " width: 1 " " data_filler { " - " type: 'constant' " - " value: 1.0 " - " } " - " data_filler { " " type: 'gaussian' " " std: 1.0 " " } " @@ -276,45 +270,6 @@ class GradientBasedSolverTest : public MultiDeviceTest { } } - void CheckAccumulation(const Dtype kLearningRate, const Dtype kWeightDecay, - const Dtype kMomentum, const int kNumIters, const int kIterSize) { - const double kPrecision = 1e-2; - const double kMinPrecision = 1e-7; - // Solve without accumulation and save parameters. - this->RunLeastSquaresSolver(kLearningRate, kWeightDecay, kMomentum, - kNumIters); - // Save parameters for comparison. - Net& net = *this->solver_->net(); - const vector > >& param_blobs = - net.layer_by_name("innerprod")->blobs(); - vector > > noaccum_params(param_blobs.size()); - for (int i = 0; i < param_blobs.size(); ++i) { - noaccum_params[i].reset(new Blob()); - noaccum_params[i]->CopyFrom(*param_blobs[i], false, true); - } - // Solve by equivalent accumulation of gradients over divided batches. - this->RunLeastSquaresSolver(kLearningRate, kWeightDecay, kMomentum, - kNumIters, kIterSize); - Net& net_accum = *this->solver_->net(); - const vector > >& accum_params = - net_accum.layer_by_name("innerprod")->blobs(); - // Compare accumulated parameters against no accumulation standard. - const int D = this->channels_ * this->height_ * this->width_; - for (int i = 0; i < D; ++i) { - const Dtype expected_param = noaccum_params[0]->cpu_data()[i]; - const Dtype accum_param = accum_params[0]->cpu_data()[i]; - const Dtype error_margin = std::max(kMinPrecision, kPrecision * - std::min(fabs(expected_param), fabs(accum_param))); - EXPECT_NEAR(expected_param, accum_param, error_margin); - } - ASSERT_EQ(1, accum_params[1]->count()); - const Dtype expected_bias = noaccum_params[1]->cpu_data()[0]; - const Dtype accum_bias = accum_params[1]->cpu_data()[0]; - const Dtype error_margin = std::max(kMinPrecision, kPrecision * - std::min(fabs(expected_bias), fabs(accum_bias))); - EXPECT_NEAR(expected_bias, accum_bias, error_margin); - } - // Test that the correct update is computed for a regularized least squares // problem: // @@ -417,16 +372,6 @@ TYPED_TEST(SGDSolverTest, TestLeastSquaresUpdateWithEverything) { } } -TYPED_TEST(SGDSolverTest, TestLeastSquaresUpdateWithEverythingAccum) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kLearningRate = 0.01; - const Dtype kWeightDecay = 0.1; - const Dtype kMomentum = 0.9; - const int kNumIters = 4; - const int kIterSize = 2; - this->CheckAccumulation(kLearningRate, kWeightDecay, kMomentum, kNumIters, - kIterSize); -} template class AdaGradSolverTest : public GradientBasedSolverTest { @@ -471,16 +416,6 @@ TYPED_TEST(AdaGradSolverTest, TestAdaGradLeastSquaresUpdateWithEverything) { } } -TYPED_TEST(AdaGradSolverTest, TestLeastSquaresUpdateWithEverythingAccum) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kLearningRate = 0.01; - const Dtype kWeightDecay = 0.1; - const Dtype kMomentum = 0.0; - const int kNumIters = 4; - const int kIterSize = 2; - this->CheckAccumulation(kLearningRate, kWeightDecay, kMomentum, kNumIters, - kIterSize); -} template class NesterovSolverTest : public GradientBasedSolverTest { @@ -547,15 +482,4 @@ TYPED_TEST(NesterovSolverTest, TestNesterovLeastSquaresUpdateWithEverything) { } } -TYPED_TEST(NesterovSolverTest, TestLeastSquaresUpdateWithEverythingAccum) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kLearningRate = 0.01; - const Dtype kWeightDecay = 0.1; - const Dtype kMomentum = 0.9; - const int kNumIters = 4; - const int kIterSize = 2; - this->CheckAccumulation(kLearningRate, kWeightDecay, kMomentum, kNumIters, - kIterSize); -} - } // namespace caffe diff --git a/src/caffe/test/test_im2col_kernel.cu b/src/caffe/test/test_im2col_kernel.cu index 0017ac23e69..ee684c00255 100644 --- a/src/caffe/test/test_im2col_kernel.cu +++ b/src/caffe/test/test_im2col_kernel.cu @@ -25,7 +25,7 @@ __global__ void im2col_gpu_kernel(const int n, const Dtype* data_im, extern cudaDeviceProp CAFFE_TEST_CUDA_PROP; template -class Im2colKernelTest : public GPUDeviceTest { +class Im2colKernelTest : public ::testing::Test { protected: Im2colKernelTest() // big so launches > 1024 threads @@ -68,6 +68,8 @@ class Im2colKernelTest : public GPUDeviceTest { TYPED_TEST_CASE(Im2colKernelTest, TestDtypes); TYPED_TEST(Im2colKernelTest, TestGPU) { + Caffe::set_mode(Caffe::GPU); + // Reshape the blobs to correct size for im2col output this->blob_top_->Reshape(this->blob_bottom_->num(), this->channels_ * this->kernel_size_ * this->kernel_size_, diff --git a/src/caffe/test/test_math_functions.cpp b/src/caffe/test/test_math_functions.cpp index a095b544e17..667f744bdd7 100644 --- a/src/caffe/test/test_math_functions.cpp +++ b/src/caffe/test/test_math_functions.cpp @@ -15,10 +15,8 @@ namespace caffe { -template -class MathFunctionsTest : public MultiDeviceTest { - typedef typename TypeParam::Dtype Dtype; - +template +class MathFunctionsTest : public ::testing::Test { protected: MathFunctionsTest() : blob_bottom_(new Blob()), @@ -66,19 +64,14 @@ class MathFunctionsTest : public MultiDeviceTest { Blob* const blob_top_; }; -template -class CPUMathFunctionsTest - : public MathFunctionsTest > { -}; - -TYPED_TEST_CASE(CPUMathFunctionsTest, TestDtypes); +TYPED_TEST_CASE(MathFunctionsTest, TestDtypes); -TYPED_TEST(CPUMathFunctionsTest, TestNothing) { +TYPED_TEST(MathFunctionsTest, TestNothing) { // The first test case of a test suite takes the longest time // due to the set up overhead. } -TYPED_TEST(CPUMathFunctionsTest, TestHammingDistance) { +TYPED_TEST(MathFunctionsTest, TestHammingDistanceCPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); const TypeParam* y = this->blob_top_->cpu_data(); @@ -86,7 +79,7 @@ TYPED_TEST(CPUMathFunctionsTest, TestHammingDistance) { caffe_cpu_hamming_distance(n, x, y)); } -TYPED_TEST(CPUMathFunctionsTest, TestAsum) { +TYPED_TEST(MathFunctionsTest, TestAsumCPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); TypeParam std_asum = 0; @@ -97,7 +90,7 @@ TYPED_TEST(CPUMathFunctionsTest, TestAsum) { EXPECT_LT((cpu_asum - std_asum) / std_asum, 1e-2); } -TYPED_TEST(CPUMathFunctionsTest, TestSign) { +TYPED_TEST(MathFunctionsTest, TestSignCPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); caffe_cpu_sign(n, x, this->blob_bottom_->mutable_cpu_diff()); @@ -107,7 +100,7 @@ TYPED_TEST(CPUMathFunctionsTest, TestSign) { } } -TYPED_TEST(CPUMathFunctionsTest, TestSgnbit) { +TYPED_TEST(MathFunctionsTest, TestSgnbitCPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); caffe_cpu_sgnbit(n, x, this->blob_bottom_->mutable_cpu_diff()); @@ -117,7 +110,7 @@ TYPED_TEST(CPUMathFunctionsTest, TestSgnbit) { } } -TYPED_TEST(CPUMathFunctionsTest, TestFabs) { +TYPED_TEST(MathFunctionsTest, TestFabsCPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); caffe_abs(n, x, this->blob_bottom_->mutable_cpu_diff()); @@ -127,7 +120,7 @@ TYPED_TEST(CPUMathFunctionsTest, TestFabs) { } } -TYPED_TEST(CPUMathFunctionsTest, TestScale) { +TYPED_TEST(MathFunctionsTest, TestScaleCPU) { int n = this->blob_bottom_->count(); TypeParam alpha = this->blob_bottom_->cpu_diff()[caffe_rng_rand() % this->blob_bottom_->count()]; @@ -140,10 +133,11 @@ TYPED_TEST(CPUMathFunctionsTest, TestScale) { } } -TYPED_TEST(CPUMathFunctionsTest, TestCopy) { +TYPED_TEST(MathFunctionsTest, TestCopyCPU) { const int n = this->blob_bottom_->count(); const TypeParam* bottom_data = this->blob_bottom_->cpu_data(); TypeParam* top_data = this->blob_top_->mutable_cpu_data(); + Caffe::set_mode(Caffe::CPU); caffe_copy(n, bottom_data, top_data); for (int i = 0; i < n; ++i) { EXPECT_EQ(bottom_data[i], top_data[i]); @@ -152,14 +146,8 @@ TYPED_TEST(CPUMathFunctionsTest, TestCopy) { #ifndef CPU_ONLY -template -class GPUMathFunctionsTest : public MathFunctionsTest > { -}; - -TYPED_TEST_CASE(GPUMathFunctionsTest, TestDtypes); - // TODO: Fix caffe_gpu_hamming_distance and re-enable this test. -TYPED_TEST(GPUMathFunctionsTest, DISABLED_TestHammingDistance) { +TYPED_TEST(MathFunctionsTest, DISABLED_TestHammingDistanceGPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); const TypeParam* y = this->blob_top_->cpu_data(); @@ -170,7 +158,7 @@ TYPED_TEST(GPUMathFunctionsTest, DISABLED_TestHammingDistance) { EXPECT_EQ(reference_distance, computed_distance); } -TYPED_TEST(GPUMathFunctionsTest, TestAsum) { +TYPED_TEST(MathFunctionsTest, TestAsumGPU) { int n = this->blob_bottom_->count(); const TypeParam* x = this->blob_bottom_->cpu_data(); TypeParam std_asum = 0; @@ -182,7 +170,7 @@ TYPED_TEST(GPUMathFunctionsTest, TestAsum) { EXPECT_LT((gpu_asum - std_asum) / std_asum, 1e-2); } -TYPED_TEST(GPUMathFunctionsTest, TestSign) { +TYPED_TEST(MathFunctionsTest, TestSignGPU) { int n = this->blob_bottom_->count(); caffe_gpu_sign(n, this->blob_bottom_->gpu_data(), this->blob_bottom_->mutable_gpu_diff()); @@ -193,7 +181,7 @@ TYPED_TEST(GPUMathFunctionsTest, TestSign) { } } -TYPED_TEST(GPUMathFunctionsTest, TestSgnbit) { +TYPED_TEST(MathFunctionsTest, TestSgnbitGPU) { int n = this->blob_bottom_->count(); caffe_gpu_sgnbit(n, this->blob_bottom_->gpu_data(), this->blob_bottom_->mutable_gpu_diff()); @@ -204,7 +192,7 @@ TYPED_TEST(GPUMathFunctionsTest, TestSgnbit) { } } -TYPED_TEST(GPUMathFunctionsTest, TestFabs) { +TYPED_TEST(MathFunctionsTest, TestFabsGPU) { int n = this->blob_bottom_->count(); caffe_gpu_abs(n, this->blob_bottom_->gpu_data(), this->blob_bottom_->mutable_gpu_diff()); @@ -215,7 +203,7 @@ TYPED_TEST(GPUMathFunctionsTest, TestFabs) { } } -TYPED_TEST(GPUMathFunctionsTest, TestScale) { +TYPED_TEST(MathFunctionsTest, TestScaleGPU) { int n = this->blob_bottom_->count(); TypeParam alpha = this->blob_bottom_->cpu_diff()[caffe_rng_rand() % this->blob_bottom_->count()]; @@ -228,10 +216,11 @@ TYPED_TEST(GPUMathFunctionsTest, TestScale) { } } -TYPED_TEST(GPUMathFunctionsTest, TestCopy) { +TYPED_TEST(MathFunctionsTest, TestCopyGPU) { const int n = this->blob_bottom_->count(); const TypeParam* bottom_data = this->blob_bottom_->gpu_data(); TypeParam* top_data = this->blob_top_->mutable_gpu_data(); + Caffe::set_mode(Caffe::GPU); caffe_copy(n, bottom_data, top_data); bottom_data = this->blob_bottom_->cpu_data(); top_data = this->blob_top_->mutable_cpu_data(); diff --git a/src/caffe/test/test_multinomial_logistic_loss_layer.cpp b/src/caffe/test/test_multinomial_logistic_loss_layer.cpp index b2db984feb1..9038017e3e2 100644 --- a/src/caffe/test/test_multinomial_logistic_loss_layer.cpp +++ b/src/caffe/test/test_multinomial_logistic_loss_layer.cpp @@ -16,7 +16,7 @@ namespace caffe { template -class MultinomialLogisticLossLayerTest : public CPUDeviceTest { +class MultinomialLogisticLossLayerTest : public ::testing::Test { protected: MultinomialLogisticLossLayerTest() : blob_bottom_data_(new Blob(10, 5, 1, 1)), @@ -51,6 +51,7 @@ TYPED_TEST_CASE(MultinomialLogisticLossLayerTest, TestDtypes); TYPED_TEST(MultinomialLogisticLossLayerTest, TestGradientCPU) { LayerParameter layer_param; + Caffe::set_mode(Caffe::CPU); MultinomialLogisticLossLayer layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); GradientChecker checker(1e-2, 2*1e-2, 1701, 0, 0.05); diff --git a/src/caffe/test/test_net.cpp b/src/caffe/test/test_net.cpp index 782a96bc9b6..08106e79274 100644 --- a/src/caffe/test/test_net.cpp +++ b/src/caffe/test/test_net.cpp @@ -613,103 +613,6 @@ class NetTest : public MultiDeviceTest { InitNetFromProtoString(proto); } - virtual void InitSkipPropNet(bool test_skip_true) { - string proto = - "name: 'SkipPropTestNetwork' " - "layer { " - " name: 'data' " - " type: 'DummyData' " - " dummy_data_param { " - " shape { " - " dim: 5 " - " dim: 2 " - " dim: 3 " - " dim: 4 " - " } " - " data_filler { " - " type: 'gaussian' " - " std: 0.01 " - " } " - " shape { " - " dim: 5 " - " } " - " data_filler { " - " type: 'constant' " - " value: 0 " - " } " - " } " - " top: 'data' " - " top: 'label' " - "} " - "layer { " - " name: 'silence' " - " bottom: 'label' " - " type: 'Silence' " - "} " - "layer { " - " name: 'innerproduct' " - " type: 'InnerProduct' " - " inner_product_param { " - " num_output: 1 " - " weight_filler { " - " type: 'gaussian' " - " std: 0.01 " - " } " - " bias_filler { " - " type: 'constant' " - " value: 0 " - " } " - " } " - " param { " - " lr_mult: 1 " - " decay_mult: 1 " - " } " - " param { " - " lr_mult: 2 " - " decay_mult: 0 " - " } " - " bottom: 'data' " - " top: 'innerproduct' " - "} " - "layer { " - " name: 'ip_fake_labels' " - " type: 'InnerProduct' " - " inner_product_param { " - " num_output: 1 " - " weight_filler { " - " type: 'gaussian' " - " std: 0.01 " - " } " - " bias_filler { " - " type: 'constant' " - " value: 0 " - " } " - " } " - " bottom: 'data' " - " top: 'fake_labels' " - "} " - "layer { " - " name: 'argmax' " - " bottom: 'fake_labels' " - " top: 'label_argmax' " - " type: 'ArgMax' " - "} " - "layer { " - " name: 'loss' " - " bottom: 'innerproduct' " - " bottom: 'label_argmax' "; - if (test_skip_true) - proto += " propagate_down: [true, false] "; - else - proto += " propagate_down: [true, true] "; - proto += - " top: 'cross_entropy_loss' " - " type: 'SigmoidCrossEntropyLoss' " - " loss_weight: 0.1 " - "} "; - InitNetFromProtoString(proto); - } - int seed_; shared_ptr > net_; }; @@ -2321,52 +2224,4 @@ TYPED_TEST(NetTest, TestReshape) { } } -TYPED_TEST(NetTest, TestSkipPropagateDown) { - // check bottom_need_backward if propagate_down is true - this->InitSkipPropNet(false); - vector vec_layer_need_backward = this->net_->layer_need_backward(); - for (int layer_id = 0; layer_id < this->net_->layers().size(); ++layer_id) { - string layer_name = this->net_->layer_names()[layer_id]; - if (layer_name == "loss") { - // access to bottom_need_backward coresponding to label's blob - bool need_back = this->net_->bottom_need_backward()[layer_id][1]; - // if propagate_down is true, the loss layer will try to - // backpropagate on labels - EXPECT_TRUE(need_back) << "bottom_need_backward should be True"; - } - // layer_need_backward should be True except for data and silence layers - if (layer_name.find("data") != std::string::npos || - layer_name == "silence") { - EXPECT_FALSE(vec_layer_need_backward[layer_id]) - << "layer_need_backward for " << layer_name << " should be False"; - } else { - EXPECT_TRUE(vec_layer_need_backward[layer_id]) - << "layer_need_backward for " << layer_name << " should be True"; - } - } - // check bottom_need_backward if propagat_down is false - this->InitSkipPropNet(true); - vec_layer_need_backward.clear(); - vec_layer_need_backward = this->net_->layer_need_backward(); - for (int layer_id = 0; layer_id < this->net_->layers().size(); ++layer_id) { - string layer_name = this->net_->layer_names()[layer_id]; - if (layer_name == "loss") { - // access to bottom_need_backward coresponding to label's blob - bool need_back = this->net_->bottom_need_backward()[layer_id][1]; - // if propagate_down is false, the loss layer will not try to - // backpropagate on labels - EXPECT_FALSE(need_back) << "bottom_need_backward should be False"; - } - // layer_need_backward should be False except for innerproduct and - // loss layers - if (layer_name == "innerproduct" || layer_name == "loss") { - EXPECT_TRUE(vec_layer_need_backward[layer_id]) - << "layer_need_backward for " << layer_name << " should be True"; - } else { - EXPECT_FALSE(vec_layer_need_backward[layer_id]) - << "layer_need_backward for " << layer_name << " should be False"; - } - } -} - } // namespace caffe diff --git a/src/caffe/test/test_neuron_layer.cpp b/src/caffe/test/test_neuron_layer.cpp index c6e4d27b903..c9d52f247a6 100644 --- a/src/caffe/test/test_neuron_layer.cpp +++ b/src/caffe/test/test_neuron_layer.cpp @@ -117,49 +117,6 @@ class NeuronLayerTest : public MultiDeviceTest { + slope_data[c] * std::min(bottom_data[i], (Dtype)(0))); } } - - void LogBottomInit() { - FillerParameter filler_param; - GaussianFiller filler(filler_param); - filler.Fill(this->blob_bottom_); - Dtype* bottom_data = this->blob_bottom_->mutable_cpu_data(); - caffe_exp(this->blob_bottom_->count(), bottom_data, bottom_data); - } - - void TestLogForward(const float base, const float scale, const float shift) { - LogBottomInit(); - LayerParameter layer_param; - layer_param.mutable_log_param()->set_base(base); - layer_param.mutable_log_param()->set_scale(scale); - layer_param.mutable_log_param()->set_shift(shift); - LogLayer layer(layer_param); - layer.SetUp(blob_bottom_vec_, blob_top_vec_); - layer.Forward(blob_bottom_vec_, blob_top_vec_); - const Dtype kDelta = 2e-4; - const Dtype* bottom_data = blob_bottom_->cpu_data(); - const Dtype* top_data = blob_top_->cpu_data(); - for (int i = 0; i < blob_bottom_->count(); ++i) { - const Dtype bottom_val = bottom_data[i]; - const Dtype top_val = top_data[i]; - if (base == -1) { - EXPECT_NEAR(top_val, log(shift + scale * bottom_val), kDelta); - } else { - EXPECT_NEAR(top_val, log(shift + scale * bottom_val) / log(base), - kDelta); - } - } - } - - void TestLogGradient(const float base, const float scale, const float shift) { - LogBottomInit(); - LayerParameter layer_param; - layer_param.mutable_log_param()->set_base(base); - layer_param.mutable_log_param()->set_scale(scale); - layer_param.mutable_log_param()->set_shift(shift); - LogLayer layer(layer_param); - GradientChecker checker(1e-2, 1e-2); - checker.CheckGradientEltwise(&layer, blob_bottom_vec_, blob_top_vec_); - } }; TYPED_TEST_CASE(NeuronLayerTest, TestDtypesAndDevices); @@ -382,88 +339,6 @@ TYPED_TEST(NeuronLayerTest, TestExpGradientBase2Shift1Scale3) { this->TestExpGradient(kBase, kScale, kShift); } -TYPED_TEST(NeuronLayerTest, TestLogLayer) { - typedef typename TypeParam::Dtype Dtype; - // Test default base of "-1" -- should actually set base := e. - const Dtype kBase = -1; - const Dtype kScale = 1; - const Dtype kShift = 0; - this->TestLogForward(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogGradient) { - typedef typename TypeParam::Dtype Dtype; - // Test default base of "-1" -- should actually set base := e. - const Dtype kBase = -1; - const Dtype kScale = 1; - const Dtype kShift = 0; - this->TestLogGradient(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogLayerBase2) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kBase = 2; - const Dtype kScale = 1; - const Dtype kShift = 0; - this->TestLogForward(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogGradientBase2) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kBase = 2; - const Dtype kScale = 1; - const Dtype kShift = 0; - this->TestLogGradient(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogLayerBase2Shift1) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kBase = 2; - const Dtype kScale = 1; - const Dtype kShift = 1; - this->TestLogForward(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogGradientBase2Shift1) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kBase = 2; - const Dtype kScale = 1; - const Dtype kShift = 1; - this->TestLogGradient(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogLayerBase2Scale3) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kBase = 2; - const Dtype kScale = 3; - const Dtype kShift = 0; - this->TestLogForward(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogGradientBase2Scale3) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kBase = 2; - const Dtype kScale = 3; - const Dtype kShift = 0; - this->TestLogGradient(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogLayerBase2Shift1Scale3) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kBase = 2; - const Dtype kScale = 3; - const Dtype kShift = 1; - this->TestLogForward(kBase, kScale, kShift); -} - -TYPED_TEST(NeuronLayerTest, TestLogGradientBase2Shift1Scale3) { - typedef typename TypeParam::Dtype Dtype; - const Dtype kBase = 2; - const Dtype kScale = 3; - const Dtype kShift = 1; - this->TestLogGradient(kBase, kScale, kShift); -} - TYPED_TEST(NeuronLayerTest, TestDropoutHalf) { const float kDropoutRatio = 0.5; this->TestDropoutForward(kDropoutRatio); @@ -666,10 +541,14 @@ TYPED_TEST(NeuronLayerTest, TestPReLUInPlace) { caffe_copy(ip2.blobs()[0]->count(), ip.blobs()[0]->cpu_data(), ip2.blobs()[0]->mutable_cpu_data()); // Forward in-place + ip.Reshape(this->blob_bottom_vec_, this->blob_top_vec_); ip.Forward(this->blob_bottom_vec_, this->blob_top_vec_); + prelu.Reshape(this->blob_top_vec_, this->blob_top_vec_); prelu.Forward(this->blob_top_vec_, this->blob_top_vec_); // Forward non-in-place + ip2.Reshape(blob_bottom_vec_2, blob_middle_vec_2); ip2.Forward(blob_bottom_vec_2, blob_middle_vec_2); + prelu2.Reshape(blob_middle_vec_2, blob_top_vec_2); prelu2.Forward(blob_middle_vec_2, blob_top_vec_2); // Check numbers for (int s = 0; s < blob_top_2->count(); ++s) { @@ -711,7 +590,7 @@ TYPED_TEST(NeuronLayerTest, TestPReLUInPlace) { #ifdef USE_CUDNN template -class CuDNNNeuronLayerTest : public GPUDeviceTest { +class CuDNNNeuronLayerTest : public ::testing::Test { protected: CuDNNNeuronLayerTest() : blob_bottom_(new Blob(2, 3, 4, 5)), @@ -734,6 +613,7 @@ class CuDNNNeuronLayerTest : public GPUDeviceTest { TYPED_TEST_CASE(CuDNNNeuronLayerTest, TestDtypes); TYPED_TEST(CuDNNNeuronLayerTest, TestReLUCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CuDNNReLULayer layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); @@ -748,6 +628,7 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestReLUCuDNN) { } TYPED_TEST(CuDNNNeuronLayerTest, TestReLUGradientCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CuDNNReLULayer layer(layer_param); GradientChecker checker(1e-2, 1e-3, 1701, 0., 0.01); @@ -756,6 +637,7 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestReLUGradientCuDNN) { } TYPED_TEST(CuDNNNeuronLayerTest, TestReLUWithNegativeSlopeCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CHECK(google::protobuf::TextFormat::ParseFromString( "relu_param { negative_slope: 0.01 }", &layer_param)); @@ -775,6 +657,7 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestReLUWithNegativeSlopeCuDNN) { } TYPED_TEST(CuDNNNeuronLayerTest, TestReLUGradientWithNegativeSlopeCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CHECK(google::protobuf::TextFormat::ParseFromString( "relu_param { negative_slope: 0.01 }", &layer_param)); @@ -785,6 +668,7 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestReLUGradientWithNegativeSlopeCuDNN) { } TYPED_TEST(CuDNNNeuronLayerTest, TestSigmoidCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CuDNNSigmoidLayer layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); @@ -801,6 +685,7 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestSigmoidCuDNN) { } TYPED_TEST(CuDNNNeuronLayerTest, TestSigmoidGradientCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CuDNNSigmoidLayer layer(layer_param); GradientChecker checker(1e-2, 1e-3, 1701, 0., 0.01); @@ -809,6 +694,7 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestSigmoidGradientCuDNN) { } TYPED_TEST(CuDNNNeuronLayerTest, TestTanHCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CuDNNTanHLayer layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); @@ -831,6 +717,7 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestTanHCuDNN) { } TYPED_TEST(CuDNNNeuronLayerTest, TestTanHGradientCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CuDNNTanHLayer layer(layer_param); GradientChecker checker(1e-2, 1e-3); diff --git a/src/caffe/test/test_pooling_layer.cpp b/src/caffe/test/test_pooling_layer.cpp index 69f2d5c1135..e9964e7f0b7 100644 --- a/src/caffe/test/test_pooling_layer.cpp +++ b/src/caffe/test/test_pooling_layer.cpp @@ -608,7 +608,7 @@ TYPED_TEST(PoolingLayerTest, TestGradientAvePadded) { #ifdef USE_CUDNN template -class CuDNNPoolingLayerTest : public GPUDeviceTest { +class CuDNNPoolingLayerTest : public ::testing::Test { protected: CuDNNPoolingLayerTest() : blob_bottom_(new Blob()), @@ -963,6 +963,7 @@ class CuDNNPoolingLayerTest : public GPUDeviceTest { TYPED_TEST_CASE(CuDNNPoolingLayerTest, TestDtypes); TYPED_TEST(CuDNNPoolingLayerTest, TestSetupCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); pooling_param->set_kernel_size(3); @@ -976,6 +977,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestSetupCuDNN) { } TYPED_TEST(CuDNNPoolingLayerTest, TestSetupPaddedCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); pooling_param->set_kernel_size(3); @@ -992,6 +994,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestSetupPaddedCuDNN) { /* TYPED_TEST(CuDNNPoolingLayerTest, PrintBackwardCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; layer_param.set_kernelsize(3); layer_param.set_stride(2); @@ -1017,6 +1020,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, PrintBackwardCuDNN) { */ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxCuDNN) { + Caffe::set_mode(Caffe::GPU); this->TestForwardSquare(); this->TestForwardRectHigh(); this->TestForwardRectWide(); @@ -1026,6 +1030,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxCuDNN) { // the corresponding backward test. /* TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxTopMaskCuDNN) { + Caffe::set_mode(Caffe::GPU); this->blob_top_vec_.push_back(this->blob_top_mask_); this->TestForwardSquare(); this->TestForwardRectHigh(); @@ -1034,6 +1039,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxTopMaskCuDNN) { */ TYPED_TEST(CuDNNPoolingLayerTest, TestGradientMaxCuDNN) { + Caffe::set_mode(Caffe::GPU); for (int kernel_h = 3; kernel_h <= 4; kernel_h++) { for (int kernel_w = 3; kernel_w <= 4; kernel_w++) { LayerParameter layer_param; @@ -1053,6 +1059,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestGradientMaxCuDNN) { } TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxPaddedCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); pooling_param->set_kernel_size(3); @@ -1098,6 +1105,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxPaddedCuDNN) { /* TYPED_TEST(CuDNNPoolingLayerTest, TestGradientMaxTopMaskCuDNN) { + Caffe::set_mode(Caffe::GPU); for (int kernel_h = 3; kernel_h <= 4; kernel_h++) { for (int kernel_w = 3; kernel_w <= 4; kernel_w++) { LayerParameter layer_param; @@ -1118,6 +1126,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestGradientMaxTopMaskCuDNN) { */ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardAveCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); pooling_param->set_kernel_size(3); @@ -1143,6 +1152,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardAveCuDNN) { } TYPED_TEST(CuDNNPoolingLayerTest, TestGradientAveCuDNN) { + Caffe::set_mode(Caffe::GPU); for (int kernel_h = 3; kernel_h <= 4; kernel_h++) { for (int kernel_w = 3; kernel_w <= 4; kernel_w++) { LayerParameter layer_param; @@ -1160,6 +1170,7 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestGradientAveCuDNN) { } TYPED_TEST(CuDNNPoolingLayerTest, TestGradientAvePaddedCuDNN) { + Caffe::set_mode(Caffe::GPU); for (int kernel_h = 3; kernel_h <= 4; kernel_h++) { for (int kernel_w = 3; kernel_w <= 4; kernel_w++) { LayerParameter layer_param; diff --git a/src/caffe/test/test_softmax_layer.cpp b/src/caffe/test/test_softmax_layer.cpp index 996da4b8f7c..f6674422e56 100644 --- a/src/caffe/test/test_softmax_layer.cpp +++ b/src/caffe/test/test_softmax_layer.cpp @@ -82,7 +82,7 @@ TYPED_TEST(SoftmaxLayerTest, TestGradient) { #ifdef USE_CUDNN template -class CuDNNSoftmaxLayerTest : public GPUDeviceTest { +class CuDNNSoftmaxLayerTest : public ::testing::Test { protected: CuDNNSoftmaxLayerTest() : blob_bottom_(new Blob(2, 10, 2, 3)), @@ -104,6 +104,7 @@ class CuDNNSoftmaxLayerTest : public GPUDeviceTest { TYPED_TEST_CASE(CuDNNSoftmaxLayerTest, TestDtypes); TYPED_TEST(CuDNNSoftmaxLayerTest, TestForwardCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CuDNNSoftmaxLayer layer(layer_param); layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); @@ -137,6 +138,7 @@ TYPED_TEST(CuDNNSoftmaxLayerTest, TestForwardCuDNN) { } TYPED_TEST(CuDNNSoftmaxLayerTest, TestGradientCuDNN) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; CuDNNSoftmaxLayer layer(layer_param); GradientChecker checker(1e-2, 1e-3); diff --git a/src/caffe/test/test_stochastic_pooling.cpp b/src/caffe/test/test_stochastic_pooling.cpp index f84464c322c..12962c65d85 100644 --- a/src/caffe/test/test_stochastic_pooling.cpp +++ b/src/caffe/test/test_stochastic_pooling.cpp @@ -16,10 +16,8 @@ using std::min; namespace caffe { -template -class StochasticPoolingLayerTest : public MultiDeviceTest { - typedef typename TypeParam::Dtype Dtype; - +template +class StochasticPoolingLayerTest : public ::testing::Test { protected: StochasticPoolingLayerTest() : blob_bottom_(new Blob()), @@ -47,14 +45,9 @@ class StochasticPoolingLayerTest : public MultiDeviceTest { vector*> blob_top_vec_; }; -template -class CPUStochasticPoolingLayerTest - : public StochasticPoolingLayerTest > { -}; - -TYPED_TEST_CASE(CPUStochasticPoolingLayerTest, TestDtypes); +TYPED_TEST_CASE(StochasticPoolingLayerTest, TestDtypes); -TYPED_TEST(CPUStochasticPoolingLayerTest, TestSetup) { +TYPED_TEST(StochasticPoolingLayerTest, TestSetup) { LayerParameter layer_param; PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); pooling_param->set_kernel_size(3); @@ -67,16 +60,8 @@ TYPED_TEST(CPUStochasticPoolingLayerTest, TestSetup) { EXPECT_EQ(this->blob_top_->width(), 2); } -#ifndef CPU_ONLY - -template -class GPUStochasticPoolingLayerTest - : public StochasticPoolingLayerTest > { -}; - -TYPED_TEST_CASE(GPUStochasticPoolingLayerTest, TestDtypes); - -TYPED_TEST(GPUStochasticPoolingLayerTest, TestStochastic) { +TYPED_TEST(StochasticPoolingLayerTest, TestStochasticGPU) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; layer_param.set_phase(TRAIN); PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); @@ -119,7 +104,8 @@ TYPED_TEST(GPUStochasticPoolingLayerTest, TestStochastic) { EXPECT_GE(total / this->blob_top_->count(), 0.55); } -TYPED_TEST(GPUStochasticPoolingLayerTest, TestStochasticTestPhase) { +TYPED_TEST(StochasticPoolingLayerTest, TestStochasticGPUTestPhase) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; layer_param.set_phase(TEST); PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); @@ -156,7 +142,8 @@ TYPED_TEST(GPUStochasticPoolingLayerTest, TestStochasticTestPhase) { } } -TYPED_TEST(GPUStochasticPoolingLayerTest, TestGradient) { +TYPED_TEST(StochasticPoolingLayerTest, TestGradientGPU) { + Caffe::set_mode(Caffe::GPU); LayerParameter layer_param; layer_param.set_phase(TRAIN); PoolingParameter* pooling_param = layer_param.mutable_pooling_param(); @@ -171,6 +158,6 @@ TYPED_TEST(GPUStochasticPoolingLayerTest, TestGradient) { this->blob_top_vec_); } -#endif + } // namespace caffe diff --git a/src/caffe/test/test_triplet_loss_layer.cpp b/src/caffe/test/test_triplet_loss_layer.cpp new file mode 100644 index 00000000000..c8d9377fa23 --- /dev/null +++ b/src/caffe/test/test_triplet_loss_layer.cpp @@ -0,0 +1,107 @@ +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/vision_layers.hpp" + +#include "caffe/test/test_caffe_main.hpp" +#include "caffe/test/test_gradient_check_util.hpp" + +namespace caffe { + +template +class TripletLossLayerTest : public MultiDeviceTest { + typedef typename TypeParam::Dtype Dtype; + + protected: + TripletLossLayerTest() + : blob_bottom_data_i_(new Blob(128, 10, 1, 1)), + blob_bottom_data_j_(new Blob(128, 10, 1, 1)), + blob_bottom_data_k_(new Blob(128, 10, 1, 1)), + blob_bottom_y_(new Blob(128, 1, 1, 1)), + blob_top_loss_(new Blob()) { + // fill the values + FillerParameter filler_param; + filler_param.set_mean(0.0); + filler_param.set_std(0.3); // distances~=1.0 to test both sides of margin + GaussianFiller filler(filler_param); + filler.Fill(this->blob_bottom_data_i_); + blob_bottom_vec_.push_back(blob_bottom_data_i_); + filler.Fill(this->blob_bottom_data_j_); + blob_bottom_vec_.push_back(blob_bottom_data_j_); + filler.Fill(this->blob_bottom_data_k_); + blob_bottom_vec_.push_back(blob_bottom_data_k_); + for (int i = 0; i < blob_bottom_y_->count(); ++i) { + blob_bottom_y_->mutable_cpu_data()[i] = caffe_rng_rand() % 2; // 0 or 1 + } + blob_bottom_vec_.push_back(blob_bottom_y_); + blob_top_vec_.push_back(blob_top_loss_); + } + virtual ~TripletLossLayerTest() { + delete blob_bottom_data_i_; + delete blob_bottom_data_j_; + delete blob_bottom_data_k_; + delete blob_bottom_y_; + delete blob_top_loss_; + } + + Blob* const blob_bottom_data_i_; + Blob* const blob_bottom_data_j_; + Blob* const blob_bottom_data_k_; + Blob* const blob_bottom_y_; + Blob* const blob_top_loss_; + vector*> blob_bottom_vec_; + vector*> blob_top_vec_; +}; + +TYPED_TEST_CASE(TripletLossLayerTest, TestDtypesAndDevices); + +TYPED_TEST(TripletLossLayerTest, TestForward) { + typedef typename TypeParam::Dtype Dtype; + LayerParameter layer_param; + TripletLossLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_); + // manually compute to compare + const Dtype margin = layer_param.contrastive_loss_param().margin(); + const int num = this->blob_bottom_data_i_->num(); + const int channels = this->blob_bottom_data_i_->channels(); + Dtype loss(0); + for (int i = 0; i < num; ++i) { + Dtype dist_sq(0); + for (int j = 0; j < channels; ++j) { + Dtype diff = this->blob_bottom_data_i_->cpu_data()[i*channels+j] - + this->blob_bottom_data_j_->cpu_data()[i*channels+j]; + dist_sq += diff*diff; + } + if (this->blob_bottom_y_->cpu_data()[i]) { // similar pairs + loss += dist_sq; + } else { + loss += std::max(margin-dist_sq, Dtype(0)); + } + } + loss /= static_cast(num) * Dtype(2); + EXPECT_NEAR(this->blob_top_loss_->cpu_data()[0], loss, 1e-6); +} + +TYPED_TEST(TripletLossLayerTest, TestGradient) { + typedef typename TypeParam::Dtype Dtype; + LayerParameter layer_param; + TripletLossLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + GradientChecker checker(1e-2, 1e-2, 1701); + // check the gradient for the first two bottom layers + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 1); +} + +} // namespace caffe diff --git a/src/caffe/util/math_functions.cpp b/src/caffe/util/math_functions.cpp index 0aab6b17b85..13e17be582b 100644 --- a/src/caffe/util/math_functions.cpp +++ b/src/caffe/util/math_functions.cpp @@ -206,16 +206,6 @@ void caffe_exp(const int n, const double* a, double* y) { vdExp(n, a, y); } -template <> -void caffe_log(const int n, const float* a, float* y) { - vsLn(n, a, y); -} - -template <> -void caffe_log(const int n, const double* a, double* y) { - vdLn(n, a, y); -} - template <> void caffe_abs(const int n, const float* a, float* y) { vsAbs(n, a, y); diff --git a/src/caffe/util/math_functions.cu b/src/caffe/util/math_functions.cu index 2631a0740d6..43e65eb9a69 100644 --- a/src/caffe/util/math_functions.cu +++ b/src/caffe/util/math_functions.cu @@ -324,27 +324,6 @@ void caffe_gpu_exp(const int N, const double* a, double* y) { N, a, y); } -template -__global__ void log_kernel(const int n, const Dtype* a, Dtype* y) { - CUDA_KERNEL_LOOP(index, n) { - y[index] = log(a[index]); - } -} - -template <> -void caffe_gpu_log(const int N, const float* a, float* y) { - // NOLINT_NEXT_LINE(whitespace/operators) - log_kernel<<>>( - N, a, y); -} - -template <> -void caffe_gpu_log(const int N, const double* a, double* y) { - // NOLINT_NEXT_LINE(whitespace/operators) - log_kernel<<>>( - N, a, y); -} - template __global__ void powx_kernel(const int n, const Dtype* a, const Dtype alpha, Dtype* y) {