hot fix

Author: amsukdu

classes/neural_net.py

@@ -54,7 +54,10 @@ def predict(self, batch, label):
                 next_input *= self.dropout_p
 
         result = np.array(next_input)
-        loss, delta = utils.softmax_loss(result, label)
+        if self.loss == 'softmax':
+            loss, delta = utils.softmax_loss(result, label)
+        elif self.loss == 'logistic':
+            loss, delta = utils.logistic_loss(result, label)
 
         max_result = np.argmax(result, axis=0)
         correct_count = np.sum(max_result == label)

classes/neuron.py

@@ -4,14 +4,14 @@
 class Neuron(object):
     def __init__(self, input_size, bias=0.0):
         self.weights = (np.random.randn(input_size) * np.sqrt(2.0/input_size)).astype(np.float32)
-        self.bias = np.float32(bias)
+        self.b = np.float32(bias)
         self.last_input = None
         self.delta = 0
         self.m = 0
         self.v = 0
 
     def strength(self, values):
-        return np.dot(self.weights, values) + self.bias
+        return np.dot(self.weights, values) + self.b
 
     def regularization(self):
         return np.sum(np.square(self.weights))

classes/utils.py

@@ -65,6 +65,7 @@ def adam_update(neurons, lr, t, l2_reg=0, beta1=np.float32(0.9), beta2=np.float3
     for n in neurons:
         l2 = l2_reg * n.weights
         dx = (n.last_input.dot(n.delta)).T
+        dBias = np.average(n.delta)
 
         n.m = beta1 * n.m + (1 - beta1) * dx
         n.v = beta2 * n.v + (1 - beta2)*(dx**2)
@@ -74,33 +75,40 @@ def adam_update(neurons, lr, t, l2_reg=0, beta1=np.float32(0.9), beta2=np.float3
             n.v /= np.float32(1-beta2**t)
 
         n.weights -= lr * n.m / (np.sqrt(n.v) + 1e-8) + l2
+        n.b -= lr * dBias
 
 def nag_update(neurons, lr, l2_reg=0, mu=np.float32(0.9)):
     for n in neurons:
         l2 = l2_reg * n.weights
         dx = (n.last_input.dot(n.delta)).T
+        dBias = np.average(n.delta)
 
         n.v_prev = n.v
         n.v = mu * n.v - lr * dx
 
         n.weights += -mu * n.v_prev + (1 + mu) * n.v - l2
+        n.b -= lr * dBias
 
 def momentum_update(neurons, lr, l2_reg=0, mu=np.float32(0.9)):
     for n in neurons:
         l2 = l2_reg * n.weights
         dx = (n.last_input.dot(n.delta)).T
+        dBias = np.average(n.delta)
 
         n.v = mu * n.v - lr * dx
 
         n.weights += n.v - l2
+        n.b -= lr * dBias
 
 
 def vanila_update(neurons, lr, l2_reg=0):
     for n in neurons:
         l2 = l2_reg * n.weights
-        d = (n.last_input.dot(n.delta)).T
+        dx = (n.last_input.dot(n.delta)).T
+        dBias = np.average(n.delta)
 
-        n.weights -= lr * d + l2
+        n.weights -= lr * dx + l2
+        n.b -= lr * dBias
 
 def sigmoid(input):
     return 1/(1+np.exp(-input))

example/cifar-10-batches-py/main.py

@@ -1,3 +1,5 @@
+import sys, os
+sys.path.insert(1, os.path.split(os.path.split(sys.path[0])[0])[0])
 import cPickle as pkl
 import numpy as np
 from sklearn import preprocessing
@@ -11,15 +13,15 @@ def unpickle(file):
     return dict
 
 le = preprocessing.LabelEncoder()
-le.classes_ = unpickle('batches.meta')['label_names']
+le.classes_ = unpickle(sys.path[0] + '/batches.meta')['label_names']
 
 train_images = None
 train_labels = []
 
 test_images = None
 test_labels = []
 for i in xrange(1, 6):
-    data = unpickle('data_batch_'+str(i))
+    data = unpickle(sys.path[0] + '/data_batch_'+str(i))
     if train_images is None:
         train_images = data['data']
     else:
@@ -39,7 +41,7 @@ def unpickle(file):
 train_images[:, 1] -= g_mean
 train_images[:, 2] -= b_mean
 
-data = unpickle('test_batch')
+data = unpickle(sys.path[0] + '/test_batch')
 test_images = data['data'].reshape(-1, 3, 32, 32)
 test_images = test_images.astype(np.float32)
 test_images /= np.float32(255)
@@ -52,8 +54,8 @@ def unpickle(file):
 
 lr = 1e-4
 dropout_percent = 0.5
-l2_reg = 5e-6
-learning_rate_decay = np.float32(98e-2)
+l2_reg = 4e-6
+learning_rate_decay = np.float32(99e-2)
 batch_size = 1
 
 cnn = NeuralNetwork(train_images.shape[1:],