totally working training loop with nothing wrong

Author: kemsig

src/layers/DenseLayer.cpp

@@ -6,7 +6,7 @@
 namespace TensorSANN{
     DenseLayer::DenseLayer(size_t input_size, size_t output_size){
         weights_ = Tensor({input_size, output_size});
-        biases_ = Tensor({output_size});
+        biases_ = Tensor({1, output_size});     // YEAH THIS MIGHT BE A PROBLEM LATER
 
         std::random_device rd;
         std::mt19937 gen(rd());
@@ -55,10 +55,10 @@ namespace TensorSANN{
     }
 
     void DenseLayer::update_weights_biases(float learning_rate){
-        // std::cout << weights_.to_string() << std:: endl;
-        // std::cout << (*(weights_.grad())).to_string() << std:: endl;
-        // std::cout << biases_.to_string() << std:: endl;
-        // std::cout << (*(biases_.grad())).to_string() << std:: endl;
+        std::cout << weights_.to_string() << std:: endl;
+        std::cout << (*(weights_.grad())).to_string() << std:: endl;
+        std::cout << biases_.to_string() << std:: endl;
+        std::cout << (*(biases_.grad())).to_string() << std:: endl;
         weights_ = weights_ - ((*(weights_.grad())) * learning_rate);
         biases_ = biases_ - ((*(biases_.grad())) * learning_rate);
 

src/model_main.cpp

@@ -36,75 +36,51 @@ int main() {
     
     */
 
-    TensorSANN::DenseLayer dense1(16, 2);
-    TensorSANN::ReLU relu;
-    TensorSANN::DenseLayer dense2(2, 1);
-    TensorSANN::Softmax smax;
-
     std::vector<std::shared_ptr<TensorSANN::Layer>> layers;
     layers.push_back(std::make_shared<TensorSANN::DenseLayer>(16, 4));
     layers.push_back(std::make_shared<TensorSANN::ReLU>());
     layers.push_back(std::make_shared<TensorSANN::DenseLayer>(4, 2));
     layers.push_back(std::make_shared<TensorSANN::Softmax>());
 
-    TensorSANN::Tensor fwd_op = input_tensor.transpose();
-
-    // fwd
-    for (auto &layer : layers){
-        fwd_op = layer->forward(fwd_op);
-        std::cout << "LOOP FWD==" << fwd_op.to_string() << std::endl;
-    }
 
-    TensorSANN::SGD optimizer = TensorSANN::SGD(0.01f);
+    int epoch = 2000;
+    for (int i = 1; i <= epoch; ++i){
+        TensorSANN::Tensor fwd_op = input_tensor.transpose();
+        // fwd
+        for (auto &layer : layers){
+            fwd_op = layer->forward(fwd_op);
+            std::cout << "\nLOOP FWD==" << fwd_op.to_string() << std::endl;
+        }
 
-    // calculate loss
-    TensorSANN::Tensor loss_grad = fwd_op - 1;
-    std::cout << "LOSS GRAD==" << loss_grad.to_string() << std::endl;
-    // bkwd
-    for (int i = layers.size()-2; i >= 0; --i){
-        loss_grad = layers[i]->backward(loss_grad);
-        std::cout << "LOOP BKWD==" << loss_grad.shape()[0] << loss_grad.shape()[1] << std::endl;
-        if ((*layers[i]).isTrainable()){
-            auto yup = std::dynamic_pointer_cast<TensorSANN::DenseLayer>(layers[i]);
-            std::string weight_str = yup->weights().to_string();
-            std::cout << "L---> weight_g" << weight_str.substr(0, 50) << "..." << std::endl;
-            std::string bias_str = yup->biases().to_string();
-            std::cout << "L---> bias_g" << bias_str.substr(0, 50) << "..." << std::endl;
-            // std::cout << "L---> bias-g" << yup.biases().to_string() << std::endl;
+        TensorSANN::SGD optimizer = TensorSANN::SGD(0.01f);
+
+        // calculate loss
+        TensorSANN::Tensor loss_grad = fwd_op - 1;
+        std::cout << "\nLOSS GRAD==" << loss_grad.to_string() << std::endl;
+        // bkwd
+        for (int i = layers.size()-2; i >= 0; --i){
+            loss_grad = layers[i]->backward(loss_grad);
+            std::cout << "\nLOOP BKWD==" << loss_grad.shape()[0] << loss_grad.shape()[1] << std::endl;
+            if ((*layers[i]).isTrainable()){
+                auto yup = std::dynamic_pointer_cast<TensorSANN::DenseLayer>(layers[i]);
+                std::string weight_str = yup->weights().to_string();
+                std::cout << "L---> weight_g" << weight_str.substr(0, 50) << "..." << std::endl;
+                std::string bias_str = yup->biases().to_string();
+                std::cout << "L---> bias_g" << bias_str.substr(0, 50) << "..." << std::endl;
+                // std::cout << "L---> bias-g" << yup.biases().to_string() << std::endl;
+            }
+            
         }
 
-    }
-    
-    // update weights
-    // int i =0;
-    // for (auto &layer : layers){
-    //     if (i == 0) continue;
-    //     // std::cout << ++i << std::endl;
-    //     optimizer.update(*layer);
-    // }
-
-    
-    const TensorSANN::Tensor d1w = dense1.weights();
-    const TensorSANN::Tensor d1b = dense1.biases();
-
-    std::cout << input_tensor.to_string() << std::endl;
-    std::cout << d1w.to_string() << std::endl;
-    std::cout << d1b.to_string() << std::endl;
-
-    TensorSANN::Tensor f1 = dense1.forward(input_tensor.transpose());
-    std::cout << f1.to_string() << std::endl;
-
-    
-    TensorSANN::Tensor r1 = relu.forward(f1);
-    std::cout << r1.to_string() << std::endl;
-
-    
+        // update weights
+        // int i =0;
+        for (auto &layer : layers){
+            // if (i == 0) continue;
+            // std::cout << ++i << std::endl;
+            optimizer.update(*layer);
+        }
 
-    TensorSANN::Tensor f2 = dense2.forward(r1);
-    std::cout << f2.to_string() << std::endl;
-    
-    TensorSANN::Tensor s1 = smax.forward(f2);
-    std::cout << (s1 - 1).to_string() << std::endl;
+    }
 
     return 0;