[test] add test cases that a specific layer is non-trainable

Add test cases that a specific layer is non-trainable.

- Test based on the pytoch model with two fc hidden layers.
- Add a test when the first hidden layer is set to non-trainable.
- Add a test when the second hidden layer is set to non-trainable.

**Self evaluation:**
1. Build test: [x]Passed []Failed []Skipped
2. Run test: [x]Passed []Failed []Skipped

Signed-off-by: Seungbaek Hong <sb92.hong@samsung.com>

diff --git a/packaging/unittest_models_v2.tar.gz b/packaging/unittest_models_v2.tar.gz
index cc1eb1a..f110be2 100644 (file)
Binary files a/packaging/unittest_models_v2.tar.gz and b/packaging/unittest_models_v2.tar.gz differ

diff --git a/test/input_gen/genModelTests_v2.py b/test/input_gen/genModelTests_v2.py
index a07b859..e50ba6a 100644 (file)
--- a/test/input_gen/genModelTests_v2.py
+++ b/test/input_gen/genModelTests_v2.py
@@ -287,6 +287,29 @@ class FCRelu(torch.nn.Module):
                 {'params': non_decay_params},
                 {'params': decay_params, 'weight_decay': 0.9}], lr=0.1)
 
+# class for test non-trainable fc layer
+class NonTrainableFC(torch.nn.Module):
+    def __init__(self, idx):
+        super().__init__()
+        self.fc1 = torch.nn.Linear(3, 10)
+        self.fc2 = torch.nn.Linear(10, 10)
+        self.fc3 = torch.nn.Linear(10, 2)
+        self.loss = torch.nn.MSELoss()
+        # determine which layer to set to non-trainable
+        if idx == 1:
+            for param in self.fc1.parameters():
+                param.requires_grad = False
+        elif idx == 2:
+            for param in self.fc2.parameters():
+                param.requires_grad = False
+
+    def forward(self, inputs, labels):
+        out = torch.relu(self.fc1(inputs[0]))
+        out = torch.relu(self.fc2(out))
+        out = torch.sigmoid(self.fc3(out))
+        loss = self.loss(out, labels[0])
+        return out, loss
+
 if __name__ == "__main__":
     record_v2(
         ReduceMeanLast(),
@@ -487,4 +510,25 @@ if __name__ == "__main__":
         optimizer=fc_relu_decay.getOptimizer()
     )
 
+    non_trainable_fc_idx1 = NonTrainableFC(idx=1)
+    record_v2(
+        non_trainable_fc_idx1,
+        iteration=2,
+        input_dims=[(3,3)],
+        input_dtype=[float],
+        label_dims=[(3,2)],
+        name="non_trainable_fc_idx1"
+    )
+
+    non_trainable_fc_idx2 = NonTrainableFC(idx=2)
+    record_v2(
+        non_trainable_fc_idx2,
+        iteration=2,
+        input_dims=[(3,3)],
+        input_dtype=[float],
+        label_dims=[(3,2)],
+        name="non_trainable_fc_idx2"
+    )
+    
+    # Function to check the created golden test file
     inspect_file("fc_relu_decay.nnmodelgolden")

diff --git a/test/unittest/models/unittest_models.cpp b/test/unittest/models/unittest_models.cpp
index 15c8e2c..2205dd1 100644 (file)
--- a/test/unittest/models/unittest_models.cpp
+++ b/test/unittest/models/unittest_models.cpp
@@ -50,6 +50,50 @@ IniWrapper fc_relu_decay(
    IniSection("dense_1") + fc_base + "unit = 2" + "bias_decay=0.9",
    IniSection("act_1") + act_base + "Activation = sigmoid"});
 
+/**
+ * @brief get function to make model with non-trainable fc layer
+ * @param[in] idx index of the fc layer to be non-trainable
+ * @retval function to make model with non-trainable fc layer
+ */
+std::function<std::unique_ptr<NeuralNetwork>()>
+getFuncToMakeNonTrainableFc(int idx) {
+
+  std::string fc1_trainable = (idx == 1) ? "trainable=false" : "trainable=true";
+  std::string fc2_trainable = (idx == 2) ? "trainable=false" : "trainable=true";
+
+  return [fc1_trainable, fc2_trainable]() {
+    std::unique_ptr<NeuralNetwork> nn(new NeuralNetwork());
+
+    nn->setProperty({"batch_size=3"});
+
+    auto outer_graph = makeGraph({
+      {"input", {"name=in", "input_shape=1:1:3"}},
+      {"fully_connected",
+       {"name=fc1", "input_layers=in", "unit=10", fc1_trainable}},
+      {"activation", {"name=act1", "input_layers=fc1", "activation=relu"}},
+      {"fully_connected",
+       {"name=fc2", "input_layers=act1", "unit=10", fc2_trainable}},
+      {"activation", {"name=act2", "input_layers=fc2", "activation=relu"}},
+      {"fully_connected", {"name=fc3", "input_layers=act2", "unit=2"}},
+      {"activation", {"name=act3", "input_layers=fc3", "activation=sigmoid"}},
+      {"mse", {"name=loss", "input_layers=act3"}},
+    });
+
+    for (auto &node : outer_graph) {
+      nn->addLayer(node);
+    }
+
+    nn->setOptimizer(
+      ml::train::createOptimizer("sgd", {"learning_rate = 0.1"}));
+    nn->setProperty({"input_layers=in", "label_layers=loss"});
+
+    return nn;
+  };
+}
+
+static auto makeNonTrainableFcIdx1 = getFuncToMakeNonTrainableFc(1);
+static auto makeNonTrainableFcIdx2 = getFuncToMakeNonTrainableFc(2);
+
 static std::unique_ptr<NeuralNetwork> makeMolAttention() {
   std::unique_ptr<NeuralNetwork> nn(new NeuralNetwork());
   nn->setProperty({"batch_size=3"});
@@ -882,6 +926,10 @@ GTEST_PARAMETER_TEST(
                  "transformer_pseudo_bool_attn_mask", ModelTestOption::ALL_V2),
     mkModelIniTc(fc_relu_decay, DIM_UNUSED, NOT_USED_,
                  ModelTestOption::COMPARE_V2),
+    mkModelTc_V2(makeNonTrainableFcIdx1, "non_trainable_fc_idx1",
+                 ModelTestOption::ALL_V2),
+    mkModelTc_V2(makeNonTrainableFcIdx2, "non_trainable_fc_idx2",
+                 ModelTestOption::ALL_V2),
   }),
   [](const testing::TestParamInfo<nntrainerModelTest::ParamType> &info) {
     return std::get<1>(info.param);