[test] Add unittest for attention layer

This patch adds unittest for attention layer:
- unittest generator for layers is updated to work for multi-input
layers
- initial unittest for attention layer is added

Signed-off-by: Parichay Kapoor <pk.kapoor@samsung.com>

diff --git a/test/input_gen/genLayerTests.py b/test/input_gen/genLayerTests.py
index fe6e279c1efc7520a925846274ad62c6bb19db5c..e8f34aef201671ee3ea784bb1a9999ef345617fe 100644 (file)
--- a/test/input_gen/genLayerTests.py
+++ b/test/input_gen/genLayerTests.py
@@ -79,5 +79,9 @@ if __name__ == "__main__":
     record_single(conv, (1, 2, 5, 5), "conv_sb_1x1_kernel")
     record_single(conv, (3, 2, 5, 5), "conv_mb_1x1_kernel")
 
+    attention = K.layers.Attention()
+    record_single(attention, [(1, 2, 2), (1, 2, 2)],
+        "attention_golden_shared_kv", {"training": False})
+
 inspect_file("conv_sb_no_overlap.nnlayergolden")
 

diff --git a/test/input_gen/recorder.py b/test/input_gen/recorder.py
index 0e793f26a93d4395b8fd8f4bf456dd15192b93d5..1b9610c22848d75c73d307e193e124c3632f6606 100644 (file)
--- a/test/input_gen/recorder.py
+++ b/test/input_gen/recorder.py
@@ -399,7 +399,10 @@ def record(
 def record_single(layer, input_shape, test_name, call_args={}):
     layer = attach_trans_layer(layer)
     layer.build(input_shape)
-    inputs = _rand_like(input_shape)
+    if isinstance(input_shape, list):
+        inputs = [_rand_like(in_shape) for in_shape in input_shape]
+    else:
+        inputs = _rand_like(input_shape)
 
     initial_weights = [tf.Variable(i) for i in layer.weights]
 
@@ -407,7 +410,10 @@ def record_single(layer, input_shape, test_name, call_args={}):
         layer.call(inputs, **call_args) # warm layer multiple times
 
     with tf.GradientTape(persistent=True) as tape:
-        tape.watch(inputs)
+        if isinstance(inputs, list):
+            list([tape.watch(inp) for inp in inputs])
+        else:
+            tape.watch(inputs)
         outputs = layer.call(inputs, **call_args)
         dy_constant = outputs * 2  # set incoming derivative to 2 instead of 1
 
@@ -423,17 +429,19 @@ def record_single(layer, input_shape, test_name, call_args={}):
     with open(test_name + ".nnlayergolden", "wb") as f:
         writer = _get_writer(f)
 
-        def write_tensor(*tensors):
+        def write_tensor(tensors):
+            if not isinstance(tensors, list):
+                tensors = [tensors]
             for tensor in tensors:
-                # print(tensor)
+                print(tf.size(tensor))
                 writer(tf.size(tensor), tensor)
 
         ## @todo inputs outputs derivatives can be more than one
         ## @note please update genLayerTests.py comments when updating below
-        write_tensor(*initial_weights)
+        write_tensor(initial_weights)
         write_tensor(inputs)
         write_tensor(outputs)
-        write_tensor(*gradients)
-        write_tensor(*weights)
+        write_tensor(gradients)
+        write_tensor(weights)
         write_tensor(derivatives)
 

diff --git a/test/unittest/layers/unittest_layers_attention.cpp b/test/unittest/layers/unittest_layers_attention.cpp
index e47f152182db3d68185dbb1fd228fb3e4c5e2bb2..6bc4186e4d554558c80933d5f2ccb90c0fb039b4 100644 (file)
--- a/test/unittest/layers/unittest_layers_attention.cpp
+++ b/test/unittest/layers/unittest_layers_attention.cpp
@@ -22,3 +22,11 @@ auto semantic_attention =
 
 INSTANTIATE_TEST_CASE_P(Attention, LayerSemantics,
                         ::testing::Values(semantic_attention));
+
+auto attention_shared_kv = LayerGoldenTestParamType(
+  nntrainer::createLayer<nntrainer::AttentionLayer>, {}, "1:1:3:10,1:1:3:10",
+  "attention_golden_shared_kv.nnlayergolden",
+  LayerGoldenTestParamOptions::DEFAULT);
+
+INSTANTIATE_TEST_CASE_P(Attention, LayerGoldenTest,
+                        ::testing::Values(attention_shared_kv));