[Test] automate app tests

This patch automates app tests with simulator.

Signed-off-by: yelini-jeong <yelini.jeong@samsung.com>

diff --git a/tests/apptests/meson.build b/tests/apptests/meson.build
index 33acf8c..90b4fa2 100644 (file)
--- a/tests/apptests/meson.build
+++ b/tests/apptests/meson.build
@@ -3,6 +3,12 @@
 ne_apptest_inc = [ne_common_inc, ne_host_inc]
 b_install_dummy_apptests = false
 
+testenv = environment()
+testenv.set('LD_LIBRARY_PATH', ne_libdir)
+
+apptest_tvn_triv2_command = run_command(join_paths(meson.source_root(), 'utils/find_models.sh'), check : true)
+models = apptest_tvn_triv2_command.stdout().strip().split('\n')
+
 ##
 # Trinity Device Description
 # - TRIV2 (Trinity Vision2): SR/VD-NPU 2020
@@ -43,7 +49,7 @@ executable ('apptest_tvn_triv2_example',
 
 ## Apptests to run the compiled tvn models
 
-executable ('apptest_tvn_triv2_bulk',
+apptest_tvn_triv2_bulk = executable ('apptest_tvn_triv2_bulk',
   'tvn_triv2_bulk.cc',
   include_directories : ne_apptest_inc,
   dependencies : ne_test_utils_dep,
@@ -52,8 +58,9 @@ executable ('apptest_tvn_triv2_bulk',
   install_rpath : ne_libdir,
   install_dir : join_paths(ne_bindir, 'apptests')
 )
+test('apptest_tvn_triv2_bulk', apptest_tvn_triv2_bulk, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS') ], timeout: 2000)
 
-executable ('apptest_tvn_triv2',
+apptest_tvn_triv2 = executable ('apptest_tvn_triv2',
   'tvn_triv2.cc',
   include_directories : ne_apptest_inc,
   dependencies : ne_test_utils_dep,
@@ -62,8 +69,9 @@ executable ('apptest_tvn_triv2',
   install_rpath : ne_libdir,
   install_dir : join_paths(ne_bindir, 'apptests')
 )
+test('apptest_tvn_triv2', apptest_tvn_triv2, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/MOBILENET_V1') ], timeout: 100, suite: ['fast-test'])
 
-executable ('apptest_tvn_triv2_dmabuf',
+apptest_tvn_triv2_dmabuf = executable ('apptest_tvn_triv2_dmabuf',
   'tvn_triv2_dmabuf.cc',
   include_directories : ne_apptest_inc,
   dependencies : ne_test_utils_dep,
@@ -72,6 +80,11 @@ executable ('apptest_tvn_triv2_dmabuf',
   install_rpath : ne_libdir,
   install_dir : join_paths(ne_bindir, 'apptests')
 )
+test('apptest_tvn_triv2_dmabuf', apptest_tvn_triv2_dmabuf, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/MOBILENET_V1') ], timeout : 100, suite: ['fast-test'])
+
+foreach model : models
+  test('apptest_tvn_triv2_dmabuf_all', apptest_tvn_triv2_dmabuf, env: testenv, args: [ model ], timeout : 6000)
+endforeach
 
 ####
 # TODO Disable this test until VD I/F sync. is done
@@ -87,7 +100,7 @@ executable ('apptest_tvn_triv2_dmabuf',
 #)
 
 tinyxml2_dep = dependency ('tinyxml2')
-executable ('apptest_tvn_triv2_xml',
+apptest_tvn_triv2_xml = executable ('apptest_tvn_triv2_xml',
   'tvn_triv2_xml.cc',
   include_directories : ne_apptest_inc,
   dependencies : [ne_test_utils_dep, thread_dep, tinyxml2_dep],
@@ -96,8 +109,9 @@ executable ('apptest_tvn_triv2_xml',
   install_rpath : ne_libdir,
   install_dir : join_paths(ne_bindir, 'apptests')
 )
+test('apptest_tvn_triv2_xml', apptest_tvn_triv2_xml, env: testenv, args: [join_paths(meson.source_root(), 'utils/resource/model.xml')], timeout: 100, suite: ['fast-test'])
 
-executable ('apptest_tvn_triv2_profile',
+apptest_tvn_triv2_profile = executable ('apptest_tvn_triv2_profile',
   'tvn_triv2_profile.cc',
   include_directories : ne_apptest_inc,
   dependencies : ne_test_utils_dep,
@@ -106,6 +120,7 @@ executable ('apptest_tvn_triv2_profile',
   install_rpath : ne_libdir,
   install_dir : join_paths(ne_bindir, 'apptests')
 )
+test('apptest_tvn_triv2_profile', apptest_tvn_triv2_profile, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/MOBILENET_V1'), '-p', 'visa'], timeout: 100, suite: ['fast-test'])
 
 executable ('apptest_tvn_triv2_profile_example',
   'tvn_triv2_profile_example.cc',
@@ -127,7 +142,7 @@ executable ('apptest_tvn_triv2_preempt',
   install_dir : join_paths(ne_bindir, 'apptests')
 )
 
-executable ('apptest_tvn_triv2_aging',
+apptest_tvn_triv2_aging = executable ('apptest_tvn_triv2_aging',
   'tvn_triv2_aging.cc',
   include_directories : ne_apptest_inc,
   dependencies : ne_test_utils_dep,
@@ -136,8 +151,16 @@ executable ('apptest_tvn_triv2_aging',
   install_rpath : ne_libdir,
   install_dir : join_paths(ne_bindir, 'apptests')
 )
+test('apptest_tvn_triv2_aging_0', apptest_tvn_triv2_aging, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/CONV_2D_300'), '1', '10'], suite: ['fast-test'])
+test('apptest_tvn_triv2_aging_1', apptest_tvn_triv2_aging, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/CONV_2D_300'), '2', '10'], suite: ['fast-test'])
+test('apptest_tvn_triv2_aging_2', apptest_tvn_triv2_aging, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/CONV_2D_300'), '3', '10'], suite: ['fast-test'])
+test('apptest_tvn_triv2_aging_3', apptest_tvn_triv2_aging, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/CONV_2D_300'), '4', '10'], suite: ['fast-test'])
 
-executable ('apptest_tvn_triv2_interleave',
+foreach model : models
+  test('apptest_tvn_triv2_aging_all', apptest_tvn_triv2_aging, env: testenv, args: [ model, '4', '10' ], timeout : 6000)
+endforeach
+
+apptest_tvn_triv2_interleave = executable ('apptest_tvn_triv2_interleave',
   'tvn_triv2_interleave.cc',
   include_directories : ne_apptest_inc,
   dependencies : ne_test_utils_dep,
@@ -146,8 +169,9 @@ executable ('apptest_tvn_triv2_interleave',
   install_rpath : ne_libdir,
   install_dir : join_paths(ne_bindir, 'apptests')
 )
+test('apptest_tvn_triv2_interleave', apptest_tvn_triv2_interleave, env: testenv, args: [ '10', join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/MAX_POOL_2D_000'), join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/CONV_2D_300') ], suite: ['fast-test'])
 
-executable ('apptest_tvn_triv2_sigkill',
+apptest_tvn_triv2_sigkill = executable ('apptest_tvn_triv2_sigkill',
   'tvn_triv2_sigkill.cc',
   include_directories : ne_apptest_inc,
   dependencies : ne_test_utils_dep,
@@ -156,3 +180,8 @@ executable ('apptest_tvn_triv2_sigkill',
   install_rpath : ne_libdir,
   install_dir : join_paths(ne_bindir, 'apptests')
 )
+test('apptest_tvn_triv2_sigkill', apptest_tvn_triv2_sigkill, env: testenv, args: [ join_paths(ne_datadir, 'testdata/TRIV238_2TOPS/MOBILENET_V1') ], timeout : 200, suite: ['fast-test'])
+
+foreach model : models
+  test('apptest_tvn_triv2_sigkill_all', apptest_tvn_triv2_sigkill, env: testenv, args: [ model ], timeout : 6000)
+endforeach

diff --git a/tests/unittests/meson.build b/tests/unittests/meson.build
index efefa45..62302bc 100644 (file)
--- a/tests/unittests/meson.build
+++ b/tests/unittests/meson.build
@@ -9,7 +9,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_utils', unittest_ne_core_utils, env: testenv)
+  test('unittest_ne_core_utils', unittest_ne_core_utils, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_npu = executable('unittest_ne_core_npu',
     ['ne_core_npu_test.cc'],
@@ -18,7 +18,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_npu', unittest_ne_core_npu, env: testenv)
+  test('unittest_ne_core_npu', unittest_ne_core_npu, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_comm = executable('unittest_ne_core_comm',
     ['ne_core_comm_test.cc'],
@@ -28,7 +28,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_comm', unittest_ne_core_comm, env: testenv)
+  test('unittest_ne_core_comm', unittest_ne_core_comm, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_hwmem = executable('unittest_ne_core_hwmem',
     ['ne_core_hwmem_test.cc'],
@@ -37,7 +37,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_hwmem', unittest_ne_core_hwmem, env: testenv)
+  test('unittest_ne_core_hwmem', unittest_ne_core_hwmem, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_model = executable('unittest_ne_core_model',
     ['ne_core_model_test.cc'],
@@ -46,7 +46,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_model', unittest_ne_core_model, env: testenv)
+  test('unittest_ne_core_model', unittest_ne_core_model, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_buffer = executable('unittest_ne_core_buffer',
     ['ne_core_buffer_test.cc'],
@@ -55,7 +55,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_buffer', unittest_ne_core_buffer, env: testenv)
+  test('unittest_ne_core_buffer', unittest_ne_core_buffer, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_segment_table = executable('unittest_ne_core_segment_table',
     ['ne_core_segment_table_test.cc'],
@@ -64,7 +64,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_segment_table', unittest_ne_core_segment_table, env: testenv)
+  test('unittest_ne_core_segment_table', unittest_ne_core_segment_table, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_handler = executable('unittest_ne_core_handler',
     ['ne_core_handler_test.cc'],
@@ -73,7 +73,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_handler', unittest_ne_core_handler, env: testenv)
+  test('unittest_ne_core_handler', unittest_ne_core_handler, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_data = executable('unittest_ne_core_data',
     ['ne_core_data_test.cc'],
@@ -82,7 +82,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_data', unittest_ne_core_data, env: testenv)
+  test('unittest_ne_core_data', unittest_ne_core_data, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_mem = executable('unittest_ne_core_mem',
     ['ne_core_mem_test.cc'],
@@ -91,7 +91,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_mem', unittest_ne_core_mem, env: testenv)
+  test('unittest_ne_core_mem', unittest_ne_core_mem, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_sched = executable('unittest_ne_core_sched',
     ['ne_core_sched_test.cc'],
@@ -101,7 +101,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_sched', unittest_ne_core_sched, env: testenv)
+  test('unittest_ne_core_sched', unittest_ne_core_sched, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_thread_pool = executable('unittest_ne_core_thread_pool',
     ['ne_core_thread_pool_test.cc'],
@@ -111,7 +111,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_thread_pool', unittest_ne_core_thread_pool, env: testenv)
+  test('unittest_ne_core_thread_pool', unittest_ne_core_thread_pool, env: testenv, suite: ['fast-test'])
 
   unittest_ne_core_inputservice = executable('unittest_ne_core_inputservice',
     ['ne_core_inputservice_test.cc'],
@@ -121,7 +121,7 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_core_inputservice', unittest_ne_core_inputservice, env: testenv)
+  test('unittest_ne_core_inputservice', unittest_ne_core_inputservice, env: testenv, suite: ['fast-test'])
 
   unittest_ne_libnpuhost = executable('unittest_ne_libnpuhost',
     ['ne_libnpuhost_test.cc'],
@@ -131,5 +131,5 @@ if ne_test_utils_gtest_dep.found()
     install_rpath : ne_libdir,
     install_dir : join_paths(ne_bindir, 'unittests')
   )
-  test('unittest_ne_libnpuhost', unittest_ne_libnpuhost, env: testenv)
+  test('unittest_ne_libnpuhost', unittest_ne_libnpuhost, env: testenv, suite: ['fast-test'])
 endif

diff --git a/utils/find_models.sh b/utils/find_models.sh
new file mode 100755 (executable)
index 0000000..7f83496
--- /dev/null
+++ b/utils/find_models.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+for model in /opt/trinity/share/npu-engine/testdata/TRIV238_2TOPS/* ; do
+  echo $model
+done

diff --git a/utils/resource/model.xml b/utils/resource/model.xml
new file mode 100644 (file)
index 0000000..b8a738a
--- /dev/null
+++ b/utils/resource/model.xml
@@ -0,0 +1,8 @@
+<testcases>
+ <model name="mobilenet_v1" timeout="3000" priority="1">
+  <dirpath>/opt/trinity/share/npu-engine/testdata/TRIV238_2TOPS/MOBILENET_V1</dirpath>
+ </model>
+ <model name="mobilenet_v2" timeout="1000" priority="2">
+  <dirpath>/opt/trinity/share/npu-engine/testdata/TRIV238_2TOPS/MOBILENET_V2</dirpath>
+ </model>
+</testcases>