Prepare new test configuration for test_timetest.py (#2561)

* Add logging of DB upload to tconf with refs saving

* Prepare `.automation/tgl_test_config.yml` with models from VPUX package

* Add restriction on `exe` and `model` existence in run_timetest.py

diff --git a/tests/time_tests/scripts/run_timetest.py b/tests/time_tests/scripts/run_timetest.py
index 107c43f..e9b4495 100644 (file)
--- a/tests/time_tests/scripts/run_timetest.py
+++ b/tests/time_tests/scripts/run_timetest.py
@@ -73,8 +73,8 @@ def write_aggregated_stats(stats_path, stats: dict):
 
 def prepare_executable_cmd(args: dict):
     """Generate common part of cmd from arguments to execute"""
-    return [str(args["executable"].resolve()),
-            "-m", str(args["model"].resolve()),
+    return [str(args["executable"].resolve(strict=True)),
+            "-m", str(args["model"].resolve(strict=True)),
             "-d", args["device"]]
 
 

diff --git a/tests/time_tests/test_runner/.automation/tgl_test_config.yml b/tests/time_tests/test_runner/.automation/tgl_test_config.yml
new file mode 100644 (file)
index 0000000..1ccbd23
--- /dev/null
+++ b/tests/time_tests/test_runner/.automation/tgl_test_config.yml
@@ -0,0 +1,48 @@
+- device:
+    name: CPU
+  model:
+    path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
+    precision: FP16
+    framework: caffe2
+- device:
+    name: GPU
+  model:
+    path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16/resnet-50-pytorch.xml
+    precision: FP16
+    framework: caffe2
+- device:
+    name: CPU
+  model:
+    path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
+    precision: FP16-INT8
+    framework: caffe2
+- device:
+    name: GPU
+  model:
+    path: ${VPUX_MODELS_PKG}/resnet-50-pytorch/caffe2/FP16-INT8/resnet-50-pytorch.xml
+    precision: FP16-INT8
+    framework: caffe2
+- device:
+    name: CPU
+  model:
+    path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
+    precision: FP16
+    framework: caffe2
+- device:
+    name: GPU
+  model:
+    path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16/mobilenet-v2.xml
+    precision: FP16
+    framework: caffe2
+- device:
+    name: CPU
+  model:
+    path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
+    precision: FP16-INT8
+    framework: caffe2
+- device:
+    name: GPU
+  model:
+    path: ${VPUX_MODELS_PKG}/mobilenet-v2/caffe2/FP16-INT8/mobilenet-v2.xml
+    precision: FP16-INT8
+    framework: caffe2
\ No newline at end of file

diff --git a/tests/time_tests/test_runner/conftest.py b/tests/time_tests/test_runner/conftest.py
index 43d5775..9dc0998 100644 (file)
--- a/tests/time_tests/test_runner/conftest.py
+++ b/tests/time_tests/test_runner/conftest.py
@@ -22,6 +22,7 @@ from pathlib import Path
 import yaml
 import hashlib
 import shutil
+import logging
 
 from test_runner.utils import upload_timetest_data, \
     DATABASE, DB_COLLECTIONS
@@ -147,6 +148,7 @@ def prepare_tconf_with_refs(pytestconfig):
     yield
     new_tconf_path = pytestconfig.getoption('dump_refs')
     if new_tconf_path:
+        logging.info("Save new test config with test results as references to {}".format(new_tconf_path))
         upd_cases = pytestconfig.orig_cases.copy()
         for record in pytestconfig.session_info:
             rec_i = upd_cases.index(record["orig_instance"])
@@ -227,4 +229,5 @@ def pytest_runtest_makereport(item, call):
                 data["error_msg"] = report.longrepr.reprcrash.message
             else:
                 data["status"] = "passed"
+        logging.info("Upload data to {}/{}.{}. Data: {}".format(db_url, DATABASE, db_collection, data))
         upload_timetest_data(data, db_url, db_collection)