Support DB reporting for test_timetest.py (#2359)

diff --git a/tests/time_tests/test_runner/conftest.py b/tests/time_tests/test_runner/conftest.py
index 02de0f2c505ef7ff0c4d94e32f6545a9c40e0997..b4f87d0fc7492fa8dfccc1310f2d0f5a667e2703 100644 (file)
--- a/tests/time_tests/test_runner/conftest.py
+++ b/tests/time_tests/test_runner/conftest.py
@@ -16,37 +16,65 @@ This plugin adds the following command-line options:
 """
 
 # pylint:disable=import-error
+import sys
 import pytest
 from pathlib import Path
 import yaml
+import hashlib
 
-from test_runner.utils import expand_env_vars
+from test_runner.utils import expand_env_vars, upload_timetest_data, \
+    DATABASE, DB_COLLECTIONS
 
 
 # -------------------- CLI options --------------------
 
+
 def pytest_addoption(parser):
     """Specify command-line options for all plugins"""
-    parser.addoption(
+    test_args_parser = parser.getgroup("timetest test run")
+    test_args_parser.addoption(
         "--test_conf",
         type=Path,
-        help="Path to test config",
+        help="path to a test config",
         default=Path(__file__).parent / "test_config.yml"
     )
-    parser.addoption(
+    test_args_parser.addoption(
         "--exe",
         required=True,
         dest="executable",
         type=Path,
-        help="Path to a timetest binary to execute",
+        help="path to a timetest binary to execute"
     )
-    parser.addoption(
+    test_args_parser.addoption(
         "--niter",
         type=int,
-        help="Number of iterations to run executable and aggregate results",
+        help="number of iterations to run executable and aggregate results",
         default=3
     )
     # TODO: add support of --mo, --omz etc. required for OMZ support
+    db_args_parser = parser.getgroup("timetest database use")
+    db_args_parser.addoption(
+        '--db_submit',
+        metavar="RUN_ID",
+        type=str,
+        help='submit results to the database. ' \
+             '`RUN_ID` should be a string uniquely identifying the run' \
+             ' (like Jenkins URL or time)'
+    )
+    is_db_used = db_args_parser.parser.parse_known_args(sys.argv).db_submit
+    db_args_parser.addoption(
+        '--db_url',
+        type=str,
+        required=is_db_used,
+        help='MongoDB URL in a form "mongodb://server:port"'
+    )
+    db_args_parser.addoption(
+        '--db_collection',
+        type=str,
+        required=is_db_used,
+        help='collection name in "{}" database'.format(DATABASE),
+        choices=DB_COLLECTIONS
+    )
 
 
 @pytest.fixture(scope="session")
@@ -96,3 +124,47 @@ def pytest_make_parametrize_id(config, val, argname):
     values = list(get_dict_values(val))
 
     return "-".join(["_".join([key, val]) for key, val in zip(keys, values)])
+
+
+@pytest.mark.hookwrapper
+def pytest_runtest_makereport(item, call):
+    """Pytest hook for report preparation.
+
+    Submit tests' data to a database.
+    """
+
+    FIELDS_FOR_ID = ['timetest', 'model', 'device', 'niter', 'run_id']
+    FIELDS_FOR_SUBMIT = FIELDS_FOR_ID + ['_id', 'test_name',
+                                         'results', 'status', 'error_msg']
+
+    run_id = item.config.getoption("db_submit")
+    db_url = item.config.getoption("db_url")
+    db_collection = item.config.getoption("db_collection")
+    if not (run_id and db_url and db_collection):
+        yield
+        return
+
+    data = item.funcargs.copy()
+    data["timetest"] = data.pop("executable").stem
+    data.update({key: val for key, val in data["instance"].items()})
+
+    data['run_id'] = run_id
+    data['_id'] = hashlib.sha256(
+        ''.join([str(data[key]) for key in FIELDS_FOR_ID]).encode()).hexdigest()
+
+    data["test_name"] = item.name
+    data["results"] = {}
+    data["status"] = "not_finished"
+    data["error_msg"] = ""
+
+    data = {field: data[field] for field in FIELDS_FOR_SUBMIT}
+
+    report = (yield).get_result()
+    if call.when in ["setup", "call"]:
+        if call.when == "call":
+            if not report.passed:
+                data["status"] = "failed"
+                data["error_msg"] = report.longrepr.reprcrash.message
+            else:
+                data["status"] = "passed"
+        upload_timetest_data(data, db_url, db_collection)

diff --git a/tests/time_tests/test_runner/test_timetest.py b/tests/time_tests/test_runner/test_timetest.py
index 82858ec25221e466b1632e5b144c56bb95bff4e3..0f91c62a22326158448045e716c915a23384f6c0 100644 (file)
--- a/tests/time_tests/test_runner/test_timetest.py
+++ b/tests/time_tests/test_runner/test_timetest.py
@@ -40,3 +40,5 @@ def test_timetest(instance, executable, niter):
     }
     retcode, aggr_stats = run_timetest(exe_args, log=logging)
     assert retcode == 0, "Run of executable failed"
+
+    instance["results"] = aggr_stats    # append values to report to DB

diff --git a/tests/time_tests/test_runner/utils.py b/tests/time_tests/test_runner/utils.py
index d1767d8c77e1b83dc810c5bc2a49588c7f4308a6..8cce2e1cd2d4b1afc0f956769e28202b44a38918 100644 (file)
--- a/tests/time_tests/test_runner/utils.py
+++ b/tests/time_tests/test_runner/utils.py
@@ -4,6 +4,11 @@
 """Utility module."""
 
 import os
+from pymongo import MongoClient
+
+# constants
+DATABASE = 'timetests'   # database name for timetests results
+DB_COLLECTIONS = ["commit", "nightly", "weekly"]
 
 
 def expand_env_vars(obj):
@@ -18,3 +23,11 @@ def expand_env_vars(obj):
     else:
         obj = os.path.expandvars(obj)
     return obj
+
+
+def upload_timetest_data(data, db_url, db_collection):
+    """ Upload timetest data to database
+    """
+    client = MongoClient(db_url)
+    collection = client[DATABASE][db_collection]
+    collection.replace_one({'_id': data['_id']}, data, upsert=True)