From 12b690952c5b9ba283e570d04cd10018dfbc29e9 Mon Sep 17 00:00:00 2001 From: Vitaliy Urusovskij Date: Wed, 14 Oct 2020 20:05:52 +0300 Subject: [PATCH] Time tests improvements (#2642) * Remove extra functions from run_timetest.py * Add `log.debug` of raw and aggregated statistics in run_timetest.py * Implement storing of models locally for test_timetest.py --- tests/time_tests/scripts/run_timetest.py | 27 +++++++++++---------------- tests/time_tests/test_runner/conftest.py | 11 +++++++++++ tests/time_tests/test_runner/test_timetest.py | 13 +++++++++++-- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/tests/time_tests/scripts/run_timetest.py b/tests/time_tests/scripts/run_timetest.py index e9b4495..8dc1d6e 100644 --- a/tests/time_tests/scripts/run_timetest.py +++ b/tests/time_tests/scripts/run_timetest.py @@ -50,14 +50,6 @@ def run_cmd(args: list, log=None, verbose=True): return proc.returncode, ''.join(output) -def read_stats(stats_path, stats: dict): - """Read statistics from a file and extend provided statistics""" - with open(stats_path, "r") as file: - parsed_data = yaml.safe_load(file) - return dict((step_name, stats.get(step_name, []) + [duration]) - for step_name, duration in parsed_data.items()) - - def aggregate_stats(stats: dict): """Aggregate provided statistics""" return {step_name: {"avg": statistics.mean(duration_list), @@ -65,12 +57,6 @@ def aggregate_stats(stats: dict): for step_name, duration_list in stats.items()} -def write_aggregated_stats(stats_path, stats: dict): - """Write aggregated statistics to a file in YAML format""" - with open(stats_path, "w") as file: - yaml.safe_dump(stats, file) - - def prepare_executable_cmd(args: dict): """Generate common part of cmd from arguments to execute""" return [str(args["executable"].resolve(strict=True)), @@ -96,10 +82,18 @@ def run_timetest(args: dict, log=None): "Statistics aggregation is skipped.".format(args["executable"], retcode, msg)) return retcode, {} - stats = read_stats(tmp_stats_path, stats) + # Read raw statistics + with open(tmp_stats_path, "r") as file: + raw_data = yaml.safe_load(file) + log.debug("Raw statistics after run of executable #{}: {}".format(run_iter, raw_data)) + + # Combine statistics from several runs + stats = dict((step_name, stats.get(step_name, []) + [duration]) + for step_name, duration in raw_data.items()) # Aggregate results aggregated_stats = aggregate_stats(stats) + log.debug("Aggregated statistics after full run: {}".format(aggregated_stats)) return 0, aggregated_stats @@ -154,7 +148,8 @@ if __name__ == "__main__": if args.stats_path: # Save aggregated results to a file - write_aggregated_stats(args.stats_path, aggr_stats) + with open(args.stats_path, "w") as file: + yaml.safe_dump(aggr_stats, file) logging.info("Aggregated statistics saved to a file: '{}'".format( args.stats_path.resolve())) else: diff --git a/tests/time_tests/test_runner/conftest.py b/tests/time_tests/test_runner/conftest.py index fb950bd..e7e7c14 100644 --- a/tests/time_tests/test_runner/conftest.py +++ b/tests/time_tests/test_runner/conftest.py @@ -24,6 +24,7 @@ import yaml import hashlib import shutil import logging +import tempfile from test_runner.utils import upload_timetest_data, \ DATABASE, DB_COLLECTIONS @@ -108,6 +109,16 @@ def niter(request): @pytest.fixture(scope="function") +def temp_dir(pytestconfig): + """Create temporary directory for test purposes. + It will be cleaned up after every test run. + """ + temp_dir = tempfile.TemporaryDirectory() + yield Path(temp_dir.name) + temp_dir.cleanup() + + +@pytest.fixture(scope="function") def cl_cache_dir(pytestconfig): """Generate directory to save OpenCL cache before test run and clean up after run. diff --git a/tests/time_tests/test_runner/test_timetest.py b/tests/time_tests/test_runner/test_timetest.py index bd91cb9..9c1e11d 100644 --- a/tests/time_tests/test_runner/test_timetest.py +++ b/tests/time_tests/test_runner/test_timetest.py @@ -17,6 +17,7 @@ Options[*]: from pathlib import Path import logging import os +import shutil from scripts.run_timetest import run_timetest from test_runner.utils import expand_env_vars @@ -24,22 +25,30 @@ from test_runner.utils import expand_env_vars REFS_FACTOR = 1.2 # 120% -def test_timetest(instance, executable, niter, cl_cache_dir, test_info): +def test_timetest(instance, executable, niter, cl_cache_dir, test_info, temp_dir): """Parameterized test. :param instance: test instance. Should not be changed during test run :param executable: timetest executable to run :param niter: number of times to run executable + :param cl_cache_dir: directory to store OpenCL cache :param test_info: custom `test_info` field of built-in `request` pytest fixture + :param temp_dir: path to a temporary directory. Will be cleaned up after test run """ # Prepare model to get model_path model_path = instance["model"].get("path") assert model_path, "Model path is empty" + model_path = Path(expand_env_vars(model_path)) + + # Copy model to a local temporary directory + model_dir = temp_dir / "model" + shutil.copytree(model_path.parent, model_dir) + model_path = model_dir / model_path.name # Run executable exe_args = { "executable": Path(executable), - "model": Path(expand_env_vars(model_path)), + "model": Path(model_path), "device": instance["device"]["name"], "niter": niter } -- 2.7.4