From 18d80f25ee242ec21a4f772d107551e8dadae955 Mon Sep 17 00:00:00 2001 From: Guilherme Gallo Date: Wed, 23 Mar 2022 22:10:51 -0300 Subject: [PATCH] ci/lava: Parse all test cases from 0_mesa suite LAVA can filter which test suite to show the results from, let's list all testcases possible in the mesa test suite, to be able to divide more complex jobs into test_cases. Another advantage is that the test case can vary its name. Signed-off-by: Guilherme Gallo Part-of: --- .gitlab-ci/lava/lava_job_submitter.py | 40 ++++++++----- .gitlab-ci/tests/test_lava_job_submitter.py | 91 +++++++++++++++++++++-------- 2 files changed, 95 insertions(+), 36 deletions(-) diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index 05fca6b..fc1c1aa 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -267,7 +267,7 @@ class LAVAJob(): fatal_err(f"Could not get LAVA job logs. Reason: {mesa_exception}") -def get_job_results(proxy, job_id, test_suite, test_case): +def get_job_results(proxy, job_id, test_suite): # Look for infrastructure errors and retry if we see them. results_yaml = _call_proxy(proxy.results.get_testjob_results_yaml, job_id) results = yaml.load(results_yaml, Loader=loader(False)) @@ -275,19 +275,33 @@ def get_job_results(proxy, job_id, test_suite, test_case): metadata = res["metadata"] if "result" not in metadata or metadata["result"] != "fail": continue - if 'error_type' in metadata and metadata['error_type'] == "Infrastructure": - raise MesaCIException("LAVA job {} failed with Infrastructure Error. Retry.".format(job_id)) - if 'case' in metadata and metadata['case'] == "validate": - raise MesaCIException("LAVA job {} failed validation (possible download error). Retry.".format(job_id)) + if "error_type" in metadata and metadata["error_type"] == "Infrastructure": + raise MesaCIException( + f"LAVA job {job_id} failed with Infrastructure Error. Retry." + ) + if "case" in metadata and metadata["case"] == "validate": + raise MesaCIException( + f"LAVA job {job_id} failed validation (possible download error). Retry." + ) - results_yaml = _call_proxy(proxy.results.get_testcase_results_yaml, job_id, test_suite, test_case) - results = yaml.load(results_yaml, Loader=loader(False)) + results_yaml = _call_proxy( + proxy.results.get_testsuite_results_yaml, job_id, test_suite + ) + results: list = yaml.load(results_yaml, Loader=loader(False)) if not results: - raise MesaCIException("LAVA: no result for test_suite '{}', test_case '{}'".format(test_suite, test_case)) - - print_log("LAVA: result for test_suite '{}', test_case '{}': {}".format(test_suite, test_case, results[0]['result'])) - if results[0]['result'] != 'pass': - return False + raise MesaCIException( + f"LAVA: no result for test_suite '{test_suite}'" + ) + + for metadata in results: + test_case = metadata["name"] + result = metadata["metadata"]["result"] + print_log( + f"LAVA: result for test_suite '{test_suite}', " + f"test_case '{test_case}': {result}" + ) + if result != "pass": + return False return True @@ -353,7 +367,7 @@ def follow_job_execution(job): print(line) show_job_data(job) - return get_job_results(job.proxy, job.job_id, "0_mesa", "mesa") + return get_job_results(job.proxy, job.job_id, "0_mesa") def retriable_follow_job(proxy, job_definition): diff --git a/.gitlab-ci/tests/test_lava_job_submitter.py b/.gitlab-ci/tests/test_lava_job_submitter.py index 43896db..bd21958 100644 --- a/.gitlab-ci/tests/test_lava_job_submitter.py +++ b/.gitlab-ci/tests/test_lava_job_submitter.py @@ -26,19 +26,20 @@ import xmlrpc.client from contextlib import nullcontext as does_not_raise from datetime import datetime from itertools import cycle, repeat -from typing import Iterable, Union, Generator, Tuple +from typing import Generator, Iterable, Tuple, Union from unittest.mock import MagicMock, patch import pytest import yaml from freezegun import freeze_time from lava.lava_job_submitter import ( - NUMBER_OF_RETRIES_TIMEOUT_DETECTION, DEVICE_HANGING_TIMEOUT_SEC, + NUMBER_OF_RETRIES_TIMEOUT_DETECTION, + LAVAJob, follow_job_execution, + get_job_results, hide_sensitive_data, retriable_follow_job, - LAVAJob ) NUMBER_OF_MAX_ATTEMPTS = NUMBER_OF_RETRIES_TIMEOUT_DETECTION + 1 @@ -51,35 +52,37 @@ def jobs_logs_response(finished=False, msg=None, lvl="target") -> Tuple[bool, st return finished, yaml.safe_dump(logs) -def result_get_testjob_results_response() -> str: - result = {"result": "test"} - results = [{"metadata": result}] - - return yaml.safe_dump(results) - +RESULT_GET_TESTJOB_RESULTS = [{"metadata": {"result": "test"}}] -def result_get_testcase_results_response() -> str: - result = {"result": "pass"} - test_cases = [result] - return yaml.safe_dump(test_cases) +def generate_testsuite_result(name="test-mesa-ci", result="pass", metadata_extra = None, extra = None): + if metadata_extra is None: + metadata_extra = {} + if extra is None: + extra = {} + return {"metadata": {"result": result, **metadata_extra}, "name": name} @pytest.fixture def mock_proxy(): - def create_proxy_mock(**kwargs): + def create_proxy_mock( + job_results=RESULT_GET_TESTJOB_RESULTS, + testsuite_results=[generate_testsuite_result()], + **kwargs + ): proxy_mock = MagicMock() proxy_submit_mock = proxy_mock.scheduler.jobs.submit proxy_submit_mock.return_value = "1234" proxy_results_mock = proxy_mock.results.get_testjob_results_yaml - proxy_results_mock.return_value = result_get_testjob_results_response() + proxy_results_mock.return_value = yaml.safe_dump(job_results) - proxy_test_cases_mock = proxy_mock.results.get_testcase_results_yaml - proxy_test_cases_mock.return_value = result_get_testcase_results_response() + proxy_test_suites_mock = proxy_mock.results.get_testsuite_results_yaml + proxy_test_suites_mock.return_value = yaml.safe_dump(testsuite_results) proxy_logs_mock = proxy_mock.scheduler.jobs.logs proxy_logs_mock.return_value = jobs_logs_response() + for key, value in kwargs.items(): setattr(proxy_logs_mock, key, value) @@ -156,56 +159,93 @@ NETWORK_EXCEPTION = xmlrpc.client.ProtocolError("", 0, "test", {}) XMLRPC_FAULT = xmlrpc.client.Fault(0, "test") PROXY_SCENARIOS = { - "finish case": (generate_n_logs(1), does_not_raise(), True), + "finish case": (generate_n_logs(1), does_not_raise(), True, {}), "works at last retry": ( generate_n_logs(n=NUMBER_OF_MAX_ATTEMPTS, tick_fn=[ DEVICE_HANGING_TIMEOUT_SEC + 1 ] * NUMBER_OF_RETRIES_TIMEOUT_DETECTION + [1]), does_not_raise(), True, + {}, ), "timed out more times than retry attempts": ( generate_n_logs(n=4, tick_fn=DEVICE_HANGING_TIMEOUT_SEC + 1), pytest.raises(SystemExit), False, + {}, ), "long log case, no silence": ( generate_n_logs(n=1000, tick_fn=0), does_not_raise(), True, + {}, + ), + "no retries, testsuite succeed": ( + generate_n_logs(n=1, tick_fn=0), + does_not_raise(), + True, + { + "testsuite_results": [ + generate_testsuite_result(result="pass") + ] + }, + ), + "no retries, but testsuite fails": ( + generate_n_logs(n=1, tick_fn=0), + does_not_raise(), + False, + { + "testsuite_results": [ + generate_testsuite_result(result="fail") + ] + }, + ), + "no retries, one testsuite fails": ( + generate_n_logs(n=1, tick_fn=0), + does_not_raise(), + False, + { + "testsuite_results": [ + generate_testsuite_result(result="fail"), + generate_testsuite_result(result="pass") + ] + }, ), "very long silence": ( generate_n_logs(n=NUMBER_OF_MAX_ATTEMPTS + 1, tick_fn=100000), pytest.raises(SystemExit), False, + {}, ), # If a protocol error happens, _call_proxy will retry without affecting timeouts "unstable connection, ProtocolError followed by final message": ( (NETWORK_EXCEPTION, jobs_logs_response(finished=True)), does_not_raise(), True, + {}, ), # After an arbitrary number of retries, _call_proxy should call sys.exit "unreachable case, subsequent ProtocolErrors": ( repeat(NETWORK_EXCEPTION), pytest.raises(SystemExit), False, + {}, ), - "XMLRPC Fault": ([XMLRPC_FAULT], pytest.raises(SystemExit, match="1"), False), + "XMLRPC Fault": ([XMLRPC_FAULT], pytest.raises(SystemExit, match="1"), False, {}), } @patch("time.sleep", return_value=None) # mock sleep to make test faster @pytest.mark.parametrize( - "side_effect, expectation, has_finished", + "side_effect, expectation, job_result, proxy_args", PROXY_SCENARIOS.values(), ids=PROXY_SCENARIOS.keys(), ) def test_retriable_follow_job( - mock_sleep, side_effect, expectation, has_finished, mock_proxy + mock_sleep, side_effect, expectation, job_result, proxy_args, mock_proxy ): with expectation: - proxy = mock_proxy(side_effect=side_effect) + proxy = mock_proxy(side_effect=side_effect, **proxy_args) result = retriable_follow_job(proxy, "") - assert has_finished == result + assert job_result == result WAIT_FOR_JOB_SCENARIOS = { @@ -271,3 +311,8 @@ def test_hide_sensitive_data(input, expectation, tag): result = yaml.safe_load(yaml_result) assert result == expectation + + +def test_get_job_results(mock_proxy): + proxy = mock_proxy() + get_job_results(proxy, 1, "0_mesa") \ No newline at end of file -- 2.7.4