From 6f6b892dcad679d5eee3e46248d885242aa606b1 Mon Sep 17 00:00:00 2001 From: Guilherme Gallo Date: Mon, 3 Apr 2023 22:21:58 -0300 Subject: [PATCH] ci/lava: Move job definition stuff to another file The LAVA job submitter is too big, let's reorganize it a little. Signed-off-by: Guilherme Gallo Part-of: --- .gitlab-ci/lava/lava_job_submitter.py | 151 +-------------------------- .gitlab-ci/lava/utils/__init__.py | 1 + .gitlab-ci/lava/utils/lava_job_definition.py | 146 ++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 149 deletions(-) create mode 100644 .gitlab-ci/lava/utils/lava_job_definition.py diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index 5feb468..a8b27d4 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -22,7 +22,7 @@ import xmlrpc.client from datetime import datetime, timedelta from io import StringIO from os import getenv -from typing import Any, Optional +from typing import Optional import lavacli from lava.exceptions import ( @@ -39,6 +39,7 @@ from lava.utils import ( LogFollower, LogSectionType, fatal_err, + generate_lava_yaml_payload, hide_sensitive_data, print_log, ) @@ -58,154 +59,6 @@ LOG_POLLING_TIME_SEC = int(getenv("LAVA_LOG_POLLING_TIME_SEC", 5)) # How many retries should be made when a timeout happen. NUMBER_OF_RETRIES_TIMEOUT_DETECTION = int(getenv("LAVA_NUMBER_OF_RETRIES_TIMEOUT_DETECTION", 2)) -# How many attempts should be made when a timeout happen during LAVA device boot. -NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 3)) - -# Supports any integers in [0, 100]. -# The scheduler considers the job priority when ordering the queue -# to consider which job should run next. -JOB_PRIORITY = int(getenv("LAVA_JOB_PRIORITY", 75)) - - -def generate_lava_yaml_payload(args) -> dict[str, Any]: - # General metadata and permissions, plus also inexplicably kernel arguments - values = { - 'job_name': 'mesa: {}'.format(args.pipeline_info), - 'device_type': args.device_type, - 'visibility': { 'group': [ args.visibility_group ] }, - 'priority': JOB_PRIORITY, - 'context': { - 'extra_nfsroot_args': ' init=/init rootwait usbcore.quirks=0bda:8153:k' - }, - "timeouts": { - "job": {"minutes": args.job_timeout}, - "actions": { - "depthcharge-retry": { - # Could take between 1 and 1.5 min in slower boots - "minutes": 2 - }, - "depthcharge-start": { - # Should take less than 1 min. - "minutes": 1, - }, - "depthcharge-action": { - # This timeout englobes the entire depthcharge timing, - # including retries - "minutes": 2 * NUMBER_OF_ATTEMPTS_LAVA_BOOT, - }, - } - }, - } - - if args.lava_tags: - values['tags'] = args.lava_tags.split(',') - - # URLs to our kernel rootfs to boot from, both generated by the base - # container build - deploy = { - 'timeout': { 'minutes': 10 }, - 'to': 'tftp', - 'os': 'oe', - 'kernel': { - 'url': '{}/{}'.format(args.kernel_url_prefix, args.kernel_image_name), - }, - 'nfsrootfs': { - 'url': '{}/lava-rootfs.tar.zst'.format(args.rootfs_url_prefix), - 'compression': 'zstd', - } - } - if args.kernel_image_type: - deploy['kernel']['type'] = args.kernel_image_type - if args.dtb: - deploy['dtb'] = { - 'url': '{}/{}.dtb'.format(args.kernel_url_prefix, args.dtb) - } - - # always boot over NFS - boot = { - "failure_retry": NUMBER_OF_ATTEMPTS_LAVA_BOOT, - "method": args.boot_method, - "commands": "nfs", - "prompts": ["lava-shell:"], - } - - # skeleton test definition: only declaring each job as a single 'test' - # since LAVA's test parsing is not useful to us - run_steps = [] - test = { - 'timeout': { 'minutes': args.job_timeout }, - 'failure_retry': 1, - 'definitions': [ { - 'name': 'mesa', - 'from': 'inline', - 'lava-signal': 'kmsg', - 'path': 'inline/mesa.yaml', - 'repository': { - 'metadata': { - 'name': 'mesa', - 'description': 'Mesa test plan', - 'os': [ 'oe' ], - 'scope': [ 'functional' ], - 'format': 'Lava-Test Test Definition 1.0', - }, - 'run': { - "steps": run_steps - }, - }, - } ], - } - - # job execution script: - # - inline .gitlab-ci/common/init-stage1.sh - # - fetch and unpack per-pipeline build artifacts from build job - # - fetch and unpack per-job environment from lava-submit.sh - # - exec .gitlab-ci/common/init-stage2.sh - - with open(args.first_stage_init, "r") as init_sh: - run_steps += [ - x.rstrip() for x in init_sh if not x.startswith("#") and x.rstrip() - ] - run_steps.append( - f"curl -L --retry 4 -f --retry-all-errors --retry-delay 60 {args.job_rootfs_overlay_url} | tar -xz -C /", - ) - - if args.jwt_file: - with open(args.jwt_file) as jwt_file: - run_steps += [ - "set +x", - f'echo -n "{jwt_file.read()}" > "{args.jwt_file}" # HIDEME', - "set -x", - f'echo "export CI_JOB_JWT_FILE={args.jwt_file}" >> /set-job-env-vars.sh', - ] - else: - run_steps += [ - "echo Could not find jwt file, disabling MINIO requests...", - "sed -i '/MINIO_RESULTS_UPLOAD/d' /set-job-env-vars.sh", - ] - - run_steps += [ - 'mkdir -p {}'.format(args.ci_project_dir), - 'curl {} | tar --zstd -x -C {}'.format(args.build_url, args.ci_project_dir), - - # Sleep a bit to give time for bash to dump shell xtrace messages into - # console which may cause interleaving with LAVA_SIGNAL_STARTTC in some - # devices like a618. - 'sleep 1', - - # Putting CI_JOB name as the testcase name, it may help LAVA farm - # maintainers with monitoring - f"lava-test-case 'mesa-ci_{args.mesa_job_name}' --shell /init-stage2.sh", - ] - - values['actions'] = [ - { 'deploy': deploy }, - { 'boot': boot }, - { 'test': test }, - ] - - return values - - def setup_lava_proxy(): config = lavacli.load_config("default") uri, usr, tok = (config.get(key) for key in ("uri", "username", "token")) diff --git a/.gitlab-ci/lava/utils/__init__.py b/.gitlab-ci/lava/utils/__init__.py index 18bb459..ccb1b78 100644 --- a/.gitlab-ci/lava/utils/__init__.py +++ b/.gitlab-ci/lava/utils/__init__.py @@ -1,5 +1,6 @@ from .console_format import CONSOLE_LOG from .gitlab_section import GitlabSection +from .lava_job_definition import generate_lava_yaml_payload from .log_follower import ( LogFollower, fatal_err, diff --git a/.gitlab-ci/lava/utils/lava_job_definition.py b/.gitlab-ci/lava/utils/lava_job_definition.py new file mode 100644 index 0000000..554cc68 --- /dev/null +++ b/.gitlab-ci/lava/utils/lava_job_definition.py @@ -0,0 +1,146 @@ +# How many attempts should be made when a timeout happen during LAVA device boot. +from os import getenv +from typing import Any + +NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 3)) + +# Supports any integers in [0, 100]. +# The scheduler considers the job priority when ordering the queue +# to consider which job should run next. +JOB_PRIORITY = int(getenv("LAVA_JOB_PRIORITY", 75)) + + +def generate_lava_yaml_payload(args) -> dict[str, Any]: + # General metadata and permissions, plus also inexplicably kernel arguments + values = { + "job_name": "mesa: {}".format(args.pipeline_info), + "device_type": args.device_type, + "visibility": {"group": [args.visibility_group]}, + "priority": JOB_PRIORITY, + "context": { + "extra_nfsroot_args": " init=/init rootwait usbcore.quirks=0bda:8153:k" + }, + "timeouts": { + "job": {"minutes": args.job_timeout}, + "actions": { + "depthcharge-retry": { + # Could take between 1 and 1.5 min in slower boots + "minutes": 2 + }, + "depthcharge-start": { + # Should take less than 1 min. + "minutes": 1, + }, + "depthcharge-action": { + # This timeout englobes the entire depthcharge timing, + # including retries + "minutes": 2 + * NUMBER_OF_ATTEMPTS_LAVA_BOOT, + }, + }, + }, + } + + if args.lava_tags: + values["tags"] = args.lava_tags.split(",") + + # URLs to our kernel rootfs to boot from, both generated by the base + # container build + deploy = { + "timeout": {"minutes": 10}, + "to": "tftp", + "os": "oe", + "kernel": { + "url": "{}/{}".format(args.kernel_url_prefix, args.kernel_image_name), + }, + "nfsrootfs": { + "url": "{}/lava-rootfs.tar.zst".format(args.rootfs_url_prefix), + "compression": "zstd", + }, + } + if args.kernel_image_type: + deploy["kernel"]["type"] = args.kernel_image_type + if args.dtb: + deploy["dtb"] = {"url": "{}/{}.dtb".format(args.kernel_url_prefix, args.dtb)} + + # always boot over NFS + boot = { + "failure_retry": NUMBER_OF_ATTEMPTS_LAVA_BOOT, + "method": args.boot_method, + "commands": "nfs", + "prompts": ["lava-shell:"], + } + + # skeleton test definition: only declaring each job as a single 'test' + # since LAVA's test parsing is not useful to us + run_steps = [] + test = { + "timeout": {"minutes": args.job_timeout}, + "failure_retry": 1, + "definitions": [ + { + "name": "mesa", + "from": "inline", + "lava-signal": "kmsg", + "path": "inline/mesa.yaml", + "repository": { + "metadata": { + "name": "mesa", + "description": "Mesa test plan", + "os": ["oe"], + "scope": ["functional"], + "format": "Lava-Test Test Definition 1.0", + }, + "run": {"steps": run_steps}, + }, + } + ], + } + + # job execution script: + # - inline .gitlab-ci/common/init-stage1.sh + # - fetch and unpack per-pipeline build artifacts from build job + # - fetch and unpack per-job environment from lava-submit.sh + # - exec .gitlab-ci/common/init-stage2.sh + + with open(args.first_stage_init, "r") as init_sh: + run_steps += [ + x.rstrip() for x in init_sh if not x.startswith("#") and x.rstrip() + ] + run_steps.append( + f"curl -L --retry 4 -f --retry-all-errors --retry-delay 60 {args.job_rootfs_overlay_url} | tar -xz -C /", + ) + + if args.jwt_file: + with open(args.jwt_file) as jwt_file: + run_steps += [ + "set +x", + f'echo -n "{jwt_file.read()}" > "{args.jwt_file}" # HIDEME', + "set -x", + f'echo "export CI_JOB_JWT_FILE={args.jwt_file}" >> /set-job-env-vars.sh', + ] + else: + run_steps += [ + "echo Could not find jwt file, disabling MINIO requests...", + "sed -i '/MINIO_RESULTS_UPLOAD/d' /set-job-env-vars.sh", + ] + + run_steps += [ + "mkdir -p {}".format(args.ci_project_dir), + "curl {} | tar --zstd -x -C {}".format(args.build_url, args.ci_project_dir), + # Sleep a bit to give time for bash to dump shell xtrace messages into + # console which may cause interleaving with LAVA_SIGNAL_STARTTC in some + # devices like a618. + "sleep 1", + # Putting CI_JOB name as the testcase name, it may help LAVA farm + # maintainers with monitoring + f"lava-test-case 'mesa-ci_{args.mesa_job_name}' --shell /init-stage2.sh", + ] + + values["actions"] = [ + {"deploy": deploy}, + {"boot": boot}, + {"test": test}, + ] + + return values -- 2.7.4