From 0fd2320c945bc9278a9f87e5d63214c22e232ba2 Mon Sep 17 00:00:00 2001 From: Daniel Stone Date: Thu, 10 Jun 2021 14:53:57 +0100 Subject: [PATCH] ci/lava: Clean up variable naming, document them Our variable names haven't aged very well. Rename them to make them more clear and straightforward, especially when we bring in a third rootfs element to download. Signed-off-by: Daniel Stone Part-of: --- .gitlab-ci/lava/lava-gitlab-ci.yml | 14 +++++++++++--- .gitlab-ci/lava/lava-submit.sh | 13 +++++++------ .gitlab-ci/lava/lava.yml.jinja2 | 8 ++++---- .gitlab-ci/lava/lava_job_submitter.py | 8 ++++---- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci/lava/lava-gitlab-ci.yml b/.gitlab-ci/lava/lava-gitlab-ci.yml index 96f4cc4..3d01e11 100644 --- a/.gitlab-ci/lava/lava-gitlab-ci.yml +++ b/.gitlab-ci/lava/lava-gitlab-ci.yml @@ -8,8 +8,16 @@ ENV_VARS: "DEQP_PARALLEL=6" FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PAGES_DOMAIN=${CI_PAGES_DOMAIN} CI_PROJECT_NAME=${CI_PROJECT_NAME} CI_PROJECT_DIR=${CI_PROJECT_DIR} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_PROJECT_ROOT_NAMESPACE=${CI_PROJECT_ROOT_NAMESPACE} CI_SERVER_URL=${CI_SERVER_URL} DRIVER_NAME=${DRIVER_NAME} FDO_UPSTREAM_REPO=${FDO_UPSTREAM_REPO} PIGLIT_NO_WINDOW=1 PIGLIT_REPLAY_UPLOAD_TO_MINIO=1 MINIO_HOST=${MINIO_HOST} LAVA_TEST_SCRIPT=${LAVA_TEST_SCRIPT} VK_DRIVER=${VK_DRIVER} FLAKES_CHANNEL=${FLAKES_CHANNEL}" DEQP_VERSION: gles2 - ARTIFACTS_PREFIX: "https://${MINIO_HOST}/mesa-lava" - MESA_URL: "http://caching-proxy/cache/?uri=https://${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz" + # proxy used to cache data locally + FDO_HTTP_CACHE_URI: "http://caching-proxy/cache/?uri=" + # base system generated by the container build job, shared between many pipelines + BASE_SYSTEM_HOST_PREFIX: "${MINIO_HOST}/mesa-lava" + BASE_SYSTEM_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}" + BASE_SYSTEM_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${ARCH}" + # per-pipeline/job build artifacts + PIPELINE_ARTIFACTS_BASE: "${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}" + MESA_BUILD_PATH: "${PIPELINE_ARTIFACTS_BASE}/mesa-${ARCH}.tar.gz" + JOB_RESULTS_PATH: "${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}/results.tar.gz" script: - ./artifacts/lava/lava-submit.sh artifacts: @@ -20,7 +28,7 @@ exclude: - results/*.shader_cache after_script: - - wget -q "https://minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/${CI_JOB_ID}/results.tar.gz" -O- | tar -xz + - wget -q "https://${JOB_RESULTS_PATH}" -O- | tar -xz .lava-test:armhf: variables: diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index f3e6717..fd83c59 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -6,18 +6,19 @@ set -x rm -rf results mkdir -p results -# Try to use the kernel and rootfs built in mainline first, to save cycles -if wget -q --method=HEAD "${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}/done"; then - ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}" +# Try to use the kernel and rootfs built in mainline first, so we're more +# likely to hit cache +if wget -q --method=HEAD "https://${BASE_SYSTEM_MAINLINE_HOST_PATH}/done"; then + BASE_SYSTEM_HOST_PATH="${BASE_SYSTEM_MAINLINE_HOST_PATH}" else - ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${ARCH}" + BASE_SYSTEM_HOST_PATH="${BASE_SYSTEM_FORK_HOST_PATH}" fi artifacts/lava/lava_job_submitter.py \ --template artifacts/lava/lava.yml.jinja2 \ --pipeline-info "$CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \ - --base-artifacts-url ${ARTIFACTS_URL} \ - --mesa-url ${MESA_URL} \ + --base-system-url-prefix "https://${BASE_SYSTEM_HOST_PATH}" \ + --mesa-build-url "${FDO_HTTP_CACHE_URI:-}https://${MESA_BUILD_PATH}" \ --device-type ${DEVICE_TYPE} \ --dtb ${DTB} \ --env-vars "${ENV_VARS} ${FIXED_ENV_VARS}" \ diff --git a/.gitlab-ci/lava/lava.yml.jinja2 b/.gitlab-ci/lava/lava.yml.jinja2 index f33988ae4..f55539a 100644 --- a/.gitlab-ci/lava/lava.yml.jinja2 +++ b/.gitlab-ci/lava/lava.yml.jinja2 @@ -22,16 +22,16 @@ actions: minutes: 10 to: tftp kernel: - url: {{ base_artifacts_url }}/{{ kernel_image_name }} + url: {{ base_system_url_prefix }}/{{ kernel_image_name }} {% if kernel_image_type %} {{ kernel_image_type }} {% endif %} nfsrootfs: - url: {{ base_artifacts_url }}/lava-rootfs.tgz + url: {{ base_system_url_prefix }}/lava-rootfs.tgz compression: gz {% if dtb %} dtb: - url: {{ base_artifacts_url }}/{{ dtb }}.dtb + url: {{ base_system_url_prefix }}/{{ dtb }}.dtb {% endif %} os: oe - boot: @@ -82,7 +82,7 @@ actions: - echo "$NFS_SERVER_IP caching-proxy" >> /etc/hosts - for i in 1 2 3; do sntp -sS pool.ntp.org && break || sleep 2; done - - wget -S --progress=dot:giga -O- {{ mesa_url }} | tar -xz + - wget -S --progress=dot:giga -O- {{ mesa_build_url }} | tar -xz - mkdir -p $CI_PROJECT_DIR - ln -sf /install $CI_PROJECT_DIR/install diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index 5fee2d7..9ed098c 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -55,8 +55,8 @@ def generate_lava_yaml(args): values = {} values['pipeline_info'] = args.pipeline_info - values['base_artifacts_url'] = args.base_artifacts_url - values['mesa_url'] = args.mesa_url + values['base_system_url_prefix'] = args.base_system_url_prefix + values['mesa_build_url'] = args.mesa_build_url values['device_type'] = args.device_type values['dtb'] = args.dtb values['kernel_image_name'] = args.kernel_image_name @@ -201,8 +201,8 @@ if __name__ == '__main__': parser.add_argument("--template") parser.add_argument("--pipeline-info") - parser.add_argument("--base-artifacts-url") - parser.add_argument("--mesa-url") + parser.add_argument("--base-system-url-prefix") + parser.add_argument("--mesa-build-url") parser.add_argument("--device-type") parser.add_argument("--dtb", nargs='?', default="") parser.add_argument("--kernel-image-name") -- 2.7.4