We don't need to login anymore, but we can't use plain minio commands
now. `ci-fairy` got a helper as `s3cp` to keep an almost identical
API.
Solved Conflicts:
.gitlab-ci/common/init-stage2.sh
.gitlab-ci/container/lava_build.sh
.gitlab-ci/prepare-artifacts.sh
src/amd/ci/traces-amd.yml
src/freedreno/ci/traces-freedreno.yml
src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml
Signed-off-by: Benjamin Tissoires <benjamin.tissoires@gmail.com>
(cherry picked from commit
67cee534a88c95a8eb6839f7bcf28a5e6dac8fbf)
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19734>
rm download-git-cache.sh
set +o xtrace
CI_JOB_JWT_FILE: /minio_jwt
- MINIO_HOST: minio-packet.freedesktop.org
+ MINIO_HOST: s3.freedesktop.org
# per-pipeline artifact storage on MinIO
PIPELINE_ARTIFACTS_BASE: ${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
# per-job artifact storage on MinIO
# compress the current folder
- tar -cvzf ../$CI_PROJECT_NAME.tar.gz .
- # login with the JWT token file
- - ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
- - ci-fairy minio cp ../$CI_PROJECT_NAME.tar.gz minio://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
+ - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
# Sanity checks of MR settings and commit logs
# upload artifacts
if [ -n "$MINIO_RESULTS_UPLOAD" ]; then
tar --zstd -cf results.tar.zst results/;
- ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}";
- ci-fairy minio cp results.tar.zst minio://"$MINIO_RESULTS_UPLOAD"/results.tar.zst;
+ ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" results.tar.zst https://"$MINIO_RESULTS_UPLOAD"/results.tar.zst;
fi
# We still need to echo the hwci: mesa message, as some scripts rely on it, such
. .gitlab-ci/container/container_post_build.sh
############### Upload the files!
-ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
FILES_TO_UPLOAD="lava-rootfs.tar.zst \
$KERNEL_IMAGE_NAME"
fi
for f in $FILES_TO_UPLOAD; do
- ci-fairy minio cp /lava-files/$f \
- minio://${MINIO_PATH}/$f
+ ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \
+ https://${MINIO_PATH}/$f
done
touch /lava-files/done
-ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
+ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/done https://${MINIO_PATH}/done
echo -e "\e[0Ksection_end:$(date +%s):variables\r\e[0K"
tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
-ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
-ci-fairy minio cp job-rootfs-overlay.tar.gz "minio://${JOB_ROOTFS_OVERLAY_PATH}"
+ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"
touch results/lava.log
tail -f results/lava.log &
set -ex
INSTALL=$(realpath -s "$PWD"/install)
-MINIO_ARGS="--credentials=/tmp/.minio_credentials"
+MINIO_ARGS="--token-file ${CI_JOB_JWT_FILE}"
RESULTS=$(realpath -s "$PWD"/results)
mkdir -p "$RESULTS"
__DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
fi
- ci-fairy minio cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \
- "minio://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
+ ci-fairy s3cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \
+ "https://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
done
}
RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e"
fi
-ci-fairy minio login $MINIO_ARGS --token-file "${CI_JOB_JWT_FILE}"
-
# The replayer doesn't do any size or checksum verification for the traces in
# the replayer db, so if we had to restart the system due to intermittent device
# errors (or tried to cache replayer-db between runs, which would be nice to
# Pass needed files to the test stage
MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.zst"
zstd artifacts/install.tar -o ${MINIO_ARTIFACT_NAME}
- ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
- ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
+ ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${MINIO_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
fi
exclude:
- results/*.shader_cache
variables:
- PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-public --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE}
+ PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-public --jwt-file=${CI_JOB_JWT_FILE}
script:
- echo -e "\e[0Ksection_start:$(date +%s):variables[collapsed=true]\r\e[0KVariables passed through:"
- install/common/generate-env.sh
sudo service nginx restart
# First download will hit the internet
- wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
+ wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
# Second download should be cached.
- wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
+ wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
Now, set ``download-url`` in your ``traces-*.yml`` entry to something like
-``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public``
+``http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public``
and you should have cached downloads for traces. Add it to
``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you
can use it for cached artifact downloads instead of going all the way to
%YAML 1.2
---
traces-db:
- download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
unigine/heaven-scene1-low-d3d11.trace-dxgi:
%YAML 1.2
---
traces-db:
- download-url: "http://192.168.40.131:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "http://192.168.40.131:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
0ad/0ad-v2.trace:
- .freedreno-rules-restricted
variables:
PIGLIT_REPLAY_DESCRIPTION_FILE: "/install/restricted-traces-freedreno.yml"
- PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-private --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE}
+ PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE}
allow_failure: true
a630-traces-performance:
%YAML 1.2
---
traces-db:
- download-url: "http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-private/"
+ download-url: "http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-private/"
traces:
stellaris/Stellaris-trim--f722-v20201203-v2.trace:
%YAML 1.2
---
traces-db:
- download-url: http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/
+ download-url: http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/
text: |-
NOTE for faster runtime, try to sort tests so the longer running traces
%YAML 1.2
---
traces-db:
- download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
0ad/0ad-v2.trace:
%YAML 1.2
---
traces-db:
- download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
glxgears/glxgears-2-v2.trace:
%YAML 1.2
---
traces-db:
- download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
0ad/0ad-v2.trace:
%YAML 1.2
---
traces-db:
- download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace:
%YAML 1.2
---
traces-db:
- download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace:
%YAML 1.2
---
traces-db:
- download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
traces:
unigine/sanctuary-d3d9.trace-dxgi:
%YAML 1.2
---
traces-db:
- download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
behdad-glyphy/glyphy-v2.trace:
%YAML 1.2
---
traces-db:
- download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
+ download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
traces:
behdad-glyphy/glyphy-v2.trace: