CI: convert to use the new S3 server instead of the legacy minio

We don't need to login anymore, but we can't use plain minio commands
now. `ci-fairy` got a helper as `s3cp` to keep an almost identical
API.

Solved Conflicts:
	.gitlab-ci/common/init-stage2.sh
	.gitlab-ci/container/lava_build.sh
	.gitlab-ci/prepare-artifacts.sh
	src/amd/ci/traces-amd.yml
	src/freedreno/ci/traces-freedreno.yml
	src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml

Signed-off-by: Benjamin Tissoires <benjamin.tissoires@gmail.com>
(cherry picked from commit 67cee534a88c95a8eb6839f7bcf28a5e6dac8fbf)

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19734>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index acd0181..e3c170e 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,7 +8,7 @@ variables:
           rm download-git-cache.sh
           set +o xtrace
   CI_JOB_JWT_FILE: /minio_jwt
-  MINIO_HOST: minio-packet.freedesktop.org
+  MINIO_HOST: s3.freedesktop.org
   # per-pipeline artifact storage on MinIO
   PIPELINE_ARTIFACTS_BASE: ${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
   # per-job artifact storage on MinIO
@@ -264,9 +264,7 @@ make git archive:
     # compress the current folder
     - tar -cvzf ../$CI_PROJECT_NAME.tar.gz .
 
-    # login with the JWT token file
-    - ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
-    - ci-fairy minio cp ../$CI_PROJECT_NAME.tar.gz minio://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
+    - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
 
 
 # Sanity checks of MR settings and commit logs

diff --git a/.gitlab-ci/common/init-stage2.sh b/.gitlab-ci/common/init-stage2.sh
index ad6528d..a0718bf 100755 (executable)
--- a/.gitlab-ci/common/init-stage2.sh
+++ b/.gitlab-ci/common/init-stage2.sh
@@ -150,8 +150,7 @@ cleanup
 # upload artifacts
 if [ -n "$MINIO_RESULTS_UPLOAD" ]; then
   tar --zstd -cf results.tar.zst results/;
-  ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}";
-  ci-fairy minio cp results.tar.zst minio://"$MINIO_RESULTS_UPLOAD"/results.tar.zst;
+  ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" results.tar.zst https://"$MINIO_RESULTS_UPLOAD"/results.tar.zst;
 fi
 
 # We still need to echo the hwci: mesa message, as some scripts rely on it, such

diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh
index 59276ec..8cfcc39 100755 (executable)
--- a/.gitlab-ci/container/lava_build.sh
+++ b/.gitlab-ci/container/lava_build.sh
@@ -267,7 +267,6 @@ popd
 . .gitlab-ci/container/container_post_build.sh
 
 ############### Upload the files!
-ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
 FILES_TO_UPLOAD="lava-rootfs.tar.zst \
                  $KERNEL_IMAGE_NAME"
 
@@ -276,9 +275,9 @@ if [[ -n $DEVICE_TREES ]]; then
 fi
 
 for f in $FILES_TO_UPLOAD; do
-    ci-fairy minio cp /lava-files/$f \
-             minio://${MINIO_PATH}/$f
+    ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \
+             https://${MINIO_PATH}/$f
 done
 
 touch /lava-files/done
-ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done
+ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/done https://${MINIO_PATH}/done

diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh
index 8cce9ba..d0df3c7 100755 (executable)
--- a/.gitlab-ci/lava/lava-submit.sh
+++ b/.gitlab-ci/lava/lava-submit.sh
@@ -26,8 +26,7 @@ cat results/job-rootfs-overlay/set-job-env-vars.sh
 echo -e "\e[0Ksection_end:$(date +%s):variables\r\e[0K"
 
 tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
-ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
-ci-fairy minio cp job-rootfs-overlay.tar.gz "minio://${JOB_ROOTFS_OVERLAY_PATH}"
+ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"
 
 touch results/lava.log
 tail -f results/lava.log &

diff --git a/.gitlab-ci/piglit/piglit-traces.sh b/.gitlab-ci/piglit/piglit-traces.sh
index ddb2fea..089b744 100755 (executable)
--- a/.gitlab-ci/piglit/piglit-traces.sh
+++ b/.gitlab-ci/piglit/piglit-traces.sh
@@ -3,7 +3,7 @@
 set -ex
 
 INSTALL=$(realpath -s "$PWD"/install)
-MINIO_ARGS="--credentials=/tmp/.minio_credentials"
+MINIO_ARGS="--token-file ${CI_JOB_JWT_FILE}"
 
 RESULTS=$(realpath -s "$PWD"/results)
 mkdir -p "$RESULTS"
@@ -137,8 +137,8 @@ replay_minio_upload_images() {
             __DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
         fi
 
-        ci-fairy minio cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \
-            "minio://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
+        ci-fairy s3cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \
+            "https://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
     done
 }
 
@@ -173,8 +173,6 @@ if [ "$RUN_CMD_WRAPPER" ]; then
     RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e"
 fi
 
-ci-fairy minio login $MINIO_ARGS --token-file "${CI_JOB_JWT_FILE}"
-
 # The replayer doesn't do any size or checksum verification for the traces in
 # the replayer db, so if we had to restart the system due to intermittent device
 # errors (or tried to cache replayer-db between runs, which would be nice to

diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index 3933a2c..bd098d5 100755 (executable)
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -54,6 +54,5 @@ if [ -n "$MINIO_ARTIFACT_NAME" ]; then
     # Pass needed files to the test stage
     MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.zst"
     zstd artifacts/install.tar -o ${MINIO_ARTIFACT_NAME}
-    ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
-    ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
+    ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${MINIO_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
 fi

diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml
index 1b15116..e82b7e0 100644 (file)
--- a/.gitlab-ci/test/gitlab-ci.yml
+++ b/.gitlab-ci/test/gitlab-ci.yml
@@ -87,7 +87,7 @@ rustfmt:
     exclude:
       - results/*.shader_cache
   variables:
-    PIGLIT_REPLAY_EXTRA_ARGS:  --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-public --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE}
+    PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-public --jwt-file=${CI_JOB_JWT_FILE}
   script:
     - echo -e "\e[0Ksection_start:$(date +%s):variables[collapsed=true]\r\e[0KVariables passed through:"
     - install/common/generate-env.sh

diff --git a/docs/ci/bare-metal.rst b/docs/ci/bare-metal.rst
index f8c51b2..544f418 100644 (file)
--- a/docs/ci/bare-metal.rst
+++ b/docs/ci/bare-metal.rst
@@ -219,12 +219,12 @@ Enable the site and restart nginx:
   sudo service nginx restart
 
   # First download will hit the internet
-  wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
+  wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
   # Second download should be cached.
-  wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
+  wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace
 
 Now, set ``download-url`` in your ``traces-*.yml`` entry to something like
-``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public``
+``http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public``
 and you should have cached downloads for traces.  Add it to
 ``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you
 can use it for cached artifact downloads instead of going all the way to

diff --git a/src/amd/ci/traces-amd.yml b/src/amd/ci/traces-amd.yml
index e2f7025..63da700 100644 (file)
--- a/src/amd/ci/traces-amd.yml
+++ b/src/amd/ci/traces-amd.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   unigine/heaven-scene1-low-d3d11.trace-dxgi:

diff --git a/src/broadcom/ci/traces-broadcom.yml b/src/broadcom/ci/traces-broadcom.yml
index 07baf34..1f79b62 100644 (file)
--- a/src/broadcom/ci/traces-broadcom.yml
+++ b/src/broadcom/ci/traces-broadcom.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "http://192.168.40.131:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "http://192.168.40.131:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   0ad/0ad-v2.trace:

diff --git a/src/freedreno/ci/gitlab-ci.yml b/src/freedreno/ci/gitlab-ci.yml
index 13949d4..f15e464 100644 (file)
--- a/src/freedreno/ci/gitlab-ci.yml
+++ b/src/freedreno/ci/gitlab-ci.yml
@@ -277,7 +277,7 @@ a630-traces-restricted:
     - .freedreno-rules-restricted
   variables:
     PIGLIT_REPLAY_DESCRIPTION_FILE: "/install/restricted-traces-freedreno.yml"
-    PIGLIT_REPLAY_EXTRA_ARGS:  --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-private --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE}
+    PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE}
   allow_failure: true
 
 a630-traces-performance:

diff --git a/src/freedreno/ci/restricted-traces-freedreno.yml b/src/freedreno/ci/restricted-traces-freedreno.yml
index b5ac491..9b7913e 100644 (file)
--- a/src/freedreno/ci/restricted-traces-freedreno.yml
+++ b/src/freedreno/ci/restricted-traces-freedreno.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-private/"
+  download-url: "http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-private/"
 
 traces:
   stellaris/Stellaris-trim--f722-v20201203-v2.trace:

diff --git a/src/freedreno/ci/traces-freedreno.yml b/src/freedreno/ci/traces-freedreno.yml
index 8b3a430..44d117f 100644 (file)
--- a/src/freedreno/ci/traces-freedreno.yml
+++ b/src/freedreno/ci/traces-freedreno.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/
+  download-url: http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/
 
 text: |-
   NOTE for faster runtime, try to sort tests so the longer running traces

diff --git a/src/gallium/drivers/crocus/ci/traces-crocus.yml b/src/gallium/drivers/crocus/ci/traces-crocus.yml
index 09af0a2..d777e01 100644 (file)
--- a/src/gallium/drivers/crocus/ci/traces-crocus.yml
+++ b/src/gallium/drivers/crocus/ci/traces-crocus.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   0ad/0ad-v2.trace:

diff --git a/src/gallium/drivers/i915/ci/traces-i915.yml b/src/gallium/drivers/i915/ci/traces-i915.yml
index 26a5aaa..f163090 100644 (file)
--- a/src/gallium/drivers/i915/ci/traces-i915.yml
+++ b/src/gallium/drivers/i915/ci/traces-i915.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   glxgears/glxgears-2-v2.trace:

diff --git a/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml b/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml
index 70e621e..32fe5e4 100644 (file)
--- a/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml
+++ b/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   0ad/0ad-v2.trace:

diff --git a/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml b/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml
index 05aaf2c..5e9a21c 100644 (file)
--- a/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml
+++ b/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace:

diff --git a/src/gallium/drivers/virgl/ci/traces-virgl.yml b/src/gallium/drivers/virgl/ci/traces-virgl.yml
index 2be64e8..f480553 100644 (file)
--- a/src/gallium/drivers/virgl/ci/traces-virgl.yml
+++ b/src/gallium/drivers/virgl/ci/traces-virgl.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace:

diff --git a/src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml b/src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml
index 3421845..e9e3546 100644 (file)
--- a/src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml
+++ b/src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   unigine/sanctuary-d3d9.trace-dxgi:

diff --git a/src/intel/ci/traces-iris.yml b/src/intel/ci/traces-iris.yml
index 0880d0d..891f085 100644 (file)
--- a/src/intel/ci/traces-iris.yml
+++ b/src/intel/ci/traces-iris.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   behdad-glyphy/glyphy-v2.trace:

diff --git a/src/panfrost/ci/traces-panfrost.yml b/src/panfrost/ci/traces-panfrost.yml
index 968135e..56adcf4 100644 (file)
--- a/src/panfrost/ci/traces-panfrost.yml
+++ b/src/panfrost/ci/traces-panfrost.yml
@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 traces-db:
-  download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/"
+  download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/"
 
 traces:
   behdad-glyphy/glyphy-v2.trace: