From: Marek Pikuła Date: Tue, 16 Jan 2024 16:21:28 +0000 (+0100) Subject: Add mk_delta GitHub action and test workflow X-Git-Tag: accepted/tizen/unified/20240419.110853~8 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=df9ac6d8b32a5cfc339909085ca96816f470d64d;p=platform%2Fcore%2Fsystem%2Fupgrade-tools.git Add mk_delta GitHub action and test workflow Change-Id: I0266a6442be27e3c444c20cfa4c7644d85750777 Signed-off-by: Marek Pikuła --- diff --git a/.github/workflows/test-image-build.yml b/.github/workflows/test-image-build.yml new file mode 100644 index 0000000..4012c90 --- /dev/null +++ b/.github/workflows/test-image-build.yml @@ -0,0 +1,136 @@ +name: Test build delta images + +on: + push: + branches: [tizen] + pull_request: + workflow_dispatch: + schedule: + - cron: "0 * * * *" + +env: + BART_REPO: tizen-system-upgrade-tools-generic-local + BART_OUTPUT_DIR: tests + + # Overall default quota is 100G per BART repo. Here, we do separate cleanup + # for 7-8 and 8-9 artifacts, preserving 4G margin. + BART_TARGET_QUOTA: 48 + +jobs: + prepare: + runs-on: + - code-linux + - code-default + outputs: + docker_tag: ${{ steps.environment.outputs.docker_tag }} + + steps: + - name: Get Docker tag for this branch + id: environment + run: | + GITHUB_REF=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME} + echo "docker_tag=$([ "${GITHUB_REF}" == "tizen" ] && echo latest || echo ${GITHUB_REF})" >> "$GITHUB_OUTPUT" + + - name: Cleanup Tizen 7-8 artifacts + uses: srpol-ar-glasses/actions/bart-size-cleanup@main + with: + bart_id: ${{ secrets.BART_ID }} + bart_key: ${{ secrets.BART_KEY }} + search_pattern: "${{ env.BART_REPO }}/${{ env.BART_OUTPUT_DIR }}/Tizen-7.0_Tizen-8.0_*" + target_quota: ${{ env.BART_TARGET_QUOTA }} + - name: Cleanup Tizen 8-9 artifacts + uses: srpol-ar-glasses/actions/bart-size-cleanup@main + with: + bart_id: ${{ secrets.BART_ID }} + bart_key: ${{ secrets.BART_KEY }} + search_pattern: "${{ env.BART_REPO }}/${{ env.BART_OUTPUT_DIR }}/Tizen-8.0_Tizen_*" + target_quota: ${{ env.BART_TARGET_QUOTA }} + + build-delta: + runs-on: + - code-linux + - code-large + needs: prepare + strategy: + matrix: + old: + - Tizen-7.0 + - Tizen-8.0 + new: + - Tizen-8.0 + - Tizen + variant: + - headed + - headless + arch: + - armv7hl + - armv7l + - aarch64 + target: + - rpi4 + include_boot: + - true + - false + exclude: + - old: Tizen-7.0 + new: Tizen + - old: Tizen-8.0 + new: Tizen-8.0 + - variant: headless + arch: armv7hl + + steps: + - name: Checkout sources + uses: code-actions/checkout@v4 + + - name: Prepare environment + run: | + IMAGE_TYPE=${{ matrix.include_boot && 'all' || 'platform' }} + echo "IMAGE_TYPE=${IMAGE_TYPE}" >> "$GITHUB_ENV" + + echo "RUN_NAME=${{ matrix.old }}_${{ matrix.new }}_${{ matrix.variant }}_${{ matrix.target }}_${IMAGE_TYPE}_${{ matrix.arch }}" >> "$GITHUB_ENV" + + BOOT_ARCH=$([ ${{ matrix.arch }} == "aarch64" ] && echo arm64 || echo ${{ matrix.arch }} ) + echo "BOOT_OLD=https://download.tizen.org/snapshots/TIZEN/${{ matrix.old }}/${{ matrix.old }}-Unified/latest/images/standard/tizen-boot-${BOOT_ARCH}-${{ matrix.target }}/" >> "$GITHUB_ENV" + echo "BOOT_NEW=https://download.tizen.org/snapshots/TIZEN/${{ matrix.new }}/${{ matrix.new }}-Unified/latest/images/standard/tizen-boot-${BOOT_ARCH}-${{ matrix.target }}/" >> "$GITHUB_ENV" + + echo "MAIN_OLD=https://download.tizen.org/snapshots/TIZEN/${{ matrix.old }}/${{ matrix.old }}-Unified/latest/images/standard/tizen-${{ matrix.variant }}-${{ matrix.arch }}/" >> "$GITHUB_ENV" + echo "MAIN_NEW=https://download.tizen.org/snapshots/TIZEN/${{ matrix.new }}/${{ matrix.new }}-Unified/latest/images/standard/tizen-${{ matrix.variant }}-${{ matrix.arch }}/" >> "$GITHUB_ENV" + + - name: Build delta images + uses: ./mk_delta # For external action execution, use `tizen/upgrade-tools/mk_delta@tizen`. + with: + docker_image_tag: ${{ needs.prepare.outputs.docker_tag }} + bart_id: ${{ secrets.BART_ID }} + bart_key: ${{ secrets.BART_KEY }} + github_token: ${{ secrets.GITHUB_TOKEN }} + old_images: | + ${{ matrix.include_boot && env.BOOT_OLD || '' }} + ${{ env.MAIN_OLD }} + new_images: | + ${{ matrix.include_boot && env.BOOT_NEW || '' }} + ${{ env.MAIN_NEW }} + image_type: ${{ env.IMAGE_TYPE }} + run_name: ${{ env.RUN_NAME }} + arch: ${{ matrix.arch }} + output_repository: ${{ env.BART_REPO }} + output_directory: ${{ env.BART_OUTPUT_DIR }} + + summarize: + runs-on: + - code-linux + - code-large + needs: build-delta + if: ${{ always() }} + + steps: + - name: Checkout sources + uses: code-actions/checkout@v4 + + - name: Summarize workflow + uses: ./mk_delta/summarize # For external action execution, use `tizen/upgrade-tools/mk_delta/summarize@tizen`. + with: + bart_id: ${{ secrets.BART_ID }} + bart_key: ${{ secrets.BART_KEY }} + output_repository: ${{ env.BART_REPO }} + output_directory: ${{ env.BART_OUTPUT_DIR }} diff --git a/README b/README index 9c25ad1..e37fe5d 100644 --- a/README +++ b/README @@ -92,3 +92,191 @@ $ docker run --privileged=true \ ``` Generated delta images will be stored in `images/result`. + + +## GitHub Actions + +### `mk_delta` action + +`mk_delta` action creates delta OS Upgrade image for two sets of source images +(old and new), and stores the result on BART. + +Before executing the delta script, the action checks if the delta needs to be +built – if input image checksums match with the previous build the delta image +is not built, thus reducing run time of the action (about 15 seconds for cache +check). This in turn allows to run the action frequently to check if new input +images are available. + +Example use case of the action can be found in this repository in +[test-image-build](./.github/workflows/test-image-build.yml) workflow. For +external repositories this action should be referenced as +`srpol-syskern/upgrade-tools/mk_delta@tizen`. + +**The action does the following things:** + +1. Install JFrog tool – used for BART interaction. +2. Prepare environment – generate environment variables and pull cached input + checksums from BART. +3. Download source images: + 1. Download `SHA256SUM` files if available. + 2. Check if new images need to be generated – **if not, no additional steps + are executed**. + 3. Download input tar.gz images. +4. Log in to Docker registry. +5. Generate delta image – execute `delta-generation.sh` script in Docker + environment. +6. Upload to artifactory: + 1. Compress resulting tar archive to `.tar.gz`. + 2. Upload delta archive to BART. + 3. Upload input file SHA256 checksums. +7. Add GitHub PR comment with links to generated delta image. + +The tool uploads two copies of the archive to `${{ inputs.output_repository +}}`/`${{ inputs.output_directory }}` directory. One has a deterministic name, +which can be inferred from the upgraded system filesystem, and the second has +some additional information and is uploaded to `archive` subdirectory: + +- deterministic name: `${device_type}--${model_name}--${image_type}--${arch}--${tz_build_date}@${{ inputs.docker_image_tag }}.tar.gz`, +- archival name: `archive/${device_type}--${model_name}--${image_type}--${arch}--${build_string.txt}--${delta_timestamp}.tar.gz`. + +**Variables:** + +- `device_type`, `model_name`, `tz_build_date` – taken from `update-info.ini` + delta file. +- `image_type`: + - `all` contains delta of all sub-images, + - `platform` doesn't contain boot delta. +- `build_string.txt` – contents of `build_string.txt` delta file. +- `delta_timestamp` – delta build timestamp in format `%Y%m%d_%H%M%S`. +- `arch` – `${{ inputs.arch }}` for now, until `tz_build_arch` from + `update-info.ini` is fixed. + +In addition to the main `.tar.gz` archive, a `.sha256sums` file containing input +file checksums is uploaded along the delta files, and in `action_checksums/${{ +inputs.run_name }}` directory. + +**Example file tree on BART after a successful run:** + +- `${{ inputs.output_repository }}`/`${{ inputs.output_directory }}/` + - `IoT_Headless--rpi4--all--aarch64--20240124_061003@latest.tar.gz` – + resulting delta archive after latest run on the main branch (`tizen`). + - `IoT_Headless--rpi4--all--aarch64--20240124_061003@latest.tar.gz.sha256sums` + – SHA256 checksums of input files for the run above. + - `archive/` + - `IoT_Headless--rpi4--all--aarch64--20240124_061003@20240123_164559--20240124_121048.tar.gz` + – resulting delta archive after run executed on 2024-01-24 at 12:10:48 + UTC. + - `IoT_Headless--rpi4--all--aarch64--20240124_061003@20240123_164559--20240124_121048.sha256sums` + – SHA256 checksums of input files for the run above. + - `action_checksums/${{ inputs.run_name }}/` + - `latest.sha256sums` – copy of + `IoT_Headless--rpi4--all--aarch64--20240124_061003@latest.tar.gz.sha256sums` + (used for cache check in cache check in the second step). + +#### Action inputs + +For default values of some inputs please refer to [the action +file](./mk_delta/action.yml). + +**Docker image configuration:** + +For regular run it is not required to change these, but it can be useful for +checking if new images work properly. + +- `docker_image_registry` – Docker image registry used for the delta build. +- `docker_image_name` – Docker image used for the delta build. +- `docker_image_tag` – Docker image tag used for the delta build. + +**Secrets (BART and GitHub):** + +It is advised to use GitHub secrets to store the BART credentials. Example on +how to use them, refer to the +[test-image-build](./.github/workflows/test-image-build.yml) workflow. It is the +safest to create service account especially for CI purpose and store the secrets +in GitHub repository (not organization-wide). You can access GitHub secrets in +project *Settings → Secrets and variables*. + +- `bart_id` – user ID used for BART authentication. +- `bart_key` – BART Identity Token which can be generated + [here](https://bart.sec.samsung.net/ui/user_profile). +- `github_token` – GitHub Token used for API access to fetch the job ID. Can be + passed from `${{ secrets.GITHUB_TOKEN }}`. + +**Delta tool configuration:** + +Input image URL can be either directory or `tar.gz` file. Mind that previous +build detection doesn't currently work for `tar.gz` variant – only for +directories with `SHA256SUMS` file (as on https://download.tizen.org). + +For multiple images, separate them with newline, e.g.: + +```yaml +- name: Build delta images + uses: srpol-syskern/upgrade-tools/mk_delta@tizen + with: + old_images: | + https://download.tizen.org/snapshots/TIZEN/Tizen/Tizen-Unified/latest/images/standard/tizen-boot-arm64-rpi4/ + https://download.tizen.org/snapshots/TIZEN/Tizen/Tizen-Unified/latest/images/standard/tizen-headless-aarch64/ +``` + +- `old_images` – Old image URLs. +- `new_images` – New image URLs. +- `arch` – Build architecture. Used for artifact naming. +- `target` – `delta-generation.sh` script target. +- `image_type` – Name of the image type for given input images. + +**Output configuration:** + +- `run_name` – Delta generation run name. Used in artifact path and messages. +- `output_repository` – Output BART repository. +- `output_directory` – Output directory in BART repository. +- `output_gzip_flags` – Flags given to gzip command used to compress the final + delta tar. + +#### Action outputs + +- `artifact_path_latest` – Path to result delta image artifact in BART + repository with `latest` tag. +- `artifact_path_archvie` – Path to result delta image artifact in BART + repository with deterministic path. + + +### `mk_delta/summarize` action + +`mk_delta/summarize` action processes job state and presents it in a concise +GitHub PR message. + +Example use case of the action can be found in this repository in +[test-image-build](./.github/workflows/test-image-build.yml) workflow. For +external repositories this action should be referenced as +`srpol-syskern/upgrade-tools/mk_delta/summarize@tizen`. + +#### Action inputs + +**BART secrets:** Same as in `mk_delta` action. + +**Output configuration:** `output_repository` and `output_directory` same as in `mk_delta` action. + +#### Example output + +> Here is the list of all jobs in run [8394854](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854): +> - [Tizen-7.0_Tizen-8.0_headed_rpi4_all_aarch64](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478842): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen7_Unified/archive/IoT_Headed--rpi4--all--aarch64--20240205_160201@20240205_160245--20240205_171010.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen7_Unified/archive/IoT_Headed--rpi4--all--aarch64--20240205_160201@20240205_160245--20240205_171010.tar.gz)). +> - [Tizen-7.0_Tizen-8.0_headed_rpi4_all_armv7hl](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478838): ✔️ Inputs are not different from the last run. Not generating deltas. +> - [Tizen-7.0_Tizen-8.0_headed_rpi4_all_armv7l](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478840): ✔️ Inputs are not different from the last run. Not generating deltas. +> - [Tizen-7.0_Tizen-8.0_headed_rpi4_platform_aarch64](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478845): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen7_Unified/archive/IoT_Headed----platform--aarch64--20240205_160201@20240205_160245--20240205_171104.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen7_Unified/archive/IoT_Headed----platform--aarch64--20240205_160201@20240205_160245--20240205_171104.tar.gz)). +> - [Tizen-7.0_Tizen-8.0_headed_rpi4_platform_armv7hl](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478839): ✔️ Inputs are not different from the last run. Not generating deltas. +> - [Tizen-7.0_Tizen-8.0_headed_rpi4_platform_armv7l](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478841): ✔️ Inputs are not different from the last run. Not generating deltas. +> - [Tizen-7.0_Tizen-8.0_headless_rpi4_all_aarch64](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478848): ✔️ Inputs are not different from the last run. Not generating deltas. +> - [Tizen-7.0_Tizen-8.0_headless_rpi4_all_armv7l](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478846): ✔️ Inputs are not different from the last run. Not generating deltas. +> - [Tizen-7.0_Tizen-8.0_headless_rpi4_platform_aarch64](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478849): ✔️ Inputs are not different from the last run. Not generating deltas. +> - [Tizen-7.0_Tizen-8.0_headless_rpi4_platform_armv7l](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478847): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen7_Unified/archive/IoT_Headless----platform--armv7l--20240205_155803@20240205_155843--20240205_165500.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen7_Unified/archive/IoT_Headless----platform--armv7l--20240205_155803@20240205_155843--20240205_165500.tar.gz)). +> - [Tizen-8.0_Tizen_headed_rpi4_all_aarch64](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478855): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed--rpi4--all--aarch64--20240205_160245@20240205_161650--20240205_170156.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed--rpi4--all--aarch64--20240205_160245@20240205_161650--20240205_170156.tar.gz)). +> - [Tizen-8.0_Tizen_headed_rpi4_all_armv7hl](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478850): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed--rpi4--all--armv7hl--20240205_160219@20240205_161637--20240205_165902.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed--rpi4--all--armv7hl--20240205_160219@20240205_161637--20240205_165902.tar.gz)). +> - [Tizen-8.0_Tizen_headed_rpi4_all_armv7l](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478853): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed--rpi4--all--armv7l--20240205_160233@20240205_161647--20240205_165952.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed--rpi4--all--armv7l--20240205_160233@20240205_161647--20240205_165952.tar.gz)). +> - [Tizen-8.0_Tizen_headed_rpi4_platform_aarch64](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478856): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed----platform--aarch64--20240205_160245@20240205_161650--20240205_170306.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed----platform--aarch64--20240205_160245@20240205_161650--20240205_170306.tar.gz)). +> - [Tizen-8.0_Tizen_headed_rpi4_platform_armv7hl](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478851): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed----platform--armv7hl--20240205_160219@20240205_161637--20240205_170102.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed----platform--armv7hl--20240205_160219@20240205_161637--20240205_170102.tar.gz)). +> - [Tizen-8.0_Tizen_headed_rpi4_platform_armv7l](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478854): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed----platform--armv7l--20240205_160233@20240205_161647--20240205_165955.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headed----platform--armv7l--20240205_160233@20240205_161647--20240205_165955.tar.gz)). +> - [Tizen-8.0_Tizen_headless_rpi4_all_aarch64](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478859): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headless--rpi4--all--aarch64--20240205_155854@20240205_161251--20240205_165319.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headless--rpi4--all--aarch64--20240205_155854@20240205_161251--20240205_165319.tar.gz)). +> - [Tizen-8.0_Tizen_headless_rpi4_all_armv7l](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478857): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headless--rpi4--all--armv7l--20240205_155843@20240205_161300--20240205_165430.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headless--rpi4--all--armv7l--20240205_155843@20240205_161300--20240205_165430.tar.gz)). +> - [Tizen-8.0_Tizen_headless_rpi4_platform_aarch64](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478860): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headless----platform--aarch64--20240205_155854@20240205_161251--20240205_165319.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headless----platform--aarch64--20240205_155854@20240205_161251--20240205_165319.tar.gz)). +> - [Tizen-8.0_Tizen_headless_rpi4_platform_armv7l](https://github.sec.samsung.net/srpol-syskern/upgrade-tools/actions/runs/8394854/job/21478858): ✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headless----platform--armv7l--20240205_155843@20240205_161300--20240205_165224.tar.gz) ([direct](https://bart.sec.samsung.net/artifactory/tizen-system-upgrade-tools-generic-local/tests/Tizen8.0_Unified/archive/IoT_Headless----platform--armv7l--20240205_155843@20240205_161300--20240205_165224.tar.gz)). diff --git a/mk_delta/action.yml b/mk_delta/action.yml new file mode 100644 index 0000000..b57d016 --- /dev/null +++ b/mk_delta/action.yml @@ -0,0 +1,263 @@ +name: Make OS Upgrade delta +description: Make OS Upgrade delta image. +author: "Marek Pikuła " + +inputs: + # Docker image configuration. + docker_image_registry: + description: Docker image registry used for the delta build. + default: tizen-system-upgrade-tools-docker-local.bart.sec.samsung.net + docker_image_name: + description: Docker image used for the delta build. + default: upgrade-tools + docker_image_tag: + description: Docker image tag used for the delta build. + default: latest + + # Secrets (BART and GitHub). + bart_id: + description: BART user ID. + required: true + bart_key: + description: BART Identity Token. + required: true + github_token: + description: GitHub Token. + required: true + + # Delta tool configuration. + old_images: + description: Old image URLs. For multiple images, separate them with newline. + required: true + new_images: + description: New image URLs. For multiple images, separate them with newline. + required: true + arch: + description: Build architecture. Used for artifact naming. + required: true + target: + description: delta-generation.sh script target. + default: rpi4 + image_type: + description: Name of the image type for given input images. + default: all + + # Output + run_name: + description: Delta generation run name. Used in artifact path and messages. + default: "tizen_osu" + output_repository: + description: Output BART repository. + required: true + output_directory: + description: Output directory in BART repository. + default: "" + output_gzip_flags: + description: Flags given to gzip command used to compress the final delta tar. + default: "" + +outputs: + artifact_path: + description: Path to result delta image artifact in BART repository. + value: ${{ steps.upload_artifacts.outputs.artifact_path }} + +runs: + using: composite + steps: + - name: Install JFrog tool + uses: code-actions/jfrog-setup-jfrog-cli@v3 + env: + JF_URL: https://bart.sec.samsung.net + JF_USER: ${{ inputs.bart_id }} + JF_PASSWORD: ${{ inputs.bart_key }} + + - name: Prepare environment + shell: bash + run: | + echo Preparing environment... + + # Environment + OUTPUT_REPO_DIR=${{ inputs.output_repository }}/${{ inputs.output_directory }} + ACTION_STATE=${OUTPUT_REPO_DIR}/action_state + + INPUT_SUMS=input_sums.sha256sums + INPUT_SUMS_BART=${{ inputs.docker_image_tag }}.sha256sums + INPUT_SUMS_BART_DIR=${ACTION_STATE}/checksums/${{ inputs.run_name }} + + echo "OUTPUT_REPO_DIR=${OUTPUT_REPO_DIR}" >> "$GITHUB_ENV" + echo "INPUT_SUMS=${INPUT_SUMS}" >> "$GITHUB_ENV" + echo "INPUT_SUMS_BART=${INPUT_SUMS_BART}" >> "$GITHUB_ENV" + echo "INPUT_SUMS_BART_DIR=${INPUT_SUMS_BART_DIR}" >> "$GITHUB_ENV" + + # Pull input checksums from BART + jf rt dl --flat "${INPUT_SUMS_BART_DIR}/${INPUT_SUMS_BART}" "${INPUT_SUMS_BART}" + if [ ! -f "${INPUT_SUMS_BART}" ]; then + echo "::notice::Input checksum state file not found on BART. Creating a new one." + touch "${INPUT_SUMS_BART}" + fi + + # Action execution state + get_job_id() { + curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ inputs.github_token }}"\ + -H "X-GitHub-Api-Version: 2022-11-28" \ + ${{ github.api_url }}/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs \ + | jq -r '.jobs | map(select(.runner_name | contains("${{ runner.name }}"))) | .[0].id' + } + while ! get_job_id; do sleep 1; done # Needed as sometimes race condition occurrs. + JOB_ID=$(get_job_id) + + ACTION_EXECUTION_STATE_LOCAL=$(pwd)/state + ACTION_EXECUTION_STATE_REMOTE=${ACTION_STATE}/execution/${{ github.run_id }}/${{ inputs.run_name }}/${JOB_ID} + PUSH_ACTION_EXECUTION_STATE="jf rt u --flat=true --fail-no-op=true ${ACTION_EXECUTION_STATE_LOCAL} ${ACTION_EXECUTION_STATE_REMOTE}" + + echo "ACTION_EXECUTION_STATE_LOCAL=${ACTION_EXECUTION_STATE_LOCAL}" >> "$GITHUB_ENV" + echo "PUSH_ACTION_EXECUTION_STATE=${PUSH_ACTION_EXECUTION_STATE}" >> "$GITHUB_ENV" + + # Push default action state + echo "❌ Job failed or cancelled." > "${ACTION_EXECUTION_STATE_LOCAL}" + ${PUSH_ACTION_EXECUTION_STATE} + + - name: Download source images + shell: bash + run: | + echo Downloading source images... + + WGET_DEFAULT="wget --no-parent -r --no-verbose --no-directories" + WGET_BART="${WGET_DEFAULT} --user ${{ inputs.bart_id }} --password ${{ inputs.bart_key }}" + WGET_SHA256SUMS="--accept SHA256SUMS" + WGET_TARGZ="--accept .tar.gz" + + INPUT_SUMS=$(pwd)/${{ env.INPUT_SUMS }} + touch ${INPUT_SUMS} + + download_images() { + pushd $1 + for item in $2; do + # Ignore empty lines + if [ "x$item" == "x" ]; then + continue + fi + + echo "Downloading $item..." + DL="$([[ $item == "https://bart.sec.samsung.net/artifactory/"* ]] && echo ${WGET_BART} $3 $item || echo ${WGET_DEFAULT} $3 $item)" + if ! ${DL}; then + MSG="Failed to download $item." + echo "::error::${MSG}" + echo "❌ ${MSG}" > "${ACTION_EXECUTION_STATE_LOCAL}" + ${PUSH_ACTION_EXECUTION_STATE} + exit 1 + fi + + # Concat input file checksums + if [ -f SHA256SUMS ]; then + cat SHA256SUMS >> ${INPUT_SUMS} + rm SHA256SUMS + fi + done + popd + } + + # Download SHA256SUMS + mkdir -p images/old images/new images/result + download_images images/old "${{ inputs.old_images }}" "${WGET_SHA256SUMS}" + download_images images/new "${{ inputs.new_images }}" "${WGET_SHA256SUMS}" + + echo "Input file difference since the last run:" + if [ -s ${{ env.INPUT_SUMS }} ] && diff ${{ env.INPUT_SUMS_BART }} ${{ env.INPUT_SUMS }}; then + MSG="Inputs are not different from the last run. Not generating deltas." + echo "::notice::${MSG}" + echo "✔️ ${MSG}" > "${ACTION_EXECUTION_STATE_LOCAL}" + ${PUSH_ACTION_EXECUTION_STATE} + exit + fi + + echo "::notice::Inputs are different from the last run. Generating deltas." + echo SUM_DIFFERS=1 >> "$GITHUB_ENV" + + # Download tar.gz images + download_images images/old "${{ inputs.old_images }}" "${WGET_TARGZ}" + download_images images/new "${{ inputs.new_images }}" "${WGET_TARGZ}" + + echo "Downloaded files:" + if ! find images/old/* images/new/*; then + MSG="No source images exist." + echo "::error::${MSG}" + echo "❌ ${MSG}" > "${ACTION_EXECUTION_STATE_LOCAL}" + ${PUSH_ACTION_EXECUTION_STATE} + exit 1 + fi + + - name: Log in to Docker registry + uses: code-actions/docker-login-action@v3 + with: + registry: ${{ inputs.docker_image_registry }} + username: ${{ inputs.bart_id }} + password: ${{ inputs.bart_key }} + + - if: ${{ env.SUM_DIFFERS }} + name: Generate delta image + shell: bash + run: | + echo Generating delta images... + + docker run --privileged=true \ + -v $(pwd)/images/old:/tota-upg/mk_delta/${{ inputs.target }}/data/old_tar:ro \ + -v $(pwd)/images/new:/tota-upg/mk_delta/${{ inputs.target }}/data/new_tar:ro \ + -v $(pwd)/images/result:/tota-upg/mk_delta/${{ inputs.target }}/result \ + ${{ inputs.docker_image_registry }}/${{ inputs.docker_image_name }}:${{ inputs.docker_image_tag }} \ + /tota-upg/scripts/delta-generation.sh /tota-upg/ ${{ inputs.target }} + + echo "Generated files:" + if ! find images/result/*; then + MSG="No files generated." + echo "::error::${MSG}" + echo "❌ ${MSG}" > "${ACTION_EXECUTION_STATE_LOCAL}" + ${PUSH_ACTION_EXECUTION_STATE} + exit 1 + fi + + - if: ${{ env.SUM_DIFFERS }} + name: Upload to artifactory + shell: bash + id: upload_artifacts + run: | + echo Uploading artifacts... + + # Get variables from delta update info + set -x + source images/result/*/FW_DELTA/DELTA/update-info.ini + set +x + tz_build_release_name=${tz_build_release_name//\//_} + BUILD_STRING=$(cat images/result/*/FW_DELTA/DELTA/build_string.txt | sed "s/_/\./g") + + # Build file name + DELTA_BUILD_TIMESTAMP=$(TZ=GMT date +"%Y%m%d.%H%M%S") + # ARTIFACT_FILE_NAME_BASE=${device_type}--${model_name}--${{ inputs.image_type }}--${tz_build_arch} # TODO: Use it once `tz_build_arch` correctly distinguishes between armv7l and armv7hl. + ARTIFACT_FILE_NAME_BASE=${device_type}--${model_name}--${{ inputs.image_type }}--${{ inputs.arch }} + TAG_SUFFIX="$([[ "${{ inputs.docker_image_tag }}" == "latest" ]] && echo "" || echo "--${{ inputs.docker_image_tag }}")" + ARTIFACT_FILE_NAME_LATEST=${OUTPUT_REPO_DIR}/${tz_build_release_name}/${ARTIFACT_FILE_NAME_BASE}--${BUILD_STRING}${TAG_SUFFIX}.tar.gz + ARTIFACT_FILE_NAME_ARCHIVE=${OUTPUT_REPO_DIR}/${tz_build_release_name}/archive/${ARTIFACT_FILE_NAME_BASE}--${BUILD_STRING}--${DELTA_BUILD_TIMESTAMP}.tar.gz + + JF_UPLOAD="jf rt u --flat=true --fail-no-op=true" + + set -x + pushd images/result + gzip ${{ inputs.output_gzip_flags }} */FW_DELTA/delta.tar + + ${JF_UPLOAD} */FW_DELTA/delta.tar.gz "${ARTIFACT_FILE_NAME_LATEST}" + ${JF_UPLOAD} */FW_DELTA/delta.tar.gz "${ARTIFACT_FILE_NAME_ARCHIVE}" + popd + + ${JF_UPLOAD} ${{ env.INPUT_SUMS }} "${ARTIFACT_FILE_NAME_LATEST}.sha256sums" + ${JF_UPLOAD} ${{ env.INPUT_SUMS }} "${ARTIFACT_FILE_NAME_ARCHIVE}.sha256sums" + ${JF_UPLOAD} ${{ env.INPUT_SUMS }} "${INPUT_SUMS_BART_DIR}/${INPUT_SUMS_BART}" + set +x + + echo artifact_path_latest=${ARTIFACT_FILE_NAME_LATEST} >> "$GITHUB_OUTPUT" + echo artifact_path_archive=${ARTIFACT_FILE_NAME_ARCHIVE} >> "$GITHUB_OUTPUT" + echo "::notice title="Artifact path on BART for run ${{ inputs.run_name }}"::${ARTIFACT_FILE_NAME_LATEST}\n${ARTIFACT_FILE_NAME_ARCHIVE}" + + echo "✔️ You can find the delta image [here](https://bart.sec.samsung.net/ui/repos/tree/General/${ARTIFACT_FILE_NAME_ARCHIVE}) ([direct](https://bart.sec.samsung.net/artifactory/${ARTIFACT_FILE_NAME_ARCHIVE}))." > "${ACTION_EXECUTION_STATE_LOCAL}" + ${PUSH_ACTION_EXECUTION_STATE} diff --git a/mk_delta/summarize/action.yml b/mk_delta/summarize/action.yml new file mode 100644 index 0000000..0d9104f --- /dev/null +++ b/mk_delta/summarize/action.yml @@ -0,0 +1,72 @@ +name: Summarize OSU delta build +description: Generate a summary for OS Upgrade delta image matrix build. +author: "Marek Pikuła " + +inputs: + # BART credentials. + bart_id: + description: BART user ID. + required: true + bart_key: + description: BART Identity Token. + required: true + + # Output + output_repository: + description: Output BART repository. + required: true + output_directory: + description: Output directory in BART repository. + default: "" + +runs: + using: composite + steps: + - name: Install JFrog tool + uses: code-actions/jfrog-setup-jfrog-cli@v3 + env: + JF_URL: https://bart.sec.samsung.net + JF_USER: ${{ inputs.bart_id }} + JF_PASSWORD: ${{ inputs.bart_key }} + + - name: Fetch state + shell: bash + run: | + echo Fetch run state... + + OUTPUT_REPO_DIR=${{ inputs.output_repository }}/${{ inputs.output_directory }} + ACTION_STATE=${OUTPUT_REPO_DIR}/action_state + ACTION_EXECUTION_STATE_REMOTE=${ACTION_STATE}/execution/${{ github.run_id }}/ + RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + jf rt dl --fail-no-op --threads=16 "${ACTION_EXECUTION_STATE_REMOTE}" ./ + + echo Build message... + MSG="Here is the list of all jobs in run [${{ github.run_id }}](${RUN_URL}):" + for f in $(find ${{ inputs.output_directory }} -type f | sort); do + JOB_ID=$(basename ${f}) + JOB_RUN_NAME=$(basename $(dirname ${f})) + MSG+="\n- [${JOB_RUN_NAME}](${RUN_URL}/job/${JOB_ID}): $(cat ${f})" + done + + echo "MSG=${MSG}" >> "$GITHUB_ENV" + + echo Clean state... + jf rt del --fail-no-op "${ACTION_EXECUTION_STATE_REMOTE}" + + - if: github.event_name == 'pull_request' + name: Push summary as a PR comment + uses: code-actions/github-script@v6 + with: + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: "${{ env.MSG }}" + }) + + - if: github.event_name != 'pull_request' + name: Print summary + shell: bash + run: echo -e "::notice::${MSG}"