("5.4", [ # All this subtree rebases to master and then build
("3.6", [
("important", [X(True)]),
+ ("parallel_tbb", [X(True)]),
+ ("parallel_native", [X(True)]),
("pure_torch", [X(True)]),
]),
]),
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
resource_class: large
- pytorch_linux_build:
+ name: pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_build
+ requires:
+ - "docker-pytorch-linux-xenial-py3.6-gcc5.4"
+ filters:
+ branches:
+ only:
+ - master
+ - /ci-all\/.*/
+ - /release\/.*/
+ build_environment: "pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4-build"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ - pytorch_linux_test:
+ name: pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_test
+ requires:
+ - pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_build
+ filters:
+ branches:
+ only:
+ - master
+ - /ci-all\/.*/
+ - /release\/.*/
+ build_environment: "pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4-test"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ resource_class: large
+ - pytorch_linux_test:
+ name: pytorch_linux_pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_distributed_test
+ requires:
+ - pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_build
+ build_environment: "pytorch-linux-pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_distributed-test"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ resource_class: large
+ - pytorch_linux_build:
+ name: pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_build
+ requires:
+ - "docker-pytorch-linux-xenial-py3.6-gcc5.4"
+ filters:
+ branches:
+ only:
+ - master
+ - /ci-all\/.*/
+ - /release\/.*/
+ build_environment: "pytorch-parallelnative-linux-xenial-py3.6-gcc5.4-build"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ - pytorch_linux_test:
+ name: pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_test
+ requires:
+ - pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_build
+ filters:
+ branches:
+ only:
+ - master
+ - /ci-all\/.*/
+ - /release\/.*/
+ build_environment: "pytorch-parallelnative-linux-xenial-py3.6-gcc5.4-test"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ resource_class: large
+ - pytorch_linux_test:
+ name: pytorch_linux_pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_distributed_test
+ requires:
+ - pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_build
+ build_environment: "pytorch-linux-pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_distributed-test"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ resource_class: large
+ - pytorch_linux_build:
name: pytorch_pure_torch_linux_xenial_py3_6_gcc5_4_build
requires:
- "docker-pytorch-linux-xenial-py3.6-gcc5.4"
name: "docker-pytorch-linux-xenial-py3.6-gcc7"
image_name: "pytorch-linux-xenial-py3.6-gcc7"
- pytorch_linux_build:
+ name: pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_build
+ requires:
+ - "docker-pytorch-linux-xenial-py3.6-gcc5.4"
+ build_environment: "pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4-build"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ - pytorch_linux_test:
+ name: pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_test
+ requires:
+ - pytorch_paralleltbb_linux_xenial_py3_6_gcc5_4_build
+ build_environment: "pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4-test"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ resource_class: large
+ - pytorch_linux_build:
+ name: pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_build
+ requires:
+ - "docker-pytorch-linux-xenial-py3.6-gcc5.4"
+ build_environment: "pytorch-parallelnative-linux-xenial-py3.6-gcc5.4-build"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ - pytorch_linux_test:
+ name: pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_test
+ requires:
+ - pytorch_parallelnative_linux_xenial_py3_6_gcc5_4_build
+ build_environment: "pytorch-parallelnative-linux-xenial-py3.6-gcc5.4-test"
+ docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
+ resource_class: large
+ - pytorch_linux_build:
name: pytorch_pure_torch_linux_xenial_py3_6_gcc5_4_build
requires:
- "docker-pytorch-linux-xenial-py3.6-gcc5.4"
"linux-xenial-cuda11.3-py3.6-gcc7",
"linux-xenial-py3.6-gcc5.4",
"linux-xenial-py3.6-gcc7-bazel-test",
- "parallelnative-linux-xenial-py3.6-gcc5.4",
- "paralleltbb-linux-xenial-py3.6-gcc5.4",
"periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7",
"periodic-linux-xenial-cuda11.1-py3.6-gcc7",
"periodic-win-vs2019-cuda11.1-py3",
"linux-bionic-py3.8-gcc9-coverage",
"linux-xenial-py3.6-gcc5.4",
"linux-xenial-py3.6-gcc7-bazel-test",
- "parallelnative-linux-xenial-py3.6-gcc5.4",
- "paralleltbb-linux-xenial-py3.6-gcc5.4",
"win-vs2019-cpu-py3"
],
"ciflow/cuda": [
"linux-xenial-cuda11.3-py3.6-gcc7",
"linux-xenial-py3.6-gcc5.4",
"linux-xenial-py3.6-gcc7-bazel-test",
- "parallelnative-linux-xenial-py3.6-gcc5.4",
- "paralleltbb-linux-xenial-py3.6-gcc5.4",
"periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7",
"periodic-linux-xenial-cuda11.1-py3.6-gcc7"
],
labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_LINUX, LABEL_CIFLOW_CPU}
),
),
- CIWorkflow(
- arch="linux",
- build_environment="paralleltbb-linux-xenial-py3.6-gcc5.4",
- docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
- test_runner_type=LINUX_CPU_TEST_RUNNER,
- # This is a master only job despit on_pull_request is set to True
- on_pull_request=True,
- ciflow_config=CIFlowConfig(
- enabled=True,
- trigger_action_only=True,
- labels={LABEL_CIFLOW_LINUX, LABEL_CIFLOW_CPU},
- ),
- ),
- CIWorkflow(
- arch="linux",
- build_environment="parallelnative-linux-xenial-py3.6-gcc5.4",
- docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
- test_runner_type=LINUX_CPU_TEST_RUNNER,
- # This is a master only job despit on_pull_request is set to True
- on_pull_request=True,
- ciflow_config=CIFlowConfig(
- enabled=True,
- trigger_action_only=True,
- labels={LABEL_CIFLOW_LINUX, LABEL_CIFLOW_CPU},
- ),
- ),
+ # CIWorkflow(
+ # arch="linux",
+ # build_environment="paralleltbb-linux-xenial-py3.6-gcc5.4",
+ # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
+ # test_runner_type=LINUX_CPU_TEST_RUNNER,
+ # ),
+ # CIWorkflow(
+ # arch="linux",
+ # build_environment="parallelnative-linux-xenial-py3.6-gcc5.4",
+ # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
+ # test_runner_type=LINUX_CPU_TEST_RUNNER,
+ # ),
# CIWorkflow(
# arch="linux",
# build_environment="pure_torch-linux-xenial-py3.6-gcc5.4",
+++ /dev/null
-# @generated DO NOT EDIT MANUALLY
-
-# Template is at: .github/templates/linux_ci_workflow.yml.j2
-# Generation script: .github/scripts/generate_ci_workflows.py
-name: parallelnative-linux-xenial-py3.6-gcc5.4
-
-on:
- pull_request:
- types: [unassigned]
- push:
- branches:
- - master
- - release/*
- workflow_dispatch:
-
-env:
- BUILD_ENVIRONMENT: parallelnative-linux-xenial-py3.6-gcc5.4
- DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4
- SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2
- TORCH_CUDA_ARCH_LIST: 5.2
- IN_CI: 1
- # This is used for the phase of adding wheel tests only, will be removed once completed
- IN_WHEEL_TEST: 1
- # Used for custom_opertor, jit_hooks, custom_backend, see .jenkins/pytorch/build.sh
- CUSTOM_TEST_ARTIFACT_BUILD_DIR: build/custom_test_artifacts
- ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine"
-
-concurrency:
- group: parallelnative-linux-xenial-py3.6-gcc5.4-${{ github.event.pull_request.number || github.sha }}
- cancel-in-progress: true
-
-jobs:
- ciflow_should_run:
- runs-on: ubuntu-18.04
- if: ${{ (github.event_name != 'pull_request') || (github.event.action !='unassigned') || (contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/cpu') || contains(github.event.pull_request.labels.*.name, 'ciflow/linux')) }}
- steps:
- - name: noop
- run: echo running ciflow_should_run
- calculate-docker-image:
- if: ${{ github.repository_owner == 'pytorch' }}
- runs-on: linux.2xlarge
- needs: [ciflow_should_run]
- env:
- DOCKER_BUILDKIT: 1
- timeout-minutes: 90
- outputs:
- docker_image: ${{ steps.calculate-tag.outputs.docker_image }}
- steps:
- - name: Log in to ECR
- run: |
- aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
- bash /tmp/ecr-login.sh
- rm /tmp/ecr-login.sh
- - name: Chown workspace
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Clean workspace
- run: |
- rm -rf "${GITHUB_WORKSPACE:?}/*"
- rm -f ~/.ssh/authorized_keys
- - name: Checkout PyTorch
- uses: actions/checkout@v2
- with:
- # deep clone, to allow use of git merge-base
- fetch-depth: 0
- - name: Display EC2 information
- shell: bash
- run: |
- .github/scripts/display_ec2_information.sh
- - name: Calculate docker image tag
- id: calculate-tag
- run: |
- DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)
- echo "::set-output name=docker_tag::${DOCKER_TAG}"
- echo "::set-output name=docker_image::${DOCKER_IMAGE_BASE}:${DOCKER_TAG}"
- - name: Check if image should be built
- id: check
- env:
- DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker_tag }}
- BASE_REVISION: ${{ github.event.pull_request.base.sha || github.sha }}
- run: |
- set -x
- # Check if image already exists, if it does then skip building it
- if docker manifest inspect "${DOCKER_IMAGE_BASE}:${DOCKER_TAG}"; then
- exit 0
- fi
- if [[ "$BASE_REVISION" = "$(git rev-parse HEAD)" ]]; then
- # if we're on the base branch then use the parent commit
- MERGE_BASE=$(git rev-parse HEAD~)
- else
- # otherwise we're on a PR, so use the most recent base commit
- MERGE_BASE=$(git merge-base HEAD "$BASE_REVISION")
- fi
- # Covers the case where a previous tag doesn't exist for the tree
- # this is only really applicable on trees that don't have `.circleci/docker` at its merge base, i.e. nightly
- if ! git rev-parse "$MERGE_BASE:.circleci/docker"; then
- echo "Directory '.circleci/docker' not found in commit $MERGE_BASE, you should probably rebase onto a more recent commit"
- exit 1
- fi
- PREVIOUS_DOCKER_TAG=$(git rev-parse "$MERGE_BASE:.circleci/docker")
- # If no image exists but the hash is the same as the previous hash then we should error out here
- if [[ "${PREVIOUS_DOCKER_TAG}" = "${DOCKER_TAG}" ]]; then
- echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
- echo " contact the PyTorch team to restore the original images"
- exit 1
- fi
- echo ::set-output name=rebuild::yes
- - name: Build and push docker image
- if: ${{ steps.check.outputs.rebuild }}
- env:
- DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker_tag }}
- DOCKER_SKIP_S3_UPLOAD: 1
- run: |
- export IMAGE_NAME=${DOCKER_IMAGE_BASE#308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/}
- cd .circleci/docker && ./build_docker.sh
-
- build:
- runs-on: linux.2xlarge
- needs: [calculate-docker-image, ciflow_should_run]
- env:
- DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }}
- JOB_BASE_NAME: parallelnative-linux-xenial-py3.6-gcc5.4-build
- steps:
- - name: Log in to ECR
- run: |
- aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
- bash /tmp/ecr-login.sh
- rm /tmp/ecr-login.sh
- - name: Chown workspace
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Clean workspace
- run: |
- rm -rf "${GITHUB_WORKSPACE:?}/*"
- rm -f ~/.ssh/authorized_keys
- - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)"
- uses: seemethere/add-github-ssh-key@v1
- with:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Checkout PyTorch
- uses: actions/checkout@v2
- with:
- fetch-depth: 0 # deep clone, to allow sharding to use git rev-list
- submodules: recursive
- - name: Display EC2 information
- shell: bash
- run: |
- .github/scripts/display_ec2_information.sh
- - name: Pull docker image
- run: |
- docker pull "${DOCKER_IMAGE}"
- - name: Preserve github env variables for use in docker
- run: |
- env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}"
- - name: Build PyTorch
- run: |
- docker run \
- -e BUILD_ENVIRONMENT \
- -e JOB_BASE_NAME \
- -e MAX_JOBS="$(nproc --ignore=2)" \
- -e SCCACHE_BUCKET \
- -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \
- -e SKIP_SCCACHE_INITIALIZATION=1 \
- -e TORCH_CUDA_ARCH_LIST \
- -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \
- --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
- --security-opt seccomp=unconfined \
- --cap-add=SYS_PTRACE \
- --tty \
- --user jenkins \
- -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
- -w /var/lib/jenkins/workspace \
- "${DOCKER_IMAGE}" \
- sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh'
- - name: Parse ref
- id: parse-ref
- run: .github/scripts/parse_ref.py
- - name: Display and upload binary build size statistics (Click Me)
- # temporary hack: set CIRCLE_* vars, until we update
- # tools/stats/print_test_stats.py to natively support GitHub Actions
- env:
- AWS_DEFAULT_REGION: us-east-1
- SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
- CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
- CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
- CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
- CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
- CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
- run: |
- COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
- export COMMIT_TIME
- pip3 install requests
- python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
- - name: Chown workspace
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Archive artifacts into zip
- run: |
- zip -1 -r artifacts.zip dist/ build/custom_test_artifacts build/lib build/bin .pytorch-test-times.json
- - uses: seemethere/upload-artifact-s3@9d7ceb0ab39c2c88d93ef7792b27425b27d59162
- name: Store PyTorch Build Artifacts on S3
- with:
- name: ${{ env.BUILD_ENVIRONMENT }}
- retention-days: 14
- if-no-files-found: error
- path:
- artifacts.zip
- - name: Hold runner for 2 hours or until ssh sessions have drained
- # Always hold for active ssh sessions
- if: always()
- run: .github/scripts/wait_for_ssh_to_drain.sh
- - name: Clean up docker images
- if: always()
- run: |
- # Prune all of the docker images
- docker system prune -af
-
- generate-test-matrix:
- if: ${{ github.repository_owner == 'pytorch' }}
- runs-on: ubuntu-18.04
- needs: [ciflow_should_run]
- env:
- TEST_RUNNER_TYPE: linux.2xlarge
- ENABLE_DISTRIBUTED_TEST: 1
- ENABLE_JIT_LEGACY_TEST: ''
- ENABLE_MULTIGPU_TEST: ''
- ENABLE_NOGPU_NO_AVX_TEST: ''
- ENABLE_NOGPU_NO_AVX2_TEST: ''
- ENABLE_SLOW_TEST: ''
- NUM_TEST_SHARDS: 1
- MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu
- NOGPU_RUNNER_TYPE: linux.2xlarge
- PR_BODY: ${{ github.event.pull_request.body }}
- outputs:
- matrix: ${{ steps.set-matrix.outputs.matrix }}
- render-matrix: ${{ steps.set-matrix.outputs.render-matrix }}
- ignore-disabled-issues: ${{ steps.set-matrix.outputs.ignore-disabled-issues }}
- container:
- image: python:3.9
- steps:
- - name: Install dependencies
- run: pip install typing-extensions
- - name: Clone pytorch/pytorch
- uses: actions/checkout@v2
- - name: Generating test matrix
- id: set-matrix
- run: .github/scripts/generate_pytorch_test_matrix.py
-
- test:
- needs: [calculate-docker-image, build, generate-test-matrix, ciflow_should_run]
- strategy:
- matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }}
- fail-fast: false
- runs-on: ${{ matrix.runner }}
- env:
- DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }}
- JOB_BASE_NAME: parallelnative-linux-xenial-py3.6-gcc5.4-test
- TEST_CONFIG: ${{ matrix.config }}
- SHARD_NUMBER: ${{ matrix.shard }}
- NUM_TEST_SHARDS: ${{ matrix.num_shards }}
- PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }}
- CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }}
- steps:
- - name: Log in to ECR
- run: |
- aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
- bash /tmp/ecr-login.sh
- rm /tmp/ecr-login.sh
- - name: Chown workspace
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)/../":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Clean workspace
- run: |
- rm -rf "${GITHUB_WORKSPACE:?}/*"
- rm -f ~/.ssh/authorized_keys
- - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)"
- uses: seemethere/add-github-ssh-key@v1
- with:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Checkout PyTorch
- uses: actions/checkout@v2
- with:
- fetch-depth: 0 # deep clone, to allow sharding to use git rev-list
- submodules: recursive
- - name: Display EC2 information
- shell: bash
- run: |
- .github/scripts/display_ec2_information.sh
- - name: Pull docker image
- run: |
- docker pull "${DOCKER_IMAGE}"
- - name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
- if: ${{ contains(env.BUILD_ENVIRONMENT, 'cuda') && !contains(matrix.config, 'nogpu') }}
- run: |
- bash .github/scripts/install_nvidia_utils_linux.sh
- echo "GPU_FLAG=--gpus all" >> "${GITHUB_ENV}"
- - name: Determine shm-size
- run: |
- shm_size="1g"
- case "${BUILD_ENVIRONMENT}" in
- *cuda*)
- shm_size="2g"
- ;;
- *rocm*)
- shm_size="8g"
- ;;
- esac
- echo "SHM_SIZE=${shm_size}" >> "${GITHUB_ENV}"
- - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
- name: Download PyTorch Build Artifacts
- with:
- name: ${{ env.BUILD_ENVIRONMENT }}
- - name: Unzip artifacts
- run: |
- unzip -o artifacts.zip
- - name: Output disk space left
- run: |
- sudo df -H
- - name: Preserve github env variables for use in docker
- run: |
- env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}"
- - name: Test PyTorch
- env:
- BUILD_ENVIRONMENT: parallelnative-linux-xenial-py3.6-gcc5.4-${{ matrix.config }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- run: |
- if [[ $TEST_CONFIG == 'multigpu' ]]; then
- TEST_COMMAND=.jenkins/pytorch/multigpu-test.sh
- else
- TEST_COMMAND=.jenkins/pytorch/test.sh
- fi
- if [[ $NUM_TEST_SHARDS -ne 2 ]]; then
- export SHARD_NUMBER=0
- fi
- # TODO: Stop building test binaries as part of the build phase
- # Used for GPU_FLAG since that doesn't play nice
- # shellcheck disable=SC2086
- docker run \
- ${GPU_FLAG:-} \
- -e BUILD_ENVIRONMENT \
- -e PR_NUMBER \
- -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \
- -e GITHUB_ACTIONS \
- -e IN_CI \
- -e IN_WHEEL_TEST \
- -e SHARD_NUMBER \
- -e JOB_BASE_NAME \
- -e TEST_CONFIG \
- -e NUM_TEST_SHARDS \
- -e PYTORCH_IGNORE_DISABLED_ISSUES \
- -e CONTINUE_THROUGH_ERROR \
- -e MAX_JOBS="$(nproc --ignore=2)" \
- -e SCCACHE_BUCKET \
- -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \
- --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
- --security-opt seccomp=unconfined \
- --cap-add=SYS_PTRACE \
- --shm-size="${SHM_SIZE}" \
- --tty \
- --user jenkins \
- -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
- -w /var/lib/jenkins/workspace \
- "${DOCKER_IMAGE}" \
- sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND
- - name: Chown workspace
- if: always()
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Zip test reports for upload
- if: always()
- env:
- COMMIT_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
- WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
- run: |
- # Remove any previous test reports if they exist
- rm -f test-reports-*.zip
- zip -r "test-reports-${COMMIT_SHA1}-${WORKFLOW_ID}.zip" test -i '*.xml'
- - uses: actions/upload-artifact@v2
- name: Store PyTorch Test Reports
- if: always()
- with:
- name: test-reports-${{ matrix.config }}
- retention-days: 14
- if-no-files-found: error
- path:
- test-reports-*.zip
- - uses: seemethere/upload-artifact-s3@9d7ceb0ab39c2c88d93ef7792b27425b27d59162
- name: Store PyTorch Test Reports on S3
- if: always()
- with:
- name: test-reports-${{ matrix.config }}
- retention-days: 14
- if-no-files-found: error
- path:
- test-reports-*.zip
- - name: Parse ref
- id: parse-ref
- run: .github/scripts/parse_ref.py
- - name: Display and upload test statistics (Click Me)
- # temporary hack: set CIRCLE_* vars, until we update
- # tools/stats/print_test_stats.py to natively support GitHub Actions
- env:
- AWS_DEFAULT_REGION: us-east-1
- CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
- JOB_BASE_NAME: parallelnative-linux-xenial-py3.6-gcc5.4-test
- CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
- CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
- CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
- CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
- shell: bash
- run: |
- python3 -m pip install -r requirements.txt
- python3 -m pip install boto3==1.16.34
- python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
- - name: Hold runner for 2 hours or until ssh sessions have drained
- # Always hold for active ssh sessions
- if: always()
- run: .github/scripts/wait_for_ssh_to_drain.sh
- - name: Clean up docker images
- if: always()
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- # Prune all of the docker images
- docker system prune -af
+++ /dev/null
-# @generated DO NOT EDIT MANUALLY
-
-# Template is at: .github/templates/linux_ci_workflow.yml.j2
-# Generation script: .github/scripts/generate_ci_workflows.py
-name: paralleltbb-linux-xenial-py3.6-gcc5.4
-
-on:
- pull_request:
- types: [unassigned]
- push:
- branches:
- - master
- - release/*
- workflow_dispatch:
-
-env:
- BUILD_ENVIRONMENT: paralleltbb-linux-xenial-py3.6-gcc5.4
- DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4
- SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2
- TORCH_CUDA_ARCH_LIST: 5.2
- IN_CI: 1
- # This is used for the phase of adding wheel tests only, will be removed once completed
- IN_WHEEL_TEST: 1
- # Used for custom_opertor, jit_hooks, custom_backend, see .jenkins/pytorch/build.sh
- CUSTOM_TEST_ARTIFACT_BUILD_DIR: build/custom_test_artifacts
- ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine"
-
-concurrency:
- group: paralleltbb-linux-xenial-py3.6-gcc5.4-${{ github.event.pull_request.number || github.sha }}
- cancel-in-progress: true
-
-jobs:
- ciflow_should_run:
- runs-on: ubuntu-18.04
- if: ${{ (github.event_name != 'pull_request') || (github.event.action !='unassigned') || (contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/cpu') || contains(github.event.pull_request.labels.*.name, 'ciflow/linux')) }}
- steps:
- - name: noop
- run: echo running ciflow_should_run
- calculate-docker-image:
- if: ${{ github.repository_owner == 'pytorch' }}
- runs-on: linux.2xlarge
- needs: [ciflow_should_run]
- env:
- DOCKER_BUILDKIT: 1
- timeout-minutes: 90
- outputs:
- docker_image: ${{ steps.calculate-tag.outputs.docker_image }}
- steps:
- - name: Log in to ECR
- run: |
- aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
- bash /tmp/ecr-login.sh
- rm /tmp/ecr-login.sh
- - name: Chown workspace
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Clean workspace
- run: |
- rm -rf "${GITHUB_WORKSPACE:?}/*"
- rm -f ~/.ssh/authorized_keys
- - name: Checkout PyTorch
- uses: actions/checkout@v2
- with:
- # deep clone, to allow use of git merge-base
- fetch-depth: 0
- - name: Display EC2 information
- shell: bash
- run: |
- .github/scripts/display_ec2_information.sh
- - name: Calculate docker image tag
- id: calculate-tag
- run: |
- DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker)
- echo "::set-output name=docker_tag::${DOCKER_TAG}"
- echo "::set-output name=docker_image::${DOCKER_IMAGE_BASE}:${DOCKER_TAG}"
- - name: Check if image should be built
- id: check
- env:
- DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker_tag }}
- BASE_REVISION: ${{ github.event.pull_request.base.sha || github.sha }}
- run: |
- set -x
- # Check if image already exists, if it does then skip building it
- if docker manifest inspect "${DOCKER_IMAGE_BASE}:${DOCKER_TAG}"; then
- exit 0
- fi
- if [[ "$BASE_REVISION" = "$(git rev-parse HEAD)" ]]; then
- # if we're on the base branch then use the parent commit
- MERGE_BASE=$(git rev-parse HEAD~)
- else
- # otherwise we're on a PR, so use the most recent base commit
- MERGE_BASE=$(git merge-base HEAD "$BASE_REVISION")
- fi
- # Covers the case where a previous tag doesn't exist for the tree
- # this is only really applicable on trees that don't have `.circleci/docker` at its merge base, i.e. nightly
- if ! git rev-parse "$MERGE_BASE:.circleci/docker"; then
- echo "Directory '.circleci/docker' not found in commit $MERGE_BASE, you should probably rebase onto a more recent commit"
- exit 1
- fi
- PREVIOUS_DOCKER_TAG=$(git rev-parse "$MERGE_BASE:.circleci/docker")
- # If no image exists but the hash is the same as the previous hash then we should error out here
- if [[ "${PREVIOUS_DOCKER_TAG}" = "${DOCKER_TAG}" ]]; then
- echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
- echo " contact the PyTorch team to restore the original images"
- exit 1
- fi
- echo ::set-output name=rebuild::yes
- - name: Build and push docker image
- if: ${{ steps.check.outputs.rebuild }}
- env:
- DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker_tag }}
- DOCKER_SKIP_S3_UPLOAD: 1
- run: |
- export IMAGE_NAME=${DOCKER_IMAGE_BASE#308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/}
- cd .circleci/docker && ./build_docker.sh
-
- build:
- runs-on: linux.2xlarge
- needs: [calculate-docker-image, ciflow_should_run]
- env:
- DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }}
- JOB_BASE_NAME: paralleltbb-linux-xenial-py3.6-gcc5.4-build
- steps:
- - name: Log in to ECR
- run: |
- aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
- bash /tmp/ecr-login.sh
- rm /tmp/ecr-login.sh
- - name: Chown workspace
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Clean workspace
- run: |
- rm -rf "${GITHUB_WORKSPACE:?}/*"
- rm -f ~/.ssh/authorized_keys
- - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)"
- uses: seemethere/add-github-ssh-key@v1
- with:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Checkout PyTorch
- uses: actions/checkout@v2
- with:
- fetch-depth: 0 # deep clone, to allow sharding to use git rev-list
- submodules: recursive
- - name: Display EC2 information
- shell: bash
- run: |
- .github/scripts/display_ec2_information.sh
- - name: Pull docker image
- run: |
- docker pull "${DOCKER_IMAGE}"
- - name: Preserve github env variables for use in docker
- run: |
- env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}"
- - name: Build PyTorch
- run: |
- docker run \
- -e BUILD_ENVIRONMENT \
- -e JOB_BASE_NAME \
- -e MAX_JOBS="$(nproc --ignore=2)" \
- -e SCCACHE_BUCKET \
- -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \
- -e SKIP_SCCACHE_INITIALIZATION=1 \
- -e TORCH_CUDA_ARCH_LIST \
- -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \
- --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
- --security-opt seccomp=unconfined \
- --cap-add=SYS_PTRACE \
- --tty \
- --user jenkins \
- -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
- -w /var/lib/jenkins/workspace \
- "${DOCKER_IMAGE}" \
- sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh'
- - name: Parse ref
- id: parse-ref
- run: .github/scripts/parse_ref.py
- - name: Display and upload binary build size statistics (Click Me)
- # temporary hack: set CIRCLE_* vars, until we update
- # tools/stats/print_test_stats.py to natively support GitHub Actions
- env:
- AWS_DEFAULT_REGION: us-east-1
- SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
- CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
- CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
- CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
- CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
- CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
- run: |
- COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
- export COMMIT_TIME
- pip3 install requests
- python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
- - name: Chown workspace
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Archive artifacts into zip
- run: |
- zip -1 -r artifacts.zip dist/ build/custom_test_artifacts build/lib build/bin .pytorch-test-times.json
- - uses: seemethere/upload-artifact-s3@9d7ceb0ab39c2c88d93ef7792b27425b27d59162
- name: Store PyTorch Build Artifacts on S3
- with:
- name: ${{ env.BUILD_ENVIRONMENT }}
- retention-days: 14
- if-no-files-found: error
- path:
- artifacts.zip
- - name: Hold runner for 2 hours or until ssh sessions have drained
- # Always hold for active ssh sessions
- if: always()
- run: .github/scripts/wait_for_ssh_to_drain.sh
- - name: Clean up docker images
- if: always()
- run: |
- # Prune all of the docker images
- docker system prune -af
-
- generate-test-matrix:
- if: ${{ github.repository_owner == 'pytorch' }}
- runs-on: ubuntu-18.04
- needs: [ciflow_should_run]
- env:
- TEST_RUNNER_TYPE: linux.2xlarge
- ENABLE_DISTRIBUTED_TEST: 1
- ENABLE_JIT_LEGACY_TEST: ''
- ENABLE_MULTIGPU_TEST: ''
- ENABLE_NOGPU_NO_AVX_TEST: ''
- ENABLE_NOGPU_NO_AVX2_TEST: ''
- ENABLE_SLOW_TEST: ''
- NUM_TEST_SHARDS: 1
- MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu
- NOGPU_RUNNER_TYPE: linux.2xlarge
- PR_BODY: ${{ github.event.pull_request.body }}
- outputs:
- matrix: ${{ steps.set-matrix.outputs.matrix }}
- render-matrix: ${{ steps.set-matrix.outputs.render-matrix }}
- ignore-disabled-issues: ${{ steps.set-matrix.outputs.ignore-disabled-issues }}
- container:
- image: python:3.9
- steps:
- - name: Install dependencies
- run: pip install typing-extensions
- - name: Clone pytorch/pytorch
- uses: actions/checkout@v2
- - name: Generating test matrix
- id: set-matrix
- run: .github/scripts/generate_pytorch_test_matrix.py
-
- test:
- needs: [calculate-docker-image, build, generate-test-matrix, ciflow_should_run]
- strategy:
- matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }}
- fail-fast: false
- runs-on: ${{ matrix.runner }}
- env:
- DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }}
- JOB_BASE_NAME: paralleltbb-linux-xenial-py3.6-gcc5.4-test
- TEST_CONFIG: ${{ matrix.config }}
- SHARD_NUMBER: ${{ matrix.shard }}
- NUM_TEST_SHARDS: ${{ matrix.num_shards }}
- PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }}
- CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }}
- steps:
- - name: Log in to ECR
- run: |
- aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh
- bash /tmp/ecr-login.sh
- rm /tmp/ecr-login.sh
- - name: Chown workspace
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)/../":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Clean workspace
- run: |
- rm -rf "${GITHUB_WORKSPACE:?}/*"
- rm -f ~/.ssh/authorized_keys
- - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)"
- uses: seemethere/add-github-ssh-key@v1
- with:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Checkout PyTorch
- uses: actions/checkout@v2
- with:
- fetch-depth: 0 # deep clone, to allow sharding to use git rev-list
- submodules: recursive
- - name: Display EC2 information
- shell: bash
- run: |
- .github/scripts/display_ec2_information.sh
- - name: Pull docker image
- run: |
- docker pull "${DOCKER_IMAGE}"
- - name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
- if: ${{ contains(env.BUILD_ENVIRONMENT, 'cuda') && !contains(matrix.config, 'nogpu') }}
- run: |
- bash .github/scripts/install_nvidia_utils_linux.sh
- echo "GPU_FLAG=--gpus all" >> "${GITHUB_ENV}"
- - name: Determine shm-size
- run: |
- shm_size="1g"
- case "${BUILD_ENVIRONMENT}" in
- *cuda*)
- shm_size="2g"
- ;;
- *rocm*)
- shm_size="8g"
- ;;
- esac
- echo "SHM_SIZE=${shm_size}" >> "${GITHUB_ENV}"
- - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b
- name: Download PyTorch Build Artifacts
- with:
- name: ${{ env.BUILD_ENVIRONMENT }}
- - name: Unzip artifacts
- run: |
- unzip -o artifacts.zip
- - name: Output disk space left
- run: |
- sudo df -H
- - name: Preserve github env variables for use in docker
- run: |
- env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}"
- - name: Test PyTorch
- env:
- BUILD_ENVIRONMENT: paralleltbb-linux-xenial-py3.6-gcc5.4-${{ matrix.config }}
- PR_NUMBER: ${{ github.event.pull_request.number }}
- run: |
- if [[ $TEST_CONFIG == 'multigpu' ]]; then
- TEST_COMMAND=.jenkins/pytorch/multigpu-test.sh
- else
- TEST_COMMAND=.jenkins/pytorch/test.sh
- fi
- if [[ $NUM_TEST_SHARDS -ne 2 ]]; then
- export SHARD_NUMBER=0
- fi
- # TODO: Stop building test binaries as part of the build phase
- # Used for GPU_FLAG since that doesn't play nice
- # shellcheck disable=SC2086
- docker run \
- ${GPU_FLAG:-} \
- -e BUILD_ENVIRONMENT \
- -e PR_NUMBER \
- -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \
- -e GITHUB_ACTIONS \
- -e IN_CI \
- -e IN_WHEEL_TEST \
- -e SHARD_NUMBER \
- -e JOB_BASE_NAME \
- -e TEST_CONFIG \
- -e NUM_TEST_SHARDS \
- -e PYTORCH_IGNORE_DISABLED_ISSUES \
- -e CONTINUE_THROUGH_ERROR \
- -e MAX_JOBS="$(nproc --ignore=2)" \
- -e SCCACHE_BUCKET \
- -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \
- --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
- --security-opt seccomp=unconfined \
- --cap-add=SYS_PTRACE \
- --shm-size="${SHM_SIZE}" \
- --tty \
- --user jenkins \
- -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
- -w /var/lib/jenkins/workspace \
- "${DOCKER_IMAGE}" \
- sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND
- - name: Chown workspace
- if: always()
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- - name: Zip test reports for upload
- if: always()
- env:
- COMMIT_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
- WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
- run: |
- # Remove any previous test reports if they exist
- rm -f test-reports-*.zip
- zip -r "test-reports-${COMMIT_SHA1}-${WORKFLOW_ID}.zip" test -i '*.xml'
- - uses: actions/upload-artifact@v2
- name: Store PyTorch Test Reports
- if: always()
- with:
- name: test-reports-${{ matrix.config }}
- retention-days: 14
- if-no-files-found: error
- path:
- test-reports-*.zip
- - uses: seemethere/upload-artifact-s3@9d7ceb0ab39c2c88d93ef7792b27425b27d59162
- name: Store PyTorch Test Reports on S3
- if: always()
- with:
- name: test-reports-${{ matrix.config }}
- retention-days: 14
- if-no-files-found: error
- path:
- test-reports-*.zip
- - name: Parse ref
- id: parse-ref
- run: .github/scripts/parse_ref.py
- - name: Display and upload test statistics (Click Me)
- # temporary hack: set CIRCLE_* vars, until we update
- # tools/stats/print_test_stats.py to natively support GitHub Actions
- env:
- AWS_DEFAULT_REGION: us-east-1
- CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }}
- JOB_BASE_NAME: paralleltbb-linux-xenial-py3.6-gcc5.4-test
- CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }}
- CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
- CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
- CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
- shell: bash
- run: |
- python3 -m pip install -r requirements.txt
- python3 -m pip install boto3==1.16.34
- python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
- - name: Hold runner for 2 hours or until ssh sessions have drained
- # Always hold for active ssh sessions
- if: always()
- run: .github/scripts/wait_for_ssh_to_drain.sh
- - name: Clean up docker images
- if: always()
- run: |
- # Ensure the working directory gets chowned back to the current user
- docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" .
- # Prune all of the docker images
- docker system prune -af