From 3c79e0b314c56c01e119b22e834922923a63ad9e Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 2 Sep 2021 14:49:47 -0700 Subject: [PATCH] .github: Migrate pytorch_linux_bionic_py_3_6_clang9 to GHA (#64218) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/64218 Relies on https://github.com/fairinternal/pytorch-gha-infra/pull/11 Signed-off-by: Eli Uriegas cc ezyang seemethere malfet walterddr lg20987 pytorch/pytorch-dev-infra bdhirsh Test Plan: Imported from OSS Reviewed By: malfet, H-Huang, janeyx99 Differential Revision: D30651516 Pulled By: seemethere fbshipit-source-id: e5843dfe84f096f2872d88f2e53e9408ad2fe399 --- .circleci/cimodel/data/pytorch_build_data.py | 5 - .circleci/config.yml | 13 - .github/generated-ciflow-ruleset.json | 10 + .github/scripts/generate_ci_workflows.py | 36 +- .github/scripts/generate_pytorch_test_matrix.py | 4 + .github/templates/linux_ci_workflow.yml.j2 | 5 + ...d-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml | 2 + ...d-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml | 2 + .../generated-linux-bionic-cuda10.2-py3.9-gcc7.yml | 5 + .../generated-linux-bionic-py3.6-clang9.yml | 506 +++++++++++++++++++++ .../generated-linux-bionic-py3.8-gcc9-coverage.yml | 5 + .../generated-linux-xenial-cuda10.2-py3.6-gcc7.yml | 5 + .../generated-linux-xenial-cuda11.3-py3.6-gcc7.yml | 5 + .../generated-linux-xenial-py3.6-gcc5.4.yml | 5 + ...enerated-linux-xenial-py3.6-gcc7-bazel-test.yml | 1 + ...c-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml | 2 + ...d-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml | 5 + 17 files changed, 580 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/generated-linux-bionic-py3.6-clang9.yml diff --git a/.circleci/cimodel/data/pytorch_build_data.py b/.circleci/cimodel/data/pytorch_build_data.py index 7d43a73..dbe17bf 100644 --- a/.circleci/cimodel/data/pytorch_build_data.py +++ b/.circleci/cimodel/data/pytorch_build_data.py @@ -53,11 +53,6 @@ CONFIG_TREE_DATA = [ ("clang", [ ("9", [ ("3.6", [ - ("noarch", [XImportant(True)]), - ]), - ]), - ("9", [ - ("3.6", [ ("xla", [XImportant(True)]), ("vulkan", [XImportant(True)]), ]), diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e17576..ffc67a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7408,19 +7408,6 @@ workflows: docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7" resource_class: large - pytorch_linux_build: - name: pytorch_linux_bionic_py3_6_clang9_noarch_build - requires: - - "docker-pytorch-linux-bionic-py3.6-clang9" - build_environment: "pytorch-linux-bionic-py3.6-clang9-noarch-build" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9" - - pytorch_linux_test: - name: pytorch_linux_bionic_py3_6_clang9_noarch_test - requires: - - pytorch_linux_bionic_py3_6_clang9_noarch_build - build_environment: "pytorch-linux-bionic-py3.6-clang9-noarch-test" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9" - resource_class: large - - pytorch_linux_build: name: pytorch_xla_linux_bionic_py3_6_clang9_build requires: - "docker-pytorch-linux-bionic-py3.6-clang9" diff --git a/.github/generated-ciflow-ruleset.json b/.github/generated-ciflow-ruleset.json index d135611..d3ebad3 100644 --- a/.github/generated-ciflow-ruleset.json +++ b/.github/generated-ciflow-ruleset.json @@ -5,6 +5,7 @@ "libtorch-linux-xenial-cuda10.2-py3.6-gcc7", "libtorch-linux-xenial-cuda11.3-py3.6-gcc7", "linux-bionic-cuda10.2-py3.9-gcc7", + "linux-bionic-py3.6-clang9", "linux-bionic-py3.8-gcc9-coverage", "linux-xenial-cuda10.2-py3.6-gcc7", "linux-xenial-cuda11.3-py3.6-gcc7", @@ -24,6 +25,7 @@ "linux-bionic-py3.8-gcc9-coverage" ], "ciflow/cpu": [ + "linux-bionic-py3.6-clang9", "linux-bionic-py3.8-gcc9-coverage", "linux-xenial-py3.6-gcc5.4", "linux-xenial-py3.6-gcc7-bazel-test", @@ -42,6 +44,7 @@ "win-vs2019-cuda11.3-py3" ], "ciflow/default": [ + "linux-bionic-py3.6-clang9", "linux-bionic-py3.8-gcc9-coverage", "linux-xenial-cuda11.3-py3.6-gcc7", "linux-xenial-py3.6-gcc5.4", @@ -58,6 +61,7 @@ "libtorch-linux-xenial-cuda10.2-py3.6-gcc7", "libtorch-linux-xenial-cuda11.3-py3.6-gcc7", "linux-bionic-cuda10.2-py3.9-gcc7", + "linux-bionic-py3.6-clang9", "linux-bionic-py3.8-gcc9-coverage", "linux-xenial-cuda10.2-py3.6-gcc7", "linux-xenial-cuda11.3-py3.6-gcc7", @@ -66,6 +70,9 @@ "periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7", "periodic-linux-xenial-cuda11.1-py3.6-gcc7" ], + "ciflow/noarch": [ + "linux-bionic-py3.6-clang9" + ], "ciflow/scheduled": [ "periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7", "periodic-linux-xenial-cuda11.1-py3.6-gcc7", @@ -80,6 +87,9 @@ "win-vs2019-cpu-py3", "win-vs2019-cuda10.1-py3", "win-vs2019-cuda11.3-py3" + ], + "ciflow/xla": [ + "linux-bionic-py3.6-clang9" ] }, "version": "v1" diff --git a/.github/scripts/generate_ci_workflows.py b/.github/scripts/generate_ci_workflows.py index 467d13d..16100f7 100755 --- a/.github/scripts/generate_ci_workflows.py +++ b/.github/scripts/generate_ci_workflows.py @@ -49,6 +49,8 @@ LABEL_CIFLOW_LINUX = "ciflow/linux" LABEL_CIFLOW_SCHEDULED = "ciflow/scheduled" LABEL_CIFLOW_SLOW = "ciflow/slow" LABEL_CIFLOW_WIN = "ciflow/win" +LABEL_CIFLOW_XLA = "ciflow/xla" +LABEL_CIFLOW_NOARCH = "ciflow/noarch" @dataclass @@ -150,6 +152,8 @@ class CIWorkflow: enable_slow_test: YamlShellBool = "''" enable_docs_test: YamlShellBool = "''" enable_backwards_compat_test: YamlShellBool = "''" + enable_xla_test: YamlShellBool = "''" + enable_noarch_test: YamlShellBool = "''" def __post_init__(self) -> None: if self.is_libtorch: @@ -409,24 +413,6 @@ LINUX_WORKFLOWS = [ labels={LABEL_CIFLOW_SCHEDULED, LABEL_CIFLOW_LINUX, LABEL_CIFLOW_LIBTORCH, LABEL_CIFLOW_CUDA}, ), ), - # CIWorkflow( - # arch="linux", - # build_environment="linux-bionic-py3.6-clang9-noarch", - # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9", - # test_runner_type=LINUX_CPU_TEST_RUNNER, - # ), - # CIWorkflow( - # arch="linux", - # build_environment="xla-linux-bionic-py3.6-clang9", - # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9", - # test_runner_type=LINUX_CPU_TEST_RUNNER, - # ), - # CIWorkflow( - # arch="linux", - # build_environment="vulkan-linux-bionic-py3.6-clang9", - # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9", - # test_runner_type=LINUX_CPU_TEST_RUNNER, - # ), CIWorkflow( arch="linux", build_environment="linux-bionic-py3.8-gcc9-coverage", @@ -440,6 +426,20 @@ LINUX_WORKFLOWS = [ labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_COVERAGE, LABEL_CIFLOW_LINUX, LABEL_CIFLOW_CPU}, ), ), + CIWorkflow( + arch="linux", + build_environment="linux-bionic-py3.6-clang9", + docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9", + test_runner_type=LINUX_CPU_TEST_RUNNER, + on_pull_request=True, + num_test_shards=2, + distributed_test=False, + enable_noarch_test=1, + ciflow_config=CIFlowConfig( + enabled=True, + labels={LABEL_CIFLOW_DEFAULT, LABEL_CIFLOW_LINUX, LABEL_CIFLOW_CPU, LABEL_CIFLOW_XLA, LABEL_CIFLOW_NOARCH}, + ), + ), # CIWorkflow( # arch="linux", # build_environment="linux-bionic-rocm3.9-py3.6", diff --git a/.github/scripts/generate_pytorch_test_matrix.py b/.github/scripts/generate_pytorch_test_matrix.py index cb71f58..beb1b9d 100755 --- a/.github/scripts/generate_pytorch_test_matrix.py +++ b/.github/scripts/generate_pytorch_test_matrix.py @@ -59,6 +59,10 @@ def main() -> None: configs['docs_test'] = {'num_shards': 1, 'runner': TEST_RUNNER_TYPE} if os.getenv('ENABLE_BACKWARDS_COMPAT_TEST'): configs['backwards_compat'] = {'num_shards': 1, 'runner': TEST_RUNNER_TYPE} + if os.getenv('ENABLE_XLA_TEST'): + configs['xla'] = {'num_shards': 1, 'runner': TEST_RUNNER_TYPE} + if os.getenv('ENABLE_NOARCH_TEST'): + configs['noarch'] = {'num_shards': 1, 'runner': TEST_RUNNER_TYPE} matrix = { 'include': [ { diff --git a/.github/templates/linux_ci_workflow.yml.j2 b/.github/templates/linux_ci_workflow.yml.j2 index 8aa8547..314122b 100644 --- a/.github/templates/linux_ci_workflow.yml.j2 +++ b/.github/templates/linux_ci_workflow.yml.j2 @@ -35,6 +35,7 @@ env: BUILD_ENVIRONMENT: !{{ build_environment }} DOCKER_IMAGE_BASE: !{{ docker_image_base }} SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -137,6 +138,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ @@ -214,6 +216,8 @@ jobs: ENABLE_SLOW_TEST: !{{ enable_slow_test }} ENABLE_DOCS_TEST: !{{ enable_docs_test }} ENABLE_BACKWARDS_COMPAT_TEST: !{{ enable_backwards_compat_test }} + ENABLE_XLA_TEST: !{{ enable_xla_test }} + ENABLE_NOARCH_TEST: !{{ enable_noarch_test }} NUM_TEST_SHARDS: !{{ num_test_shards }} MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu NOGPU_RUNNER_TYPE: linux.2xlarge @@ -313,6 +317,7 @@ jobs: -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="!{{ common.squid_proxy }}" -e https_proxy="!{{ common.squid_proxy }}" -e no_proxy="!{{ common.squid_no_proxy }}" \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ diff --git a/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml b/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml index 396284c..d211189 100644 --- a/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml +++ b/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml @@ -16,6 +16,7 @@ env: BUILD_ENVIRONMENT: libtorch-linux-xenial-cuda10.2-py3.6-gcc7 DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -192,6 +193,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ diff --git a/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml b/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml index cb0a985..51811ae 100644 --- a/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml +++ b/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml @@ -16,6 +16,7 @@ env: BUILD_ENVIRONMENT: libtorch-linux-xenial-cuda11.3-py3.6-gcc7 DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda11.3-cudnn8-py3-gcc7 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -192,6 +193,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ diff --git a/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml b/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml index ddb38b7..f410f1f 100644 --- a/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml +++ b/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml @@ -16,6 +16,7 @@ env: BUILD_ENVIRONMENT: linux-bionic-cuda10.2-py3.9-gcc7 DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -192,6 +193,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ @@ -282,6 +284,8 @@ jobs: ENABLE_SLOW_TEST: '' ENABLE_DOCS_TEST: '' ENABLE_BACKWARDS_COMPAT_TEST: '' + ENABLE_XLA_TEST: '' + ENABLE_NOARCH_TEST: '' NUM_TEST_SHARDS: 2 MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu NOGPU_RUNNER_TYPE: linux.2xlarge @@ -420,6 +424,7 @@ jobs: -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ diff --git a/.github/workflows/generated-linux-bionic-py3.6-clang9.yml b/.github/workflows/generated-linux-bionic-py3.6-clang9.yml new file mode 100644 index 0000000..e54555d --- /dev/null +++ b/.github/workflows/generated-linux-bionic-py3.6-clang9.yml @@ -0,0 +1,506 @@ +# @generated DO NOT EDIT MANUALLY +# Template is at: .github/templates/linux_ci_workflow.yml.j2 +# Generation script: .github/scripts/generate_ci_workflows.py +name: linux-bionic-py3.6-clang9 + +on: + pull_request: + types: [opened, synchronize, reopened, unassigned] + push: + branches: + - master + - release/* + workflow_dispatch: + +env: + BUILD_ENVIRONMENT: linux-bionic-py3.6-clang9 + DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.6-clang9 + SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla + TORCH_CUDA_ARCH_LIST: 5.2 + IN_CI: 1 + # This is used for the phase of adding wheel tests only, will be removed once completed + IN_WHEEL_TEST: 1 + # Used for custom_opertor, jit_hooks, custom_backend, see .jenkins/pytorch/build.sh + CUSTOM_TEST_ARTIFACT_BUILD_DIR: build/custom_test_artifacts + ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" + PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} + +concurrency: + group: linux-bionic-py3.6-clang9-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + ciflow_should_run: + runs-on: ubuntu-18.04 + if: ${{ (github.event_name != 'pull_request') || (github.event.action !='unassigned') || (contains(github.event.pull_request.labels.*.name, 'ciflow/all') || contains(github.event.pull_request.labels.*.name, 'ciflow/cpu') || contains(github.event.pull_request.labels.*.name, 'ciflow/default') || contains(github.event.pull_request.labels.*.name, 'ciflow/linux') || contains(github.event.pull_request.labels.*.name, 'ciflow/noarch') || contains(github.event.pull_request.labels.*.name, 'ciflow/xla')) }} + steps: + - name: noop + run: echo running ciflow_should_run + calculate-docker-image: + if: ${{ github.repository_owner == 'pytorch' }} + runs-on: linux.2xlarge + needs: [ciflow_should_run] + env: + DOCKER_BUILDKIT: 1 + timeout-minutes: 90 + outputs: + docker_image: ${{ steps.calculate-tag.outputs.docker_image }} + steps: + - name: Display EC2 information + shell: bash + run: | + set -euo pipefail + function get_ec2_metadata() { + # Pulled from instance metadata endpoint for EC2 + # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html + category=$1 + curl -fsSL "http://169.254.169.254/latest/meta-data/${category}" + } + echo "ami-id: $(get_ec2_metadata ami-id)" + echo "instance-id: $(get_ec2_metadata instance-id)" + echo "instance-type: $(get_ec2_metadata instance-type)" + - name: Log in to ECR + run: | + aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh + bash /tmp/ecr-login.sh + rm /tmp/ecr-login.sh + - name: Chown workspace + env: + ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" + run: | + # Ensure the working directory gets chowned back to the current user + docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Clean workspace + run: | + rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Preserve github env variables for use in docker + run: | + env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}" + - name: Checkout PyTorch + uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 + with: + # deep clone, to allow use of git merge-base + fetch-depth: 0 + submodules: false + - name: Calculate docker image tag + id: calculate-tag + run: | + DOCKER_TAG=$(git rev-parse HEAD:.circleci/docker) + echo "::set-output name=docker_tag::${DOCKER_TAG}" + echo "::set-output name=docker_image::${DOCKER_IMAGE_BASE}:${DOCKER_TAG}" + - name: Check if image should be built + id: check + env: + DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker_tag }} + BASE_REVISION: ${{ github.event.pull_request.base.sha || github.sha }} + run: | + set -x + # Check if image already exists, if it does then skip building it + if docker manifest inspect "${DOCKER_IMAGE_BASE}:${DOCKER_TAG}"; then + exit 0 + fi + if [[ "$BASE_REVISION" = "$(git rev-parse HEAD)" ]]; then + # if we're on the base branch then use the parent commit + MERGE_BASE=$(git rev-parse HEAD~) + else + # otherwise we're on a PR, so use the most recent base commit + MERGE_BASE=$(git merge-base HEAD "$BASE_REVISION") + fi + # Covers the case where a previous tag doesn't exist for the tree + # this is only really applicable on trees that don't have `.circleci/docker` at its merge base, i.e. nightly + if ! git rev-parse "$MERGE_BASE:.circleci/docker"; then + echo "Directory '.circleci/docker' not found in commit $MERGE_BASE, you should probably rebase onto a more recent commit" + exit 1 + fi + PREVIOUS_DOCKER_TAG=$(git rev-parse "$MERGE_BASE:.circleci/docker") + # If no image exists but the hash is the same as the previous hash then we should error out here + if [[ "${PREVIOUS_DOCKER_TAG}" = "${DOCKER_TAG}" ]]; then + echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch" + echo " contact the PyTorch team to restore the original images" + exit 1 + fi + echo ::set-output name=rebuild::yes + - name: Build and push docker image + if: ${{ steps.check.outputs.rebuild }} + env: + DOCKER_TAG: ${{ steps.calculate-tag.outputs.docker_tag }} + DOCKER_SKIP_S3_UPLOAD: 1 + run: | + export IMAGE_NAME=${DOCKER_IMAGE_BASE#308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/} + cd .circleci/docker && ./build_docker.sh + + build: + runs-on: linux.2xlarge + needs: [calculate-docker-image, ciflow_should_run] + env: + DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }} + JOB_BASE_NAME: linux-bionic-py3.6-clang9-build + steps: + - name: Display EC2 information + shell: bash + run: | + set -euo pipefail + function get_ec2_metadata() { + # Pulled from instance metadata endpoint for EC2 + # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html + category=$1 + curl -fsSL "http://169.254.169.254/latest/meta-data/${category}" + } + echo "ami-id: $(get_ec2_metadata ami-id)" + echo "instance-id: $(get_ec2_metadata instance-id)" + echo "instance-type: $(get_ec2_metadata instance-type)" + - name: Log in to ECR + run: | + aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh + bash /tmp/ecr-login.sh + rm /tmp/ecr-login.sh + - name: Chown workspace + env: + ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" + run: | + # Ensure the working directory gets chowned back to the current user + docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Clean workspace + run: | + rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Preserve github env variables for use in docker + run: | + env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}" + - name: Checkout PyTorch + uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 + with: + # deep clone, to allow use of git merge-base + fetch-depth: 0 + submodules: recursive + - name: Pull docker image + run: | + docker pull "${DOCKER_IMAGE}" + - name: Build PyTorch + run: | + docker run \ + -e BUILD_ENVIRONMENT \ + -e JOB_BASE_NAME \ + -e MAX_JOBS="$(nproc --ignore=2)" \ + -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ + -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ + -e SKIP_SCCACHE_INITIALIZATION=1 \ + -e TORCH_CUDA_ARCH_LIST \ + -e PR_LABELS \ + -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ + --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ + --security-opt seccomp=unconfined \ + --cap-add=SYS_PTRACE \ + --tty \ + --user jenkins \ + -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ + -w /var/lib/jenkins/workspace \ + "${DOCKER_IMAGE}" \ + sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + - name: Parse ref + id: parse-ref + run: .github/scripts/parse_ref.py + - name: Display and upload binary build size statistics (Click Me) + # temporary hack: set CIRCLE_* vars, until we update + # tools/stats/print_test_stats.py to natively support GitHub Actions + env: + AWS_DEFAULT_REGION: us-east-1 + SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} + CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} + CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} + CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }} + CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} + CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' + run: | + COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0) + export COMMIT_TIME + pip3 install requests + python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 + - name: Chown workspace + run: | + # Ensure the working directory gets chowned back to the current user + docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Archive artifacts into zip + run: | + zip -1 -r artifacts.zip dist/ build/custom_test_artifacts build/lib build/bin .pytorch-test-times.json + - uses: seemethere/upload-artifact-s3@v3 + name: Store PyTorch Build Artifacts on S3 + with: + name: ${{ env.BUILD_ENVIRONMENT }} + retention-days: 14 + if-no-files-found: error + path: + artifacts.zip + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh + - name: Kill containers, clean up images + if: always() + run: | + # ignore expansion of "docker ps -q" since it could be empty + # shellcheck disable=SC2046 + docker stop $(docker ps -q) || true + # Prune all of the docker images + docker system prune -af + - name: Chown workspace + env: + ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" + run: | + # Ensure the working directory gets chowned back to the current user + docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh + - name: Clean up docker images + if: always() + run: | + # Prune all of the docker images + docker system prune -af + + generate-test-matrix: + if: ${{ github.repository_owner == 'pytorch' }} + runs-on: ubuntu-18.04 + needs: [ciflow_should_run] + env: + TEST_RUNNER_TYPE: linux.2xlarge + ENABLE_DISTRIBUTED_TEST: '' + ENABLE_JIT_LEGACY_TEST: '' + ENABLE_MULTIGPU_TEST: '' + ENABLE_NOGPU_NO_AVX_TEST: '' + ENABLE_NOGPU_NO_AVX2_TEST: '' + ENABLE_SLOW_TEST: '' + ENABLE_DOCS_TEST: '' + ENABLE_BACKWARDS_COMPAT_TEST: '' + ENABLE_XLA_TEST: '' + ENABLE_NOARCH_TEST: 1 + NUM_TEST_SHARDS: 2 + MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu + NOGPU_RUNNER_TYPE: linux.2xlarge + PR_BODY: ${{ github.event.pull_request.body }} + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + render-matrix: ${{ steps.set-matrix.outputs.render-matrix }} + ignore-disabled-issues: ${{ steps.set-matrix.outputs.ignore-disabled-issues }} + container: + image: python:3.9 + steps: + - name: Install dependencies + run: pip install typing-extensions + - name: Clone pytorch/pytorch + uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 + - name: Generating test matrix + id: set-matrix + run: .github/scripts/generate_pytorch_test_matrix.py + + test: + needs: [calculate-docker-image, build, generate-test-matrix, ciflow_should_run] + strategy: + matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }} + fail-fast: false + runs-on: ${{ matrix.runner }} + env: + DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }} + JOB_BASE_NAME: linux-bionic-py3.6-clang9-test + TEST_CONFIG: ${{ matrix.config }} + SHARD_NUMBER: ${{ matrix.shard }} + NUM_TEST_SHARDS: ${{ matrix.num_shards }} + PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} + steps: + - name: Display EC2 information + shell: bash + run: | + set -euo pipefail + function get_ec2_metadata() { + # Pulled from instance metadata endpoint for EC2 + # see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html + category=$1 + curl -fsSL "http://169.254.169.254/latest/meta-data/${category}" + } + echo "ami-id: $(get_ec2_metadata ami-id)" + echo "instance-id: $(get_ec2_metadata instance-id)" + echo "instance-type: $(get_ec2_metadata instance-type)" + - name: Log in to ECR + run: | + aws ecr get-login --no-include-email --region us-east-1 > /tmp/ecr-login.sh + bash /tmp/ecr-login.sh + rm /tmp/ecr-login.sh + - name: Chown workspace + env: + ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" + run: | + # Ensure the working directory gets chowned back to the current user + docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Clean workspace + run: | + rm -rf "${GITHUB_WORKSPACE:?}/*" + rm -f ~/.ssh/authorized_keys + - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" + uses: seemethere/add-github-ssh-key@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Preserve github env variables for use in docker + run: | + env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}" + - name: Checkout PyTorch + uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9 + with: + # deep clone, to allow use of git merge-base + fetch-depth: 0 + submodules: recursive + - name: Pull docker image + run: | + docker pull "${DOCKER_IMAGE}" + - name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG + if: ${{ contains(env.BUILD_ENVIRONMENT, 'cuda') && !contains(matrix.config, 'nogpu') }} + run: | + bash .github/scripts/install_nvidia_utils_linux.sh + echo "GPU_FLAG=--gpus all" >> "${GITHUB_ENV}" + - name: Determine shm-size + run: | + shm_size="1g" + case "${BUILD_ENVIRONMENT}" in + *cuda*) + shm_size="2g" + ;; + *rocm*) + shm_size="8g" + ;; + esac + echo "SHM_SIZE=${shm_size}" >> "${GITHUB_ENV}" + - uses: seemethere/download-artifact-s3@0504774707cbc8603d7dca922e8026eb8bf3b47b + name: Download PyTorch Build Artifacts + with: + name: ${{ env.BUILD_ENVIRONMENT }} + - name: Unzip artifacts + run: | + unzip -o artifacts.zip + - name: Output disk space left + run: | + sudo df -H + - name: Test PyTorch + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + if [[ $TEST_CONFIG == 'multigpu' ]]; then + TEST_COMMAND=.jenkins/pytorch/multigpu-test.sh + else + TEST_COMMAND=.jenkins/pytorch/test.sh + fi + if [[ $NUM_TEST_SHARDS -ne 2 ]]; then + export SHARD_NUMBER=0 + fi + # TODO: Stop building test binaries as part of the build phase + # Used for GPU_FLAG since that doesn't play nice + # shellcheck disable=SC2086 + docker run \ + ${GPU_FLAG:-} \ + -e BUILD_ENVIRONMENT \ + -e PR_NUMBER \ + -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ + -e GITHUB_ACTIONS \ + -e IN_CI \ + -e IN_WHEEL_TEST \ + -e SHARD_NUMBER \ + -e JOB_BASE_NAME \ + -e TEST_CONFIG \ + -e NUM_TEST_SHARDS \ + -e PYTORCH_IGNORE_DISABLED_ISSUES \ + -e PR_LABELS \ + -e CONTINUE_THROUGH_ERROR \ + -e MAX_JOBS="$(nproc --ignore=2)" \ + -e SCCACHE_BUCKET \ + -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ + --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ + --security-opt seccomp=unconfined \ + --cap-add=SYS_PTRACE \ + --shm-size="${SHM_SIZE}" \ + --tty \ + --user jenkins \ + -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ + -w /var/lib/jenkins/workspace \ + "${DOCKER_IMAGE}" \ + sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + - name: Chown workspace + if: always() + run: | + # Ensure the working directory gets chowned back to the current user + docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . + - name: Zip test reports for upload + if: always() + env: + COMMIT_SHA1: ${{ github.event.pull_request.head.sha || github.sha }} + WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' + run: | + # Remove any previous test reports if they exist + rm -f test-reports-*.zip + zip -r "test-reports-${COMMIT_SHA1}-${WORKFLOW_ID}.zip" test -i '*.xml' + - uses: actions/upload-artifact@v2 + name: Store PyTorch Test Reports + if: always() + with: + name: test-reports-${{ matrix.config }} + retention-days: 14 + if-no-files-found: error + path: + test-reports-*.zip + - uses: seemethere/upload-artifact-s3@v3 + name: Store PyTorch Test Reports on S3 + if: always() + with: + name: test-reports-${{ matrix.config }} + retention-days: 14 + if-no-files-found: error + path: + test-reports-*.zip + - name: Parse ref + id: parse-ref + run: .github/scripts/parse_ref.py + - name: Display and upload test statistics (Click Me) + # temporary hack: set CIRCLE_* vars, until we update + # tools/stats/print_test_stats.py to natively support GitHub Actions + env: + AWS_DEFAULT_REGION: us-east-1 + CIRCLE_BRANCH: ${{ steps.parse-ref.outputs.branch }} + JOB_BASE_NAME: linux-bionic-py3.6-clang9-test + CIRCLE_PR_NUMBER: ${{ github.event.pull_request.number }} + CIRCLE_SHA1: ${{ github.event.pull_request.head.sha || github.sha }} + CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }} + CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}' + shell: bash + run: | + python3 -m pip install -r requirements.txt + python3 -m pip install boto3==1.16.34 + python3 -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test + - name: Hold runner for 2 hours or until ssh sessions have drained + # Always hold for active ssh sessions + if: always() + run: .github/scripts/wait_for_ssh_to_drain.sh + - name: Kill containers, clean up images + if: always() + run: | + # ignore expansion of "docker ps -q" since it could be empty + # shellcheck disable=SC2046 + docker stop $(docker ps -q) || true + # Prune all of the docker images + docker system prune -af + - name: Chown workspace + env: + ALPINE_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" + run: | + # Ensure the working directory gets chowned back to the current user + docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . diff --git a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml index c4cdd2c..2bcb426 100644 --- a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml +++ b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml @@ -16,6 +16,7 @@ env: BUILD_ENVIRONMENT: linux-bionic-py3.8-gcc9-coverage DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-py3.8-gcc9 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -192,6 +193,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ @@ -282,6 +284,8 @@ jobs: ENABLE_SLOW_TEST: '' ENABLE_DOCS_TEST: '' ENABLE_BACKWARDS_COMPAT_TEST: '' + ENABLE_XLA_TEST: '' + ENABLE_NOARCH_TEST: '' NUM_TEST_SHARDS: 2 MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu NOGPU_RUNNER_TYPE: linux.2xlarge @@ -420,6 +424,7 @@ jobs: -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ diff --git a/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml index 2d0d916..656b5654 100644 --- a/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml @@ -16,6 +16,7 @@ env: BUILD_ENVIRONMENT: linux-xenial-cuda10.2-py3.6-gcc7 DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -192,6 +193,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ @@ -282,6 +284,8 @@ jobs: ENABLE_SLOW_TEST: 1 ENABLE_DOCS_TEST: '' ENABLE_BACKWARDS_COMPAT_TEST: '' + ENABLE_XLA_TEST: '' + ENABLE_NOARCH_TEST: '' NUM_TEST_SHARDS: 2 MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu NOGPU_RUNNER_TYPE: linux.2xlarge @@ -420,6 +424,7 @@ jobs: -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ diff --git a/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml index 0e1c7ba..68b4285 100644 --- a/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml @@ -16,6 +16,7 @@ env: BUILD_ENVIRONMENT: linux-xenial-cuda11.3-py3.6-gcc7 DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda11.3-cudnn8-py3-gcc7 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -192,6 +193,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ @@ -282,6 +284,8 @@ jobs: ENABLE_SLOW_TEST: '' ENABLE_DOCS_TEST: '' ENABLE_BACKWARDS_COMPAT_TEST: '' + ENABLE_XLA_TEST: '' + ENABLE_NOARCH_TEST: '' NUM_TEST_SHARDS: 2 MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu NOGPU_RUNNER_TYPE: linux.2xlarge @@ -420,6 +424,7 @@ jobs: -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml index 6f4e5c2..b347b52 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml @@ -16,6 +16,7 @@ env: BUILD_ENVIRONMENT: linux-xenial-py3.6-gcc5.4 DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -192,6 +193,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ @@ -282,6 +284,8 @@ jobs: ENABLE_SLOW_TEST: '' ENABLE_DOCS_TEST: 1 ENABLE_BACKWARDS_COMPAT_TEST: 1 + ENABLE_XLA_TEST: '' + ENABLE_NOARCH_TEST: '' NUM_TEST_SHARDS: 2 MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu NOGPU_RUNNER_TYPE: linux.2xlarge @@ -420,6 +424,7 @@ jobs: -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml index 2d72fab..e86cc56 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml @@ -16,6 +16,7 @@ env: BUILD_ENVIRONMENT: linux-xenial-py3.6-gcc7-bazel-test DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-bionic-cuda10.2-cudnn7-py3.9-gcc7 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed diff --git a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml index b009d77..b591519 100644 --- a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml @@ -14,6 +14,7 @@ env: BUILD_ENVIRONMENT: periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7 DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -190,6 +191,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ diff --git a/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml b/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml index 738e6bb..07593ad 100644 --- a/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml @@ -14,6 +14,7 @@ env: BUILD_ENVIRONMENT: periodic-linux-xenial-cuda11.1-py3.6-gcc7 DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7 SCCACHE_BUCKET: ossci-compiler-cache-circleci-v2 + XLA_CLANG_CACHE_S3_BUCKET_NAME: ossci-compiler-clang-cache-circleci-xla TORCH_CUDA_ARCH_LIST: 5.2 IN_CI: 1 # This is used for the phase of adding wheel tests only, will be removed once completed @@ -190,6 +191,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e SKIP_SCCACHE_INITIALIZATION=1 \ -e TORCH_CUDA_ARCH_LIST \ @@ -280,6 +282,8 @@ jobs: ENABLE_SLOW_TEST: '' ENABLE_DOCS_TEST: '' ENABLE_BACKWARDS_COMPAT_TEST: '' + ENABLE_XLA_TEST: '' + ENABLE_NOARCH_TEST: '' NUM_TEST_SHARDS: 2 MULTIGPU_RUNNER_TYPE: linux.16xlarge.nvidia.gpu NOGPU_RUNNER_TYPE: linux.2xlarge @@ -418,6 +422,7 @@ jobs: -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ + -e XLA_CLANG_CACHE_S3_BUCKET_NAME \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ -- 2.7.4