From a5f35ac7cd12c32227fdcd42979dc9c6aea7ba07 Mon Sep 17 00:00:00 2001 From: Jane Xu Date: Thu, 26 Aug 2021 17:36:56 -0700 Subject: [PATCH] Run through failures on trunk (#64063) Summary: This PR runs all the tests on trunk instead of stopping on first failure. Pull Request resolved: https://github.com/pytorch/pytorch/pull/64063 Reviewed By: malfet, seemethere Differential Revision: D30592020 Pulled By: janeyx99 fbshipit-source-id: 318b225cdf918a98f73e752d1cc0227d9227f36c --- .github/templates/bazel_ci_workflow.yml.j2 | 2 ++ .github/templates/linux_ci_workflow.yml.j2 | 2 ++ .github/templates/windows_ci_workflow.yml.j2 | 1 + .github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml | 2 ++ .github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml | 2 ++ .github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml | 2 ++ .github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml | 2 ++ .github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml | 2 ++ .github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml | 2 ++ .../workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml | 2 ++ .github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml | 1 + .github/workflows/generated-win-vs2019-cpu-py3.yml | 1 + .github/workflows/generated-win-vs2019-cuda10.1-py3.yml | 1 + .github/workflows/generated-win-vs2019-cuda11.3-py3.yml | 1 + 14 files changed, 23 insertions(+) diff --git a/.github/templates/bazel_ci_workflow.yml.j2 b/.github/templates/bazel_ci_workflow.yml.j2 index d25ffe6..f4e0034 100644 --- a/.github/templates/bazel_ci_workflow.yml.j2 +++ b/.github/templates/bazel_ci_workflow.yml.j2 @@ -29,6 +29,7 @@ on: DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }} JOB_BASE_NAME: !{{ build_environment }}-build-and-test NUM_TEST_SHARDS: !{{ num_test_shards }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -121,6 +122,7 @@ on: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e CONTINUE_THROUGH_ERROR \ -e http_proxy="!{{squid_proxy}}" -e https_proxy="!{{squid_proxy}}" -e no_proxy="!{{squid_no_proxy}}" \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ diff --git a/.github/templates/linux_ci_workflow.yml.j2 b/.github/templates/linux_ci_workflow.yml.j2 index d9af899..52c0a09 100644 --- a/.github/templates/linux_ci_workflow.yml.j2 +++ b/.github/templates/linux_ci_workflow.yml.j2 @@ -286,6 +286,7 @@ jobs: SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -372,6 +373,7 @@ jobs: -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PYTORCH_IGNORE_DISABLED_ISSUES \ + -e CONTINUE_THROUGH_ERROR \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="!{{squid_proxy}}" -e https_proxy="!{{squid_proxy}}" -e no_proxy="!{{squid_no_proxy}}" \ diff --git a/.github/templates/windows_ci_workflow.yml.j2 b/.github/templates/windows_ci_workflow.yml.j2 index 1be7b32..84a30bd 100644 --- a/.github/templates/windows_ci_workflow.yml.j2 +++ b/.github/templates/windows_ci_workflow.yml.j2 @@ -195,6 +195,7 @@ jobs: https_proxy: "!{{ squid_proxy }}" RUN_SMOKE_TESTS_ONLY_ON_PR: !{{ only_run_smoke_tests_on_pull_request }} PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} needs: [build, generate-test-matrix, !{{ ciflow_config.root_job_name }}] strategy: matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }} diff --git a/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml b/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml index 769efca..c51f8f0 100644 --- a/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml +++ b/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml @@ -262,6 +262,7 @@ jobs: SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -351,6 +352,7 @@ jobs: -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PYTORCH_IGNORE_DISABLED_ISSUES \ + -e CONTINUE_THROUGH_ERROR \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ diff --git a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml index ddd81c0..014b1d1 100644 --- a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml +++ b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml @@ -262,6 +262,7 @@ jobs: SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -351,6 +352,7 @@ jobs: -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PYTORCH_IGNORE_DISABLED_ISSUES \ + -e CONTINUE_THROUGH_ERROR \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ diff --git a/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml index 5a888d0..76b973e 100644 --- a/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml @@ -262,6 +262,7 @@ jobs: SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -351,6 +352,7 @@ jobs: -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PYTORCH_IGNORE_DISABLED_ISSUES \ + -e CONTINUE_THROUGH_ERROR \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ diff --git a/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml index 25d74de..8114bd5 100644 --- a/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml @@ -262,6 +262,7 @@ jobs: SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -351,6 +352,7 @@ jobs: -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PYTORCH_IGNORE_DISABLED_ISSUES \ + -e CONTINUE_THROUGH_ERROR \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml index 341f9e6..b5f062c 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml @@ -262,6 +262,7 @@ jobs: SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -351,6 +352,7 @@ jobs: -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PYTORCH_IGNORE_DISABLED_ISSUES \ + -e CONTINUE_THROUGH_ERROR \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml index 7ca3896..71a9bf7 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml @@ -123,6 +123,7 @@ jobs: DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }} JOB_BASE_NAME: linux-xenial-py3.6-gcc7-bazel-test-build-and-test NUM_TEST_SHARDS: 1 + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -220,6 +221,7 @@ jobs: -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ + -e CONTINUE_THROUGH_ERROR \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ diff --git a/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml b/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml index 470fdaa..7b94779 100644 --- a/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml @@ -260,6 +260,7 @@ jobs: SHARD_NUMBER: ${{ matrix.shard }} NUM_TEST_SHARDS: ${{ matrix.num_shards }} PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} steps: - name: Log in to ECR run: | @@ -349,6 +350,7 @@ jobs: -e TEST_CONFIG \ -e NUM_TEST_SHARDS \ -e PYTORCH_IGNORE_DISABLED_ISSUES \ + -e CONTINUE_THROUGH_ERROR \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e SCCACHE_BUCKET \ -e http_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e https_proxy="http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" -e no_proxy="localhost,127.0.0.1,amazonaws.com,s3.amazonaws.com,169.254.169.254,169.254.170.2,/var/run/docker.sock" \ diff --git a/.github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml b/.github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml index ce4540b..6c87f40 100644 --- a/.github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml +++ b/.github/workflows/generated-periodic-win-vs2019-cuda11.1-py3.yml @@ -158,6 +158,7 @@ jobs: https_proxy: "http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" RUN_SMOKE_TESTS_ONLY_ON_PR: False PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} needs: [build, generate-test-matrix, ciflow_should_run] strategy: matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }} diff --git a/.github/workflows/generated-win-vs2019-cpu-py3.yml b/.github/workflows/generated-win-vs2019-cpu-py3.yml index d868d19..30f328a 100644 --- a/.github/workflows/generated-win-vs2019-cpu-py3.yml +++ b/.github/workflows/generated-win-vs2019-cpu-py3.yml @@ -150,6 +150,7 @@ jobs: https_proxy: "http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" RUN_SMOKE_TESTS_ONLY_ON_PR: False PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} needs: [build, generate-test-matrix, ciflow_should_run] strategy: matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }} diff --git a/.github/workflows/generated-win-vs2019-cuda10.1-py3.yml b/.github/workflows/generated-win-vs2019-cuda10.1-py3.yml index 4d4550c..72dd21d 100644 --- a/.github/workflows/generated-win-vs2019-cuda10.1-py3.yml +++ b/.github/workflows/generated-win-vs2019-cuda10.1-py3.yml @@ -160,6 +160,7 @@ jobs: https_proxy: "http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" RUN_SMOKE_TESTS_ONLY_ON_PR: True PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} needs: [build, generate-test-matrix, ciflow_should_run] strategy: matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }} diff --git a/.github/workflows/generated-win-vs2019-cuda11.3-py3.yml b/.github/workflows/generated-win-vs2019-cuda11.3-py3.yml index c5ae48a..eb6e02f 100644 --- a/.github/workflows/generated-win-vs2019-cuda11.3-py3.yml +++ b/.github/workflows/generated-win-vs2019-cuda11.3-py3.yml @@ -160,6 +160,7 @@ jobs: https_proxy: "http://internal-tf-lb-20210727220640487900000002-835786077.us-east-1.elb.amazonaws.com:3128" RUN_SMOKE_TESTS_ONLY_ON_PR: False PYTORCH_IGNORE_DISABLED_ISSUES: ${{ needs.generate-test-matrix.outputs.ignore-disabled-issues }} + CONTINUE_THROUGH_ERROR: ${{ github.repository_owner == 'pytorch' && (github.event_name == 'push' || github.event_name == 'schedule') }} needs: [build, generate-test-matrix, ciflow_should_run] strategy: matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }} -- 2.7.4