From: Eli Uriegas Date: Wed, 8 Sep 2021 17:51:29 +0000 (-0700) Subject: .github: Run docker containers in detach mode (#64459) X-Git-Tag: accepted/tizen/8.0/unified/20231005.095509~373 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=deb9775c07e39ac639daace2ef902ee7ecb550c8;p=platform%2Fupstream%2Fpytorch.git .github: Run docker containers in detach mode (#64459) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/64459 Should allow users to exec into the docker container if using with-ssh, even if the build / test command has finished executing Signed-off-by: Eli Uriegas Test Plan: Imported from OSS Reviewed By: malfet Differential Revision: D30742797 Pulled By: seemethere fbshipit-source-id: 969ed8799216c6051439c7d41ab709b2d40938ac --- diff --git a/.github/templates/bazel_ci_workflow.yml.j2 b/.github/templates/bazel_ci_workflow.yml.j2 index 9bddb89..20d33c3 100644 --- a/.github/templates/bazel_ci_workflow.yml.j2 +++ b/.github/templates/bazel_ci_workflow.yml.j2 @@ -56,7 +56,8 @@ on: env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -70,11 +71,13 @@ on: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/build.sh' !{{ common.parse_ref() }} - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update @@ -94,14 +97,12 @@ on: python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Test PyTorch run: | + # detached container should get cleaned up by teardown_ec2_linux export SHARD_NUMBER=0 # TODO: Stop building test binaries as part of the build phase - # Used for GPU_FLAG since that doesn't play nice - # shellcheck disable=SC2086 # Make sure we copy test results from bazel-testlogs symlink to # a regular directory ./test/test-reports - docker run \ - ${GPU_FLAG:-} \ + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e GITHUB_ACTIONS \ @@ -118,11 +119,13 @@ on: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/test.sh && cp -Lr ./bazel-testlogs ./test/test-reports' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/test.sh && cp -Lr ./bazel-testlogs ./test/test-reports' - name: Chown workspace if: always() run: | diff --git a/.github/templates/linux_ci_workflow.yml.j2 b/.github/templates/linux_ci_workflow.yml.j2 index f704a32..ca7ca55 100644 --- a/.github/templates/linux_ci_workflow.yml.j2 +++ b/.github/templates/linux_ci_workflow.yml.j2 @@ -135,7 +135,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -150,11 +151,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' !{{ common.parse_ref() }} - name: Display and upload binary build size statistics (Click Me) # temporary hack: set CIRCLE_* vars, until we update @@ -298,10 +301,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -325,11 +329,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | @@ -403,7 +410,8 @@ jobs: echo "${GITHUB_REF}" ref=${GITHUB_REF##*/} target=${ref//v} - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e IN_CI \ @@ -415,13 +423,14 @@ jobs: --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ - --name="$GITHUB_SHA" \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/${DOCS_TYPE}_doc_push_script.sh" + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/${DOCS_TYPE}_doc_push_script.sh" - name: Chown workspace run: | # Ensure the working directory gets chowned back to the current user diff --git a/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml b/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml index b3ab196..5f0dd95 100644 --- a/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml +++ b/.github/workflows/generated-libtorch-linux-xenial-cuda10.2-py3.6-gcc7.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py diff --git a/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml b/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml index 50622fe..202700b 100644 --- a/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml +++ b/.github/workflows/generated-libtorch-linux-xenial-cuda11.3-py3.6-gcc7.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py diff --git a/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml b/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml index 9c2b194..b409bc1 100644 --- a/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml +++ b/.github/workflows/generated-linux-bionic-cuda10.2-py3.9-gcc7.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -407,10 +410,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -434,11 +438,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-linux-bionic-py3.6-clang9.yml b/.github/workflows/generated-linux-bionic-py3.6-clang9.yml index b5856ed..adbd6cb 100644 --- a/.github/workflows/generated-linux-bionic-py3.6-clang9.yml +++ b/.github/workflows/generated-linux-bionic-py3.6-clang9.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -407,10 +410,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -434,11 +438,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml index eceab58..3abc455 100644 --- a/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml +++ b/.github/workflows/generated-linux-bionic-py3.8-gcc9-coverage.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -407,10 +410,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -434,11 +438,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml index 288288e..7e3b495 100644 --- a/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda10.2-py3.6-gcc7.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -407,10 +410,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -434,11 +438,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml b/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml index 15bee85..fc7651f 100644 --- a/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml +++ b/.github/workflows/generated-linux-xenial-cuda11.3-py3.6-gcc7.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -407,10 +410,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -434,11 +438,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml index 7a17146..f239f96 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -407,10 +410,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -434,11 +438,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | @@ -577,7 +584,8 @@ jobs: echo "${GITHUB_REF}" ref=${GITHUB_REF##*/} target=${ref//v} - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e IN_CI \ @@ -589,13 +597,14 @@ jobs: --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ - --name="$GITHUB_SHA" \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/${DOCS_TYPE}_doc_push_script.sh" + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/${DOCS_TYPE}_doc_push_script.sh" - name: Chown workspace run: | # Ensure the working directory gets chowned back to the current user diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml index f0c142c..a36e9aa 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc7-bazel-test.yml @@ -213,7 +213,8 @@ jobs: env | grep '^GITHUB' > "/tmp/github_env_${GITHUB_RUN_ID}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -227,11 +228,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -253,14 +256,12 @@ jobs: python3 -m tools.stats.upload_binary_size_to_scuba || exit 0 - name: Test PyTorch run: | + # detached container should get cleaned up by teardown_ec2_linux export SHARD_NUMBER=0 # TODO: Stop building test binaries as part of the build phase - # Used for GPU_FLAG since that doesn't play nice - # shellcheck disable=SC2086 # Make sure we copy test results from bazel-testlogs symlink to # a regular directory ./test/test-reports - docker run \ - ${GPU_FLAG:-} \ + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e CUSTOM_TEST_ARTIFACT_BUILD_DIR \ -e GITHUB_ACTIONS \ @@ -277,11 +278,13 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/test.sh && cp -Lr ./bazel-testlogs ./test/test-reports' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && sudo chown -R jenkins /dev && .jenkins/pytorch/test.sh && cp -Lr ./bazel-testlogs ./test/test-reports' - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-parallelnative-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-parallelnative-linux-xenial-py3.6-gcc5.4.yml index 8abf8d6..ecca773 100644 --- a/.github/workflows/generated-parallelnative-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-parallelnative-linux-xenial-py3.6-gcc5.4.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -407,10 +410,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -434,11 +438,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-paralleltbb-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-paralleltbb-linux-xenial-py3.6-gcc5.4.yml index 25b77a4..73415ef 100644 --- a/.github/workflows/generated-paralleltbb-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-paralleltbb-linux-xenial-py3.6-gcc5.4.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -407,10 +410,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -434,11 +438,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml index 1e6ad8b..bda48eb 100644 --- a/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-libtorch-linux-xenial-cuda11.1-py3.6-gcc7.yml @@ -190,7 +190,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -205,11 +206,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py diff --git a/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml b/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml index 5490714..d6f40c2 100644 --- a/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml +++ b/.github/workflows/generated-periodic-linux-xenial-cuda11.1-py3.6-gcc7.yml @@ -190,7 +190,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -205,11 +206,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py @@ -405,10 +408,11 @@ jobs: if [[ $NUM_TEST_SHARDS -ne 2 ]]; then export SHARD_NUMBER=0 fi + # detached container should get cleaned up by teardown_ec2_linux # TODO: Stop building test binaries as part of the build phase # Used for GPU_FLAG since that doesn't play nice # shellcheck disable=SC2086 - docker run \ + container_name=$(docker run \ ${GPU_FLAG:-} \ -e BUILD_ENVIRONMENT \ -e PR_NUMBER \ @@ -432,11 +436,14 @@ jobs: --cap-add=SYS_PTRACE \ --shm-size="${SHM_SIZE}" \ --tty \ + --detach \ + --name="${container_name}" \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && pip install dist/*.whl && '$TEST_COMMAND + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c "sudo chown -R jenkins . && pip install dist/*.whl && ${TEST_COMMAND}" - name: Chown workspace if: always() run: | diff --git a/.github/workflows/generated-puretorch-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-puretorch-linux-xenial-py3.6-gcc5.4.yml index cac4b0a..0699ba9 100644 --- a/.github/workflows/generated-puretorch-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-puretorch-linux-xenial-py3.6-gcc5.4.yml @@ -192,7 +192,8 @@ jobs: docker pull "${DOCKER_IMAGE}" - name: Build PyTorch run: | - docker run \ + # detached container should get cleaned up by teardown_ec2_linux + container_name=$(docker run \ -e BUILD_ENVIRONMENT \ -e JOB_BASE_NAME \ -e MAX_JOBS="$(nproc --ignore=2)" \ @@ -207,11 +208,13 @@ jobs: --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ --tty \ + --detach \ --user jenkins \ -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ - "${DOCKER_IMAGE}" \ - sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' + "${DOCKER_IMAGE}" + ) + docker exec -t "${container_name}" sh -c 'sudo chown -R jenkins . && .jenkins/pytorch/build.sh' - name: Parse ref id: parse-ref run: .github/scripts/parse_ref.py