From 3abbcf079d38d468a45073b13cb13627c9c0f367 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Fri, 27 Aug 2021 16:02:49 -0700 Subject: [PATCH] .github: Add cpp_docs job to current gcc5 workflow (#64044) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/64044 Adds the cpp_docs job to the current workflow, also modifies the scripts surrounding building docs so that they can be powered through environment variables with sane defaults rather than having to have passed arguments. Ideally should not break current jobs running in circleci but those should eventually be turned off anyways. Coincides with work from: * https://github.com/seemethere/upload-artifact-s3/pull/1 * https://github.com/seemethere/upload-artifact-s3/pull/2 Signed-off-by: Eli Uriegas cc ezyang seemethere malfet walterddr lg20987 pytorch/pytorch-dev-infra Test Plan: Imported from OSS Reviewed By: malfet Differential Revision: D30610010 Pulled By: seemethere fbshipit-source-id: f67adeb1bd422bb9e24e0f1ec0098cf9c648f283 --- .circleci/scripts/cpp_doc_push_script.sh | 25 ++++++++----- .circleci/scripts/python_doc_push_script.sh | 27 +++++++++----- .github/templates/linux_ci_workflow.yml.j2 | 42 +++++++++++++--------- .../generated-linux-xenial-py3.6-gcc5.4.yml | 42 +++++++++++++--------- 4 files changed, 85 insertions(+), 51 deletions(-) diff --git a/.circleci/scripts/cpp_doc_push_script.sh b/.circleci/scripts/cpp_doc_push_script.sh index c6b4f00..aa26a74 100755 --- a/.circleci/scripts/cpp_doc_push_script.sh +++ b/.circleci/scripts/cpp_doc_push_script.sh @@ -10,21 +10,30 @@ pt_checkout="/var/lib/jenkins/workspace" # Since we're cat-ing this file, we need to escape all $'s echo "cpp_doc_push_script.sh: Invoked with $*" +# for statements like ${1:-${DOCS_INSTALL_PATH:-docs/}} +# the order of operations goes: +# 1. Check if there's an argument $1 +# 2. If no argument check for environment var DOCS_INSTALL_PATH +# 3. If no environment var fall back to default 'docs/' + +# NOTE: It might seem weird to gather the second argument before gathering the first argument +# but since DOCS_INSTALL_PATH can be derived from DOCS_VERSION it's probably better to +# try and gather it first, just so we don't potentially break people who rely on this script +# Argument 2: What version of the Python API docs we are building. +version="${2:-${DOCS_VERSION:-master}}" +if [ -z "$version" ]; then +echo "error: cpp_doc_push_script.sh: version (arg2) not specified" + exit 1 +fi + # Argument 1: Where to copy the built documentation for Python API to # (pytorch.github.io/$install_path) -install_path="$1" +install_path="${1:-${DOCS_INSTALL_PATH:-docs/${DOCS_VERSION}}}" if [ -z "$install_path" ]; then echo "error: cpp_doc_push_script.sh: install_path (arg1) not specified" exit 1 fi -# Argument 2: What version of the Python API docs we are building. -version="$2" -if [ -z "$version" ]; then -echo "error: cpp_doc_push_script.sh: version (arg2) not specified" - exit 1 -fi - is_master_doc=false if [ "$version" == "master" ]; then is_master_doc=true diff --git a/.circleci/scripts/python_doc_push_script.sh b/.circleci/scripts/python_doc_push_script.sh index ed8f28d..167eaca 100755 --- a/.circleci/scripts/python_doc_push_script.sh +++ b/.circleci/scripts/python_doc_push_script.sh @@ -13,28 +13,37 @@ echo "python_doc_push_script.sh: Invoked with $*" set -ex +# for statements like ${1:-${DOCS_INSTALL_PATH:-docs/}} +# the order of operations goes: +# 1. Check if there's an argument $1 +# 2. If no argument check for environment var DOCS_INSTALL_PATH +# 3. If no environment var fall back to default 'docs/' + +# NOTE: It might seem weird to gather the second argument before gathering the first argument +# but since DOCS_INSTALL_PATH can be derived from DOCS_VERSION it's probably better to +# try and gather it first, just so we don't potentially break people who rely on this script +# Argument 2: What version of the docs we are building. +version="${2:-${DOCS_VERSION:-master}}" +if [ -z "$version" ]; then +echo "error: python_doc_push_script.sh: version (arg2) not specified" + exit 1 +fi + # Argument 1: Where to copy the built documentation to # (pytorch.github.io/$install_path) -install_path="$1" +install_path="${1:-${DOCS_INSTALL_PATH:-docs/${DOCS_VERSION}}}" if [ -z "$install_path" ]; then echo "error: python_doc_push_script.sh: install_path (arg1) not specified" exit 1 fi -# Argument 2: What version of the docs we are building. -version="$2" -if [ -z "$version" ]; then -echo "error: python_doc_push_script.sh: version (arg2) not specified" - exit 1 -fi - is_master_doc=false if [ "$version" == "master" ]; then is_master_doc=true fi # Argument 3: The branch to push to. Usually is "site" -branch="$3" +branch="${3:-${DOCS_BRANCH:-site}}" if [ -z "$branch" ]; then echo "error: python_doc_push_script.sh: branch (arg3) not specified" exit 1 diff --git a/.github/templates/linux_ci_workflow.yml.j2 b/.github/templates/linux_ci_workflow.yml.j2 index 52c0a09..7d90207 100644 --- a/.github/templates/linux_ci_workflow.yml.j2 +++ b/.github/templates/linux_ci_workflow.yml.j2 @@ -441,11 +441,15 @@ jobs: {% endblock %} {%- endif -%} {%- if enable_doc_jobs %} - pytorch_python_doc_build: + pytorch_doc_build: runs-on: linux.2xlarge + strategy: + matrix: + docs_type: [cpp, python] needs: [calculate-docker-image, build, !{{ ciflow_config.root_job_name }}] env: DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }} + DOCS_TYPE: ${{ matrix.docs_type }} steps: - name: Log in to ECR run: | @@ -483,7 +487,7 @@ jobs: - name: Unzip artifacts run: | unzip -o artifacts.zip - - name: Build Python Doc in Docker + - name: Build ${{ matrix.docs_type }} docs run: | set -ex time docker pull "${DOCKER_IMAGE}" > /dev/null @@ -496,6 +500,8 @@ jobs: -e IN_CI \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e CIRCLE_SHA1="$GITHUB_SHA" \ + -e DOCS_VERSION="${target}" \ + -e DOCS_TYPE \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ @@ -505,34 +511,36 @@ jobs: -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ "${DOCKER_IMAGE}" \ - bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/python_doc_push_script.sh docs/$target $target site" + bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/${DOCS_TYPE}_doc_push_script.sh" - name: Chown workspace run: | # Ensure the working directory gets chowned back to the current user docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . - - uses: driazati/upload-artifact-s3@21c31d0a7bcb056ca50bd6ce197ba6507c26a1be - if: ${{ github.event_name == 'pull_request' }} - name: Upload Docs Preview + - uses: seemethere/upload-artifact-s3@v3 + name: Upload Python Docs Preview + if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'python' }} with: - name: deploy retention-days: 14 if-no-files-found: error - path: pytorch.github.io/docs/merge - - name: Show Docs Preview URL (Click Me) - if: ${{ github.event_name == 'pull_request' }} - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - echo "See rendered docs at https://docs-preview.pytorch.org/$PR_NUMBER/" + path: pytorch.github.io/docs/merge/ + s3-prefix: ${{ github.repository }}/pr-previews/pr/${{ github.event.pull_request.number }} + - uses: seemethere/upload-artifact-s3@v3 + name: Upload C++ Docs Preview + if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cppdocs' }} + with: + retention-days: 14 + if-no-files-found: error + path: cppdocs/ + s3-prefix: ${{ github.repository }}/pr-previews/pr/${{ github.event.pull_request.number }}/cppdocs - name: Archive artifacts into zip run: | - zip -r pytorch_github_io.zip "${GITHUB_WORKSPACE}/pytorch.github.io" + zip -r "docs_${DOCS_TYPE}.zip" "${GITHUB_WORKSPACE}/pytorch.github.io" "${GITHUB_WORKSPACE}/cppdocs" - uses: actions/upload-artifact@v2 name: Store PyTorch Build Artifacts with: - name: pytorch_github_io + name: docs_${{ matrix.docs_type }} + path: docs_${{ matrix.docs_type }}.zip if-no-files-found: error - path: pytorch_github_io.zip - name: Hold runner for 2 hours or until ssh sessions have drained # Always hold for active ssh sessions if: always() diff --git a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml index b5f062c..6cc391b 100644 --- a/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml +++ b/.github/workflows/generated-linux-xenial-py3.6-gcc5.4.yml @@ -429,11 +429,15 @@ jobs: # Prune all of the docker images docker system prune -af - pytorch_python_doc_build: + pytorch_doc_build: runs-on: linux.2xlarge + strategy: + matrix: + docs_type: [cpp, python] needs: [calculate-docker-image, build, ciflow_should_run] env: DOCKER_IMAGE: ${{ needs.calculate-docker-image.outputs.docker_image }} + DOCS_TYPE: ${{ matrix.docs_type }} steps: - name: Log in to ECR run: | @@ -474,7 +478,7 @@ jobs: - name: Unzip artifacts run: | unzip -o artifacts.zip - - name: Build Python Doc in Docker + - name: Build ${{ matrix.docs_type }} docs run: | set -ex time docker pull "${DOCKER_IMAGE}" > /dev/null @@ -487,6 +491,8 @@ jobs: -e IN_CI \ -e MAX_JOBS="$(nproc --ignore=2)" \ -e CIRCLE_SHA1="$GITHUB_SHA" \ + -e DOCS_VERSION="${target}" \ + -e DOCS_TYPE \ --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ --security-opt seccomp=unconfined \ --cap-add=SYS_PTRACE \ @@ -496,34 +502,36 @@ jobs: -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ -w /var/lib/jenkins/workspace \ "${DOCKER_IMAGE}" \ - bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/python_doc_push_script.sh docs/$target $target site" + bash -c "sudo chown -R jenkins . && pip install dist/*.whl && ./.circleci/scripts/${DOCS_TYPE}_doc_push_script.sh" - name: Chown workspace run: | # Ensure the working directory gets chowned back to the current user docker run --rm -v "$(pwd)":/v -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . - - uses: driazati/upload-artifact-s3@21c31d0a7bcb056ca50bd6ce197ba6507c26a1be - if: ${{ github.event_name == 'pull_request' }} - name: Upload Docs Preview + - uses: seemethere/upload-artifact-s3@v3 + name: Upload Python Docs Preview + if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'python' }} with: - name: deploy retention-days: 14 if-no-files-found: error - path: pytorch.github.io/docs/merge - - name: Show Docs Preview URL (Click Me) - if: ${{ github.event_name == 'pull_request' }} - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - echo "See rendered docs at https://docs-preview.pytorch.org/$PR_NUMBER/" + path: pytorch.github.io/docs/merge/ + s3-prefix: ${{ github.repository }}/pr-previews/pr/${{ github.event.pull_request.number }} + - uses: seemethere/upload-artifact-s3@v3 + name: Upload C++ Docs Preview + if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cppdocs' }} + with: + retention-days: 14 + if-no-files-found: error + path: cppdocs/ + s3-prefix: ${{ github.repository }}/pr-previews/pr/${{ github.event.pull_request.number }}/cppdocs - name: Archive artifacts into zip run: | - zip -r pytorch_github_io.zip "${GITHUB_WORKSPACE}/pytorch.github.io" + zip -r "docs_${DOCS_TYPE}.zip" "${GITHUB_WORKSPACE}/pytorch.github.io" "${GITHUB_WORKSPACE}/cppdocs" - uses: actions/upload-artifact@v2 name: Store PyTorch Build Artifacts with: - name: pytorch_github_io + name: docs_${{ matrix.docs_type }} + path: docs_${{ matrix.docs_type }}.zip if-no-files-found: error - path: pytorch_github_io.zip - name: Hold runner for 2 hours or until ssh sessions have drained # Always hold for active ssh sessions if: always() -- 2.7.4