1name: build docs 2 3on: 4 workflow_call: 5 inputs: 6 build-environment: 7 required: true 8 type: string 9 description: Top-level label for what's being built/tested. 10 docker-image: 11 required: true 12 type: string 13 description: Docker image to run in. 14 push: 15 required: false 16 type: boolean 17 default: false 18 description: If set, push the docs to the docs website. 19 run-doxygen: 20 required: false 21 type: boolean 22 default: false 23 description: If set, will enable C++ API doc generation using doxygen / breathe / exhale. 24 sync-tag: 25 required: false 26 type: string 27 default: "" 28 description: | 29 If this is set, our linter will use this to make sure that every other 30 job with the same `sync-tag` is identical. 31 s3-bucket: 32 description: S3 bucket to download artifact 33 required: false 34 type: string 35 default: "gha-artifacts" 36 aws-role-to-assume: 37 description: role to assume for downloading artifacts 38 required: false 39 type: string 40 default: "" 41 upload-aws-role-to-assume: 42 description: role to assume for downloading artifacts 43 required: false 44 type: string 45 default: "" 46 secrets: 47 GH_PYTORCHBOT_TOKEN: 48 required: false 49 description: Permissions for pushing to the docs site. 50 51jobs: 52 build-docs: 53 # Don't run on forked repos. 54 if: github.repository_owner == 'pytorch' 55 runs-on: ${{ matrix.runner }} 56 environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'pytorchbot-env' || '' }} 57 strategy: 58 fail-fast: false 59 matrix: 60 include: 61 - docs_type: cpp 62 # We recently seeing lots of exit code 137 running this in Docker indicating 63 # an OOM issue when running the job, so this upgrades the runner from 4xlarge 64 # to the next available tier of 12xlarge. So much memory just to generate cpp 65 # doc 66 runner: linux.12xlarge 67 # TODO: Nightly cpp docs take longer and longer to finish (more than 3h now) 68 # Let's try to figure out how this can be improved 69 timeout-minutes: 240 70 - docs_type: python 71 runner: linux.2xlarge 72 # It takes less than 30m to finish python docs unless there are issues 73 timeout-minutes: 30 74 - docs_type: functorch 75 runner: linux.2xlarge 76 # It takes less than 15m to finish functorch docs unless there are issues 77 timeout-minutes: 15 78 # Set a fixed name for this job instead of using the current matrix-generated name, i.e. build-docs (cpp, linux.12xlarge, 180) 79 # The current name requires updating the Rockset last docs push query from test-infra every time the matrix is updated 80 name: build-docs-${{ matrix.docs_type }}-${{ inputs.push }} 81 steps: 82 - name: Setup SSH (Click me for login details) 83 uses: pytorch/test-infra/.github/actions/setup-ssh@release/2.4 84 with: 85 github-secret: ${{ secrets.GITHUB_TOKEN }} 86 instructions: | 87 All builds are done inside the container, to start an interactive session run: 88 docker exec -it $(docker container ps --format '{{.ID}}') bash 89 To start Python docs build type: 90 cd docs && make html && make coverage 91 92 # [see note: pytorch repo ref] 93 - name: Checkout PyTorch 94 uses: pytorch/pytorch/.github/actions/checkout-pytorch@release/2.4 95 96 - name: Setup Linux 97 uses: ./.github/actions/setup-linux 98 99 - name: configure aws credentials 100 if : ${{ inputs.aws-role-to-assume != '' }} 101 uses: aws-actions/configure-aws-credentials@v3 102 with: 103 role-to-assume: ${{ inputs.aws-role-to-assume }} 104 role-session-name: gha-linux-test 105 aws-region: us-east-1 106 107 - name: Calculate docker image 108 id: calculate-docker-image 109 uses: pytorch/test-infra/.github/actions/calculate-docker-image@release/2.4 110 with: 111 docker-image-name: ${{ inputs.docker-image }} 112 113 - name: Pull docker image 114 uses: pytorch/test-infra/.github/actions/pull-docker-image@release/2.4 115 with: 116 docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }} 117 118 - name: Download build artifacts 119 uses: ./.github/actions/download-build-artifacts 120 with: 121 name: ${{ inputs.build-environment }} 122 s3-bucket: ${{ inputs.s3-bucket }} 123 124 - name: Generate netrc (only for docs-push) 125 if: inputs.push 126 env: 127 GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }} 128 run: | 129 # sometimes .netrc exists as a directory even though this is the temp folder 130 rm -rf "${RUNNER_TEMP}/.netrc" 131 # set credentials for https pushing 132 echo "machine github.com" > "${RUNNER_TEMP}/.netrc" 133 echo "login pytorchbot" >> "${RUNNER_TEMP}/.netrc" 134 echo "password ${GITHUB_PYTORCHBOT_TOKEN}" >> "${RUNNER_TEMP}/.netrc" 135 136 - name: Build ${{ matrix.docs_type }} docs 137 timeout-minutes: ${{ matrix.timeout-minutes }} 138 id: build-docs 139 env: 140 # After https://github.com/pytorch/pytorch/pull/88373, pull workflow can now be run periodically, 141 # so using a schedule event to determine if the docs should be pushed or not doesn't hold true 142 # anymore 143 WITH_PUSH: ${{ inputs.push }} 144 DOCKER_IMAGE: ${{ inputs.docker-image }} 145 DOCS_TYPE: ${{ matrix.docs_type }} 146 RUN_DOXYGEN: ${{ inputs.run-doxygen }} 147 BUILD_ENVIRONMENT: ${{ inputs.build-environment }} 148 run: | 149 set -ex 150 # Convert refs/tags/v1.12.0rc3 into 1.12 151 if [[ "${GITHUB_REF}" =~ ^refs/tags/v([0-9]+\.[0-9]+)\.* ]]; then 152 target="${BASH_REMATCH[1]}" 153 else 154 target="main" 155 fi 156 # detached container should get cleaned up by teardown_ec2_linux 157 container_name=$(docker run \ 158 -e BUILD_ENVIRONMENT \ 159 -e MAX_JOBS="$(nproc --ignore=2)" \ 160 -e SHA1="$GITHUB_SHA" \ 161 -e DOCS_VERSION="${target}" \ 162 -e DOCS_TYPE \ 163 -e RUN_DOXYGEN \ 164 -e WITH_PUSH \ 165 --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \ 166 --security-opt seccomp=unconfined \ 167 --cap-add=SYS_PTRACE \ 168 --tty \ 169 --detach \ 170 --user jenkins \ 171 -v "${RUNNER_TEMP}/.netrc":/var/lib/jenkins/.netrc \ 172 -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \ 173 -w /var/lib/jenkins/workspace \ 174 "${DOCKER_IMAGE}" 175 ) 176 docker exec -t "${container_name}" bash -c "sudo chown -R jenkins . && pip install $(echo dist/*.whl)[opt-einsum] && ./.ci/pytorch/${DOCS_TYPE}_doc_push_script.sh" 177 178 - name: Chown workspace 179 uses: ./.github/actions/chown-workspace 180 if: always() 181 182 - name: configure aws credentials 183 if : ${{ inputs.upload-aws-role-to-assume != '' }} 184 uses: aws-actions/configure-aws-credentials@v3 185 with: 186 role-to-assume: ${{ inputs.upload-aws-role-to-assume }} 187 role-session-name: gha-linux-test 188 aws-region: us-east-1 189 190 - name: Upload Python Docs Preview 191 uses: seemethere/upload-artifact-s3@v5 192 if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'python' && steps.build-docs.outcome == 'success' }} 193 with: 194 retention-days: 14 195 s3-bucket: doc-previews 196 if-no-files-found: error 197 path: pytorch_docs/main/ 198 s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }} 199 200 - name: Upload C++ Docs Preview 201 uses: seemethere/upload-artifact-s3@v5 202 if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }} 203 with: 204 retention-days: 14 205 if-no-files-found: error 206 s3-bucket: doc-previews 207 path: cppdocs/ 208 s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}/cppdocs 209 210 - name: Upload functorch Docs Preview 211 uses: seemethere/upload-artifact-s3@v5 212 if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'functorch' && steps.build-docs.outcome == 'success' }} 213 with: 214 retention-days: 14 215 s3-bucket: doc-previews 216 if-no-files-found: error 217 path: functorch_ghpages/nightly/ 218 s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}/functorchdocs 219 220 - name: Teardown Linux 221 uses: pytorch/test-infra/.github/actions/teardown-linux@release/2.4 222 if: always() 223