1name: linux-binary-build 2 3on: 4 workflow_call: 5 inputs: 6 build_name: 7 required: true 8 type: string 9 description: The build's name 10 build_environment: 11 required: true 12 type: string 13 description: The build environment 14 runs_on: 15 required: false 16 default: linux.12xlarge.ephemeral 17 type: string 18 description: Hardware to run this "build"job on, linux.12xlarge or linux.arm64.2xlarge. 19 timeout-minutes: 20 required: false 21 default: 210 22 type: number 23 description: timeout for the job 24 ALPINE_IMAGE: 25 required: false 26 type: string 27 default: "308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine" 28 description: Alpine image to use 29 PYTORCH_ROOT: 30 required: true 31 type: string 32 description: Root directory for the pytorch/pytorch repository 33 BUILDER_ROOT: 34 required: true 35 type: string 36 description: Root directory for the pytorch/builder repository 37 PACKAGE_TYPE: 38 required: true 39 type: string 40 description: Package type 41 DESIRED_CUDA: 42 required: true 43 type: string 44 description: Desired Cuda version 45 GPU_ARCH_VERSION: 46 required: false 47 type: string 48 description: GPU Arch version 49 GPU_ARCH_TYPE: 50 required: true 51 type: string 52 description: GPU Arch type 53 DOCKER_IMAGE: 54 required: true 55 type: string 56 description: Docker image to use 57 LIBTORCH_CONFIG: 58 required: false 59 type: string 60 description: Desired libtorch config (for libtorch builds only) 61 LIBTORCH_VARIANT: 62 required: false 63 type: string 64 description: Desired libtorch variant (for libtorch builds only) 65 DESIRED_DEVTOOLSET: 66 required: false 67 type: string 68 description: Desired dev toolset 69 DESIRED_PYTHON: 70 required: false 71 type: string 72 description: Desired python version 73 PYTORCH_EXTRA_INSTALL_REQUIREMENTS: 74 required: false 75 type: string 76 description: Extra install requirements 77 default: "" 78 secrets: 79 github-token: 80 required: true 81 description: Github Token 82 83jobs: 84 build: 85 runs-on: ${{ inputs.runs_on }} 86 timeout-minutes: ${{ inputs.timeout-minutes }} 87 env: 88 PYTORCH_ROOT: ${{ inputs.PYTORCH_ROOT }} 89 BUILDER_ROOT: ${{ inputs.BUILDER_ROOT }} 90 PACKAGE_TYPE: ${{ inputs.PACKAGE_TYPE }} 91 # TODO: This is a legacy variable that we eventually want to get rid of in 92 # favor of GPU_ARCH_VERSION 93 DESIRED_CUDA: ${{ inputs.DESIRED_CUDA }} 94 GPU_ARCH_VERSION: ${{ inputs.GPU_ARCH_VERSION }} 95 GPU_ARCH_TYPE: ${{ inputs.GPU_ARCH_TYPE }} 96 DOCKER_IMAGE: ${{ inputs.DOCKER_IMAGE }} 97 SKIP_ALL_TESTS: 1 98 LIBTORCH_CONFIG: ${{ inputs.LIBTORCH_CONFIG }} 99 LIBTORCH_VARIANT: ${{ inputs.LIBTORCH_VARIANT }} 100 DESIRED_DEVTOOLSET: ${{ inputs.DESIRED_DEVTOOLSET }} 101 DESIRED_PYTHON: ${{ inputs.DESIRED_PYTHON }} 102 PYTORCH_EXTRA_INSTALL_REQUIREMENTS: ${{ inputs.PYTORCH_EXTRA_INSTALL_REQUIREMENTS }} 103 # Needed for conda builds 104 ALPINE_IMAGE: ${{ inputs.ALPINE_IMAGE }} 105 ANACONDA_USER: pytorch 106 AWS_DEFAULT_REGION: us-east-1 107 BINARY_ENV_FILE: /tmp/env 108 BUILD_ENVIRONMENT: ${{ inputs.build_environment }} 109 GITHUB_TOKEN: ${{ secrets.github-token }} 110 PR_NUMBER: ${{ github.event.pull_request.number }} 111 PYTORCH_FINAL_PACKAGE_DIR: /artifacts 112 SHA1: ${{ github.event.pull_request.head.sha || github.sha }} 113 steps: 114 - name: Make the env permanent during this workflow (but not the secrets) 115 shell: bash 116 run: | 117 { 118 echo "PYTORCH_ROOT=${{ env.PYTORCH_ROOT }}" 119 echo "BUILDER_ROOT=${{ env.BUILDER_ROOT }}" 120 echo "PACKAGE_TYPE=${{ env.PACKAGE_TYPE }}" 121 echo "DESIRED_CUDA=${{ env.DESIRED_CUDA }}" 122 echo "GPU_ARCH_VERSION=${{ env.GPU_ARCH_VERSION }}" 123 echo "GPU_ARCH_TYPE=${{ env.GPU_ARCH_TYPE }}" 124 echo "DOCKER_IMAGE=${{ env.DOCKER_IMAGE }}" 125 echo "SKIP_ALL_TESTS=${{ env.SKIP_ALL_TESTS }}" 126 echo "LIBTORCH_CONFIG=${{ env.LIBTORCH_CONFIG }}" 127 echo "LIBTORCH_VARIANT=${{ env.LIBTORCH_VARIANT }}" 128 echo "DESIRED_DEVTOOLSET=${{ env.DESIRED_DEVTOOLSET }}" 129 echo "DESIRED_PYTHON=${{ env.DESIRED_PYTHON }}" 130 echo "PYTORCH_EXTRA_INSTALL_REQUIREMENTS=${{ env.PYTORCH_EXTRA_INSTALL_REQUIREMENTS }}" 131 echo "ALPINE_IMAGE=${{ env.ALPINE_IMAGE }}" 132 echo "ANACONDA_USER=${{ env.ANACONDA_USER }}" 133 echo "AWS_DEFAULT_REGION=${{ env.AWS_DEFAULT_REGION }}" 134 echo "BINARY_ENV_FILE=${{ env.BINARY_ENV_FILE }}" 135 echo "BUILD_ENVIRONMENT=${{ env.BUILD_ENVIRONMENT }}" 136 echo "BUILD_NAME=${{ env.BUILD_NAME }}" 137 echo "PR_NUMBER=${{ env.PR_NUMBER }}" 138 echo "PYTORCH_FINAL_PACKAGE_DIR=${{ env.PYTORCH_FINAL_PACKAGE_DIR }}" 139 echo "SHA1=${{ env.SHA1 }}" 140 } >> "${GITHUB_ENV} }}" 141 142 - name: List the env 143 shell: bash 144 run: env 145 146 - name: "[FB EMPLOYEES] Enable SSH (Click me for login details)" 147 if: inputs.build_environment != 'linux-s390x-binary-manywheel' 148 uses: pytorch/test-infra/.github/actions/setup-ssh@release/2.4 149 continue-on-error: true 150 with: 151 github-secret: ${{ secrets.github-token }} 152 153 - name: Checkout PyTorch 154 uses: pytorch/pytorch/.github/actions/checkout-pytorch@release/2.4 155 with: 156 no-sudo: ${{ inputs.build_environment == 'linux-aarch64-binary-manywheel' || inputs.build_environment == 'linux-s390x-binary-manywheel' }} 157 158 - name: Setup Linux 159 if: inputs.build_environment != 'linux-s390x-binary-manywheel' 160 uses: ./.github/actions/setup-linux 161 162 - name: Chown workspace 163 if: inputs.build_environment != 'linux-s390x-binary-manywheel' 164 uses: ./.github/actions/chown-workspace 165 with: 166 ALPINE_IMAGE: ${{ inputs.ALPINE_IMAGE }} 167 168 - name: Clean workspace 169 shell: bash 170 run: | 171 set -eux 172 173 rm -rf "${GITHUB_WORKSPACE}" 174 mkdir "${GITHUB_WORKSPACE}" 175 176 if [[ ${{ inputs.build_environment }} == 'linux-aarch64-binary-manywheel' ]] || [[ ${{ inputs.build_environment }} == 'linux-s390x-binary-manywheel' ]] ; then 177 rm -rf "${RUNNER_TEMP}/artifacts" 178 mkdir "${RUNNER_TEMP}/artifacts" 179 fi 180 181 - name: Checkout PyTorch to pytorch dir 182 uses: malfet/checkout@silent-checkout 183 with: 184 submodules: recursive 185 path: pytorch 186 quiet-checkout: true 187 188 - name: Clean PyTorch checkout 189 run: | 190 # Remove any artifacts from the previous checkouts 191 git clean -fxd 192 working-directory: pytorch 193 194 - name: Checkout pytorch/builder to builder dir 195 uses: malfet/checkout@silent-checkout 196 with: 197 ref: release/2.4 198 submodules: recursive 199 repository: pytorch/builder 200 path: builder 201 quiet-checkout: true 202 203 - name: Clean pytorch/builder checkout 204 run: | 205 # Remove any artifacts from the previous checkouts 206 git clean -fxd 207 working-directory: builder 208 209 - name: Check if the job is disabled 210 id: filter 211 uses: ./pytorch/.github/actions/filter-test-configs 212 with: 213 github-token: ${{ secrets.GITHUB_TOKEN }} 214 # NB: Use a mock test matrix with a default value here. After filtering, if the 215 # returned matrix is empty, it means that the job is disabled 216 test-matrix: | 217 { include: [ 218 { config: "default" }, 219 ]} 220 221 - name: Pull Docker image 222 if: ${{ steps.filter.outputs.is-test-matrix-empty == 'False' && inputs.build_environment != 'linux-s390x-binary-manywheel' }} 223 uses: pytorch/test-infra/.github/actions/pull-docker-image@release/2.4 224 with: 225 docker-image: ${{ inputs.DOCKER_IMAGE }} 226 227 - name: Build PyTorch binary 228 if: ${{ steps.filter.outputs.is-test-matrix-empty == 'False' }} 229 run: | 230 set -x 231 mkdir -p artifacts/ 232 container_name=$(docker run \ 233 -e BINARY_ENV_FILE \ 234 -e BUILDER_ROOT \ 235 -e BUILD_ENVIRONMENT \ 236 -e DESIRED_CUDA \ 237 -e DESIRED_DEVTOOLSET \ 238 -e DESIRED_PYTHON \ 239 -e GITHUB_ACTIONS \ 240 -e GPU_ARCH_TYPE \ 241 -e GPU_ARCH_VERSION \ 242 -e LIBTORCH_VARIANT \ 243 -e PACKAGE_TYPE \ 244 -e PYTORCH_FINAL_PACKAGE_DIR \ 245 -e PYTORCH_ROOT \ 246 -e SKIP_ALL_TESTS \ 247 -e PYTORCH_EXTRA_INSTALL_REQUIREMENTS \ 248 --tty \ 249 --detach \ 250 -v "${GITHUB_WORKSPACE}/pytorch:/pytorch" \ 251 -v "${GITHUB_WORKSPACE}/builder:/builder" \ 252 -v "${RUNNER_TEMP}/artifacts:/artifacts" \ 253 -w / \ 254 "${DOCKER_IMAGE}" 255 ) 256 docker exec -t -w "${PYTORCH_ROOT}" "${container_name}" bash -c "bash .circleci/scripts/binary_populate_env.sh" 257 if [[ ${BUILD_ENVIRONMENT} == *"aarch64"* ]]; then 258 docker exec -t "${container_name}" bash -c "bash /builder/aarch64_linux/aarch64_ci_build.sh" 259 else 260 docker exec -t "${container_name}" bash -c "source ${BINARY_ENV_FILE} && bash /builder/${{ inputs.PACKAGE_TYPE }}/build.sh" 261 fi 262 263 - name: Chown artifacts 264 if: ${{ steps.filter.outputs.is-test-matrix-empty == 'False' && inputs.build_environment != 'linux-s390x-binary-manywheel' }} 265 shell: bash 266 run: | 267 # Ensure the working directory gets chowned back to the current user 268 docker run --rm -v "${RUNNER_TEMP}/artifacts:/v" -w /v "${ALPINE_IMAGE}" chown -R "$(id -u):$(id -g)" . 269 270 - uses: actions/upload-artifact@v3 271 if: ${{ steps.filter.outputs.is-test-matrix-empty == 'False' }} 272 with: 273 name: ${{ inputs.build_name }} 274 if-no-files-found: error 275 path: 276 ${{ runner.temp }}/artifacts/* 277 278 - name: Teardown Linux 279 if: always() && inputs.build_environment != 'linux-s390x-binary-manywheel' 280 uses: pytorch/test-infra/.github/actions/teardown-linux@release/2.4 281 282 - name: Chown workspace 283 if: always() && inputs.build_environment != 'linux-s390x-binary-manywheel' 284 uses: ./pytorch/.github/actions/chown-workspace 285 with: 286 ALPINE_IMAGE: ${{ inputs.ALPINE_IMAGE }} 287 288 - name: Cleanup docker 289 if: always() && inputs.build_environment == 'linux-s390x-binary-manywheel' 290 shell: bash 291 run: | 292 # on s390x stop the container for clean worker stop 293 # ignore expansion of "docker ps -q" since it could be empty 294 # shellcheck disable=SC2046 295 docker stop $(docker ps -q) || true 296