catch tensor.numel() == 0 in nan detector (#140741) #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build manywheel docker images | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| - release/* | |
| tags: | |
| # NOTE: Binary build pipelines should only get triggered on release candidate or nightly builds | |
| # Release candidate tags look like: v1.11.0-rc1 | |
| - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ | |
| paths: | |
| - '.ci/docker/manywheel/*' | |
| - '.ci/docker/manywheel/build_scripts/*' | |
| - '.ci/docker/common/*' | |
| - .github/workflows/build-manywheel-images.yml | |
| pull_request: | |
| paths: | |
| - '.ci/docker/manywheel/*' | |
| - '.ci/docker/manywheel/build_scripts/*' | |
| - '.ci/docker/common/*' | |
| - .github/workflows/build-manywheel-images.yml | |
| env: | |
| DOCKER_REGISTRY: "docker.io" | |
| DOCKER_BUILDKIT: 1 | |
| WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} | |
| cancel-in-progress: true | |
| jobs: | |
| get-label-type: | |
| if: github.repository_owner == 'pytorch' | |
| name: get-label-type | |
| uses: pytorch/pytorch/.github/workflows/_runner-determinator.yml@main | |
| with: | |
| triggering_actor: ${{ github.triggering_actor }} | |
| issue_owner: ${{ github.event.pull_request.user.login || github.event.issue.user.login }} | |
| curr_branch: ${{ github.head_ref || github.ref_name }} | |
| curr_ref_type: ${{ github.ref_type }} | |
| build-docker-cuda: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.9xlarge.ephemeral" | |
| strategy: | |
| matrix: | |
| cuda_version: ["12.6", "12.4", "12.1", "11.8"] | |
| env: | |
| GPU_ARCH_TYPE: cuda | |
| GPU_ARCH_VERSION: ${{ matrix.cuda_version }} | |
| steps: | |
| - name: Purge tools folder (free space for build) | |
| run: rm -rf /opt/hostedtoolcache | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinux-builder-cuda${{matrix.cuda_version}} | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinux-builder:cuda${{matrix.cuda_version}} | |
| # NOTE: manylinux_2_28 are still experimental, see https://github.com/pytorch/pytorch/issues/123649 | |
| build-docker-cuda-manylinux_2_28: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.9xlarge.ephemeral" | |
| strategy: | |
| matrix: | |
| cuda_version: ["12.6", "12.4", "12.1", "11.8"] | |
| env: | |
| GPU_ARCH_TYPE: cuda-manylinux_2_28 | |
| GPU_ARCH_VERSION: ${{ matrix.cuda_version }} | |
| steps: | |
| - name: Purge tools folder (free space for build) | |
| run: rm -rf /opt/hostedtoolcache | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinux2_28-builder-cuda${{matrix.cuda_version}} | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinux2_28-builder:cuda${{matrix.cuda_version}} | |
| build-docker-cuda-aarch64: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.arm64.2xlarge.ephemeral" | |
| strategy: | |
| matrix: | |
| cuda_version: ["12.4"] | |
| env: | |
| GPU_ARCH_TYPE: cuda-aarch64 | |
| GPU_ARCH_VERSION: ${{ matrix.cuda_version }} | |
| steps: | |
| - name: Checkout PyTorch | |
| uses: actions/checkout@v3 | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinuxaarch64-builder-cuda${{matrix.cuda_version}} | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinuxaarch64-builder:cuda${{matrix.cuda_version}} | |
| build-docker-rocm: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.9xlarge.ephemeral" | |
| strategy: | |
| matrix: | |
| rocm_version: ["6.1", "6.2"] | |
| env: | |
| GPU_ARCH_TYPE: rocm | |
| GPU_ARCH_VERSION: ${{ matrix.rocm_version }} | |
| steps: | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinux-builder-rocm${{matrix.rocm_version}} | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinux-builder:rocm${{matrix.rocm_version}} | |
| build-docker-cpu: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.9xlarge.ephemeral" | |
| steps: | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinux-builder-cpu | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinux-builder:cpu | |
| build-docker-cpu-manylinux_2_28: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.9xlarge.ephemeral" | |
| env: | |
| GPU_ARCH_TYPE: cpu-manylinux_2_28 | |
| steps: | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinux2_28-builder-cpu | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinux2_28-builder:cpu | |
| build-docker-cpu-aarch64: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.arm64.2xlarge.ephemeral" | |
| env: | |
| GPU_ARCH_TYPE: cpu-aarch64 | |
| steps: | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinuxaarch64-builder-cpu-aarch64 | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinuxaarch64-builder:cpu-aarch64 | |
| build-docker-cpu-aarch64-2_28: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.arm64.2xlarge.ephemeral" | |
| env: | |
| GPU_ARCH_TYPE: cpu-aarch64-2_28 | |
| steps: | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinux2_28_aarch64-builder-cpu-aarch64 | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinux2_28_aarch64-builder:cpu-aarch64 | |
| build-docker-cpu-cxx11-abi: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.9xlarge.ephemeral" | |
| env: | |
| GPU_ARCH_TYPE: cpu-cxx11-abi | |
| steps: | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinuxcxx11-abi-builder-cpu-cxx11-abi | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinuxcxx11-abi-builder:cpu-cxx11-abi | |
| build-docker-xpu: | |
| environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} | |
| needs: get-label-type | |
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.9xlarge.ephemeral" | |
| env: | |
| GPU_ARCH_TYPE: xpu | |
| steps: | |
| - name: Checkout PyTorch | |
| uses: pytorch/pytorch/.github/actions/checkout-pytorch@main | |
| with: | |
| submodules: false | |
| - name: Calculate docker image | |
| if: env.WITH_PUSH == 'false' | |
| uses: pytorch/test-infra/.github/actions/calculate-docker-image@main | |
| with: | |
| docker-image-name: manylinux2_28-builder-xpu | |
| docker-build-dir: .ci/docker/manywheel | |
| always-rebuild: true | |
| push: true | |
| - name: Authenticate if WITH_PUSH | |
| if: env.WITH_PUSH == 'true' | |
| env: | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_ID: ${{ secrets.DOCKER_ID }} | |
| run: | | |
| if [[ "${WITH_PUSH}" == true ]]; then | |
| echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin | |
| fi | |
| - name: Build Docker Image | |
| if: env.WITH_PUSH == 'true' | |
| uses: nick-fields/[email protected] | |
| with: | |
| shell: bash | |
| timeout_minutes: 90 | |
| max_attempts: 3 | |
| retry_wait_seconds: 90 | |
| command: | | |
| .ci/docker/manywheel/build.sh manylinux2_28-builder:xpu |