diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6786da5f4..ec7bb0f60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,16 +7,14 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.62 + BUILDER_VERSION: v0.9.67 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - AWS_REGION: us-east-1 + CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} + AWS_DEFAULT_REGION: us-east-1 jobs: manylinux1: @@ -31,11 +29,18 @@ jobs: - cp37-cp37m - cp38-cp38 - cp39-cp39 + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python manylinux2014: runs-on: ubuntu-20.04 # latest @@ -53,15 +58,23 @@ jobs: - cp310-cp310 - cp311-cp311 - cp312-cp312 + - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT steps: - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python musllinux-1-1: runs-on: ubuntu-22.04 # latest @@ -78,15 +91,24 @@ jobs: - cp310-cp310 - cp311-cp311 - cp312-cp312 + - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT steps: - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python raspberry: runs-on: ubuntu-20.04 # latest @@ -95,16 +117,23 @@ jobs: matrix: image: - raspbian-bullseye + permissions: + id-token: write # This is required for requesting the JWT steps: - # set arm arch - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + # set arm arch + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-compat: runs-on: ubuntu-22.04 # latest @@ -115,13 +144,18 @@ jobs: - fedora-34-x64 - opensuse-leap - rhel8-x64 + permissions: + id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-compiler-compat: runs-on: ubuntu-22.04 # latest @@ -138,8 +172,17 @@ jobs: - gcc-6 - gcc-7 - gcc-8 + permissions: + id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Caller Identity + run: | + aws sts get-caller-identity - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh @@ -147,7 +190,14 @@ jobs: use-system-libcrypto: runs-on: ubuntu-20.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} env: AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' @@ -173,30 +223,51 @@ jobs: strategy: matrix: arch: [x86, x64] + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" macos: runs-on: macos-14 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} macos-x64: runs-on: macos-14-large # latest + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} openbsd: @@ -206,7 +277,14 @@ jobs: matrix: # OpenBSD only supports the two most recent releases version: ['7.4', '7.5'] + permissions: + id-token: write # This is required for requesting the JWT steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install - uses: actions/checkout@v4 with: @@ -218,7 +296,7 @@ jobs: version: ${{ matrix.version }} cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION run: | sudo pkg_add awscli py3-pip py3-urllib3 python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" @@ -227,7 +305,14 @@ jobs: freebsd: runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - uses: actions/checkout@v4 with: submodules: true @@ -239,7 +324,7 @@ jobs: version: '14.0' cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION run: | sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" @@ -247,18 +332,13 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} # check that tests requiring custom env-vars or AWS credentials are simply skipped - tests-ok-without-env-vars: + tests-ok-without-creds: runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v4 with: submodules: true - - name: Run tests without env-vars or AWS creds - env: - # unset env-vars that provide AWS credentials - AWS_ACCESS_KEY_ID: - AWS_SECRET_ACCESS_KEY: - AWS_DEFAULT_REGION: + - name: Run tests run: | python3 -m pip install --upgrade --requirement requirements-dev.txt python3 -m pip install . --verbose @@ -266,7 +346,14 @@ jobs: package-source: runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - uses: actions/checkout@v4 with: submodules: true diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index a140b7d67..62bd8531c 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -19,9 +19,13 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 41ce9fdef..01ca703d9 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -19,9 +19,13 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh index 0cbc1a196..6f7f26645 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh @@ -19,9 +19,13 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh index 6637d95ac..dccf0eef0 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh @@ -19,9 +19,13 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index a3a92bc63..a9feb731d 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -11,7 +11,10 @@ set -ex /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. +/Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel + #now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-win32.bat b/continuous-delivery/build-wheels-win32.bat index 6a3e53977..d0f2576e3 100644 --- a/continuous-delivery/build-wheels-win32.bat +++ b/continuous-delivery/build-wheels-win32.bat @@ -7,6 +7,12 @@ "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error +:: Don't need to build wheels for Python 3.12. +:: The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +"C:\Program Files (x86)\Python313-32\python.exe" setup.py sdist bdist_wheel || goto error + goto :EOF :error diff --git a/continuous-delivery/build-wheels-win64.bat b/continuous-delivery/build-wheels-win64.bat index 5a862e6e1..fd94419e5 100644 --- a/continuous-delivery/build-wheels-win64.bat +++ b/continuous-delivery/build-wheels-win64.bat @@ -6,6 +6,12 @@ "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error +:: Don't need to build wheels for Python 3.12. +:: The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +"C:\Program Files\Python313\python.exe" setup.py sdist bdist_wheel || goto error + goto :EOF :error diff --git a/crt/aws-c-auth b/crt/aws-c-auth index 3281f8692..48d647bf4 160000 --- a/crt/aws-c-auth +++ b/crt/aws-c-auth @@ -1 +1 @@ -Subproject commit 3281f8692e6fd10562c4585a4dded5c16b322698 +Subproject commit 48d647bf43f8872e4dc5ec6343b0c5974195fbdd diff --git a/crt/aws-c-http b/crt/aws-c-http index 4e74ab1e3..6068653e1 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 4e74ab1e3702763e0b87bd1752f5a37c2f0400ac +Subproject commit 6068653e1d582bd8e7d1c9f81f86beaf10444e3d diff --git a/crt/aws-c-mqtt b/crt/aws-c-mqtt index ed7bbd68c..c43232c1b 160000 --- a/crt/aws-c-mqtt +++ b/crt/aws-c-mqtt @@ -1 +1 @@ -Subproject commit ed7bbd68c03d7022c915a2924740ab7992ad2311 +Subproject commit c43232c1bc378344bb7245d7fcb167410f3fe931 diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 502cd6249..aede1d8c2 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 502cd6249c6523583c19b122c65e02cf74301b3e +Subproject commit aede1d8c24f9f580d5a96c089878e9b258b88d04 diff --git a/crt/aws-lc b/crt/aws-lc index 2f1879759..d3a598c1b 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 2f1879759b2e0fc70592665bdf10087b64f44b7d +Subproject commit d3a598c1b419d49b5b08f0677add4581572e2edc diff --git a/crt/s2n b/crt/s2n index 87f4a0585..08d413a0b 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 87f4a0585dc3056433f193b9305f587cff239be3 +Subproject commit 08d413a0b9b3226e775a38f04e3cf02230cc97c4 diff --git a/setup.py b/setup.py index 13dd63103..50393a8b6 100644 --- a/setup.py +++ b/setup.py @@ -403,7 +403,13 @@ def awscrt_ext(): else: extra_link_args += ['-Wl,--fatal-warnings'] - if sys.version_info >= (3, 11): + # prefer building with stable ABI, so a wheel can work with multiple major versions + if sys.version_info >= (3, 13): + # 3.13 deprecates PyWeakref_GetObject(), adds alternative + define_macros.append(('Py_LIMITED_API', '0x030D0000')) + py_limited_api = True + elif sys.version_info >= (3, 11): + # 3.11 is the first stable ABI that has everything we need define_macros.append(('Py_LIMITED_API', '0x030B0000')) py_limited_api = True diff --git a/source/http_stream.c b/source/http_stream.c index bf7021862..6843e0ea8 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -203,7 +203,10 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro } /* DECREF python self, we don't need to force it to stay alive any longer. */ - Py_DECREF(PyWeakref_GetObject(stream->self_proxy)); + PyObject *self = aws_py_weakref_get_ref(stream->self_proxy); + /* DECREF twice because `aws_py_weakref_get_ref` returns a strong reference */ + Py_XDECREF(self); + Py_XDECREF(self); PyGILState_Release(state); /*************** GIL RELEASE ***************/ diff --git a/source/module.c b/source/module.c index 4230ab6bf..197ce20f1 100644 --- a/source/module.c +++ b/source/module.c @@ -516,6 +516,39 @@ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf) { return PyMemoryView_FromMemory(mem_start, mem_size, PyBUF_WRITE); } +PyObject *aws_py_weakref_get_ref(PyObject *ref) { + /* If Python >= 3.13 */ +#if PY_VERSION_HEX >= 0x030D0000 + /* Use PyWeakref_GetRef() (new in Python 3.13), which gets you: + * a new strong reference, + * or NULL because ref is dead, + * or -1 because you called it wrong */ + PyObject *obj = NULL; + if (PyWeakref_GetRef(ref, &obj) == -1) { + PyErr_WriteUnraisable(PyErr_Occurred()); + AWS_ASSERT(0 && "expected a weakref"); + } + return obj; + +#else + /* Use PyWeakref_GetObject() (deprecated as of Python 3.13), which gets you: + * a borrowed reference, + * or Py_None because ref is dead, + * or NULL because you called it wrong */ + PyObject *obj = PyWeakref_GetObject(ref); /* borrowed reference */ + if (obj == NULL) { + PyErr_WriteUnraisable(PyErr_Occurred()); + AWS_ASSERT(0 && "expected a weakref"); + } else if (obj == Py_None) { + obj = NULL; + } else { + /* Be like PyWeakref_GetRef() and make it new strong reference */ + Py_INCREF(obj); + } + return obj; +#endif +} + int aws_py_gilstate_ensure(PyGILState_STATE *out_state) { if (AWS_LIKELY(Py_IsInitialized())) { *out_state = PyGILState_Ensure(); diff --git a/source/module.h b/source/module.h index d7254d8c4..f626cb1e5 100644 --- a/source/module.h +++ b/source/module.h @@ -106,6 +106,27 @@ PyObject *aws_py_get_error_message(PyObject *self, PyObject *args); /* Create a write-only memoryview from the remaining free space in an aws_byte_buf */ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf); +/* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different + * APIs since we support Python versions before 3.13. Returns a strong reference if non-null, which you must release. */ + +/** + * Given a weak reference, returns a NEW strong reference to the referenced object, + * or NULL if the reference is dead (this function NEVER raises a python exception or AWS Error). + * + * You MUST NOT call this if ref came from a user, or ref is NULL. + * + * This is a simplified version of PyWeakref_GetRef() / PyWeakref_GetObject(). + * Simpler because: + * - Python 3.13 adds PyWeakref_GetRef() and deprecates PyWeakref_GetObject(). + * This function calls the appropriate one. + * + * - This functions has 2 outcomes instead of 3: + * The 3rd being a Python exception for calling it incorrectly. + * If that happens, this function calls PyErr_WriteUnraisable() to clear the exception, + * which is what you would have done anyway. + */ +PyObject *aws_py_weakref_get_ref(PyObject *ref); + /* Allocator that calls into PyObject_[Malloc|Free|Realloc] */ struct aws_allocator *aws_py_get_allocator(void); diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index 9eb73a950..78a26057e 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -140,8 +140,8 @@ static void s_on_connection_success( return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_success", "(iN)", return_code, PyBool_FromLong(session_present)); if (success_result) { @@ -149,6 +149,7 @@ static void s_on_connection_success( } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } PyGILState_Release(state); @@ -167,14 +168,15 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code); if (success_result) { Py_DECREF(success_result); } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } PyGILState_Release(state); @@ -194,14 +196,15 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code); if (result) { Py_DECREF(result); } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } PyGILState_Release(state); @@ -227,8 +230,8 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *result = PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present)); if (result) { @@ -236,6 +239,7 @@ static void s_on_connection_resumed( } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } PyGILState_Release(state); @@ -258,14 +262,15 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()"); if (result) { Py_DECREF(result); } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } PyGILState_Release(state); @@ -535,8 +540,9 @@ static void s_ws_handshake_transform( } /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ - if (connection_py == Py_None) { + + PyObject *connection_py = aws_py_weakref_get_ref(connection_binding->self_proxy); /* new reference */ + if (connection_py == NULL) { aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; } @@ -593,6 +599,7 @@ static void s_ws_handshake_transform( done:; /* Save off error code, so it doesn't got stomped before we pass it to callback*/ int error_code = aws_last_error(); + Py_XDECREF(connection_py); if (ws_transform_capsule) { Py_DECREF(ws_transform_capsule); diff --git a/test/test_mqtt5_credentials.py b/test/test_mqtt5_credentials.py index 03e5c401e..8a5d24b09 100644 --- a/test/test_mqtt5_credentials.py +++ b/test/test_mqtt5_credentials.py @@ -197,40 +197,6 @@ def test_mqtt5_ws_cred_static(self): input_role_secret_access_key, input_role_session_token ) - credentials = auth.AwsCredentialsProvider.new_default_chain() - - def sign_function(transform_args, **kwargs): - signing_config = auth.AwsSigningConfig( - algorithm=auth.AwsSigningAlgorithm.V4, - signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, - credentials_provider=credentials, - region=input_region, - service="iotdevicegateway", - omit_session_token=True - ) - signing_future = auth.aws_sign_request( - http_request=transform_args.http_request, - signing_config=signing_config) - signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception())) - client_options.websocket_handshake_transform = sign_function - client_options.tls_ctx = io.ClientTlsContext(io.TlsContextOptions()) - - callbacks = Mqtt5TestCallbacks() - client = self._create_client(client_options=client_options, callbacks=callbacks) - client.start() - callbacks.future_connection_success.result(TIMEOUT) - client.stop() - callbacks.future_stopped.result(TIMEOUT) - - def test_mqtt5_ws_cred_default(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") - input_region = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_REGION") - - client_options = mqtt5.ClientOptions( - host_name=input_host_name, - port=443 - ) - credentials = auth.AwsCredentialsProvider.new_default_chain() def sign_function(transform_args, **kwargs): signing_config = auth.AwsSigningConfig( @@ -380,6 +346,12 @@ def sign_function(transform_args, **kwargs): callbacks.future_stopped.result(TIMEOUT) def test_mqtt5_ws_cred_environment(self): + self._test_mqtt5_ws_cred_environment(use_default_chain=False) + + def test_mqtt5_ws_cred_default_chain(self): + self._test_mqtt5_ws_cred_environment(use_default_chain=True) + + def _test_mqtt5_ws_cred_environment(self, use_default_chain): input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") input_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_ACCESS_KEY") input_secret_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_SECRET_ACCESS_KEY") @@ -399,7 +371,10 @@ def test_mqtt5_ws_cred_environment(self): os.environ["AWS_SECRET_ACCESS_KEY"] = input_secret_access_key os.environ["AWS_SESSION_TOKEN"] = input_session_token # This should load the environment variables we just set - credentials = auth.AwsCredentialsProvider.new_environment() + if use_default_chain: + credentials = auth.AwsCredentialsProvider.new_default_chain() + else: + credentials = auth.AwsCredentialsProvider.new_environment() def sign_function(transform_args, **kwargs): signing_config = auth.AwsSigningConfig( diff --git a/test/test_mqtt_credentials.py b/test/test_mqtt_credentials.py index 15335e3e2..e1bd2c68d 100644 --- a/test/test_mqtt_credentials.py +++ b/test/test_mqtt_credentials.py @@ -136,40 +136,6 @@ def sign_function(transform_args, **kwargs): connection.connect().result(TIMEOUT) connection.disconnect().result(TIMEOUT) - def test_mqtt311_ws_cred_default(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT311_IOT_CORE_HOST") - input_region = _get_env_variable("AWS_TEST_MQTT311_IOT_CORE_REGION") - - credentials = auth.AwsCredentialsProvider.new_default_chain() - - def sign_function(transform_args, **kwargs): - signing_config = auth.AwsSigningConfig( - algorithm=auth.AwsSigningAlgorithm.V4, - signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, - credentials_provider=credentials, - region=input_region, - service="iotdevicegateway", - omit_session_token=True - ) - signing_future = auth.aws_sign_request( - http_request=transform_args.http_request, - signing_config=signing_config) - signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception())) - - elg = EventLoopGroup() - resolver = DefaultHostResolver(elg) - bootstrap = ClientBootstrap(elg, resolver) - client = Client(bootstrap, ClientTlsContext(TlsContextOptions())) - connection = Connection( - client=client, - client_id=create_client_id(), - host_name=input_host_name, - port=int(443), - use_websockets=True, - websocket_handshake_transform=sign_function) - connection.connect().result(TIMEOUT) - connection.disconnect().result(TIMEOUT) - def test_mqtt311_ws_cred_cognito(self): input_cognito_endpoint = _get_env_variable("AWS_TEST_MQTT311_COGNITO_ENDPOINT") input_cognito_identity = _get_env_variable("AWS_TEST_MQTT311_COGNITO_IDENTITY") @@ -300,6 +266,12 @@ def sign_function(transform_args, **kwargs): connection.disconnect().result(TIMEOUT) def test_mqtt311_ws_cred_environment(self): + self._test_mqtt311_ws_cred_environment(use_default_chain=False) + + def test_mqtt311_ws_cred_default(self): + self._test_mqtt311_ws_cred_environment(use_default_chain=True) + + def _test_mqtt311_ws_cred_environment(self, use_default_chain): input_access_key = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_ACCESS_KEY") input_secret_access_key = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_SECRET_ACCESS_KEY") input_session_token = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_SESSION_TOKEN") @@ -314,8 +286,11 @@ def test_mqtt311_ws_cred_environment(self): os.environ["AWS_ACCESS_KEY_ID"] = input_access_key os.environ["AWS_SECRET_ACCESS_KEY"] = input_secret_access_key os.environ["AWS_SESSION_TOKEN"] = input_session_token - # This should load the environment variables we just set - credentials = auth.AwsCredentialsProvider.new_environment() + if use_default_chain: + credentials = auth.AwsCredentialsProvider.new_default_chain() + else: + # This should load the environment variables we just set + credentials = auth.AwsCredentialsProvider.new_environment() signing_config = auth.AwsSigningConfig( algorithm=auth.AwsSigningAlgorithm.V4, signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS,