Merge pull request #70 from Chia-Network/20240305-update-types

20240305 update types

.github/workflows/build-m1-wheel.yml

@@ -111,22 +111,32 @@ jobs:
         name: wheels
         path: ./target/wheels
 
-    - name: Test for secrets access
-      id: check_secrets
-      shell: bash
-      run: |
-        unset HAS_AWS_SECRET
-
-        if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi
-        echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT
-      env:
-        AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}"
+  upload:
+    name: Upload to PyPI
+    runs-on: ubuntu-latest
+    needs: build_wheels
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
 
     - name: Set Env
       uses: Chia-Network/actions/setjobenv@main
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+    - name: Install python
+      uses: Chia-Network/actions/setup-python@main
+      with:
+        python-version: "3.10"
+
+    - name: Download artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheels
+        path: ./target/wheels/
+
     - name: publish (PyPi)
       if: env.RELEASE == 'true'
       uses: pypa/gh-action-pypi-publish@release/v1
@@ -142,6 +152,17 @@ jobs:
         packages-dir: target/wheels/
         skip-existing: true
 
+    - name: Test for secrets access
+      id: check_secrets
+      shell: bash
+      run: |
+        unset HAS_AWS_SECRET
+
+        if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi
+        echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT
+      env:
+        AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}"
+
     - name: Configure AWS credentials
       if: steps.check_secrets.outputs.HAS_AWS_SECRET
       uses: aws-actions/configure-aws-credentials@v2
@@ -151,8 +172,10 @@ jobs:
 
     - name: Publish Dev
       if: steps.check_secrets.outputs.HAS_AWS_SECRET && github.ref == 'refs/heads/dev'
+      shell: bash
+      working-directory: ./target/wheels
       run: |
-        FILES=$(find ${{ github.workspace }}/target/wheels -type f -name '*.whl')
+        FILES=$(find . -type f -name '*.whl')
         while IFS= read -r file; do
           filename=$(basename $file)
           aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename"

.github/workflows/build-riscv.yml

@@ -0,0 +1,81 @@
+name: QEMU Build and test riscv64 crate
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+    tags:
+        - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  build_crate:
+    name: Build riscv64 crate and run tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 1
+
+    - name: Set up QEMU on x86_64
+      id: qemu
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: riscv64
+
+    - name: Build and Test
+      run: |
+        docker run --rm --platform linux/riscv64 \
+          -v ${{ github.workspace }}:/ws --workdir=/ws \
+          chianetwork/ubuntu-22.04-risc-builder:latest \
+          bash -exc '\
+          cargo test --release
+          '
+
+  build_wheels:
+    name: QEMU riscv64 wheel
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 1
+
+    - name: Set up QEMU on x86_64
+      id: qemu
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: riscv64
+
+    - name: Build
+      run: |
+        docker run --rm --platform linux/riscv64 \
+          -v ${{ github.workspace }}:/ws --workdir=/ws \
+          chianetwork/ubuntu-22.04-risc-builder:latest \
+          bash -exc '\
+          pyenv global 3.10
+          python -m venv venv && \
+          source ./venv/bin/activate && \
+          pip install --upgrade pip && \
+          pip install --extra-index-url https://pypi.chia.net/simple/ maturin==1.2.3 && \
+          maturin build -i python --release --strip \
+          '
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: packages
+        path: ./target/wheels/
+

.github/workflows/build-test.yml

@@ -97,7 +97,7 @@ jobs:
         . ./activate
         maturin build -i python --release --strip
         # Find and install the newly built wheel
-        python support/wheelname.py
+        pip install --no-index --find-links target/wheels/ clvm_tools_rs
 
     - name: Install clvm_tools_rs wheel
       if: ${{ !startsWith(matrix.os, 'windows') }}
@@ -140,14 +140,38 @@ jobs:
         python -c 'import clvm_tools_rs; print(clvm_tools_rs.__file__)'
         echo "CLVM_TOOLS_RS_VERSION=$(python -c 'import clvm_tools_rs; print(clvm_tools_rs.get_version())')" >> "$GITHUB_ENV"
 
+    # Test cldb output both run from python and via its command line tool.
+    - name: "Run step run tests"
+      run: |
+        . ./activate
+        cargo build
+        # This finds and installs a compatible wheel from the target directory on linux.
+        # We do it here as a hedge against changes in the python environment that might
+        # have happened due to pulling in the trunk versions of other packages that
+        # depend on clvm_tools_rs.
+        pip install --no-index --find-links target/wheels/ clvm_tools_rs
+        pip install clvm_rs
+        pip install clvm_tools
+        cd resources/tests && python test_clvm_step.py && python mandelbrot-cldb.py && python test_compile_from_string.py
+
+    - name: "Test step run with mandelbrot, setting print only"
+      run: |
+        . ./activate
+        python ./resources/tests/lib/steprun.py ./resources/tests/mandelbrot/mandelbrot.clvm.hex resources/tests/mandelbrot/mand_args.txt ./resources/tests/mandelbrot/mandelbrot.sym > mand_output.txt
+        # git diff invoked this way returns 0 (as /bin/true) if there is no difference or 1 if there is.
+        git diff --no-index -s --quiet -- mand_output.txt ./resources/tests/mandelbrot/mand_test.txt
+        # Remove file in a compatible way using git as a general tool
+        git add mand_output.txt
+        git rm --force mand_output.txt
+
 
     - name: Verify recompilation of old sources match with new compiler
       if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
       run: |
         set -x
         . ./activate
         # Build cmd line tools
-        PYO3_PYTHON=`which python` cargo build --no-default-features --release
+        PYO3_PYTHON=`which python` cargo build --release
 
         # Grab chia-blockchain
         rm -rf chia-blockchain
@@ -157,7 +181,7 @@ jobs:
         cp support/install_deps.sh support/verify_compiler_version.sh chia-blockchain
         (cd chia-blockchain && python -m venv venv && . venv/bin/activate && pip install --upgrade pip && \
           python -m pip install maturin==1.1.0 && \
-          cd .. && python support/wheelname.py && \
+          cd .. && pip install --no-index --find-links target/wheels/ clvm_tools_rs && \
           cd chia-blockchain && \
           # deps for manage_clvm.py
           pip install click typing_extensions chia_rs clvm && \
@@ -169,7 +193,7 @@ jobs:
       run: |
         python -m pip install pytest
         # This script installs the wheel built during this workflow.
-        python support/wheelname.py
+        pip install --no-index --find-links wheel/target/wheels/ clvm_tools_rs
         # TODO: bring in the other tests in resources/tests/cmdline/tests besides cmds_test.py
         (cd resources/tests/cmdline/tests && py.test cmds_test.py )
 
@@ -182,13 +206,13 @@ jobs:
         # Ensure we're using the sources we have.  This is a hedge against
         # changes made by any other step doing operations on pip.
         # This script installs the wheel built during this workflow.
-        python support/wheelname.py
+        pip install --no-index --find-links wheel/target/wheels/ clvm_tools_rs
         (cd resources/tests && python check-recompiles.py)
 
     - name: Verify recompilation follows date and modification rules
       if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
       run: |
-        python support/wheelname.py
+        pip install --no-index --find-links wheel/target/wheels/ clvm_tools_rs
         python resources/tests/test-clvm-recompile-behavior.py
 
     - name: Install pytest
@@ -210,7 +234,7 @@ jobs:
 
     - name: Run tests
       if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
-      run: cargo test --no-default-features
+      run: cargo test
 
     - name: Exhaustive assign tests
       if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
@@ -223,43 +247,77 @@ jobs:
         rustup component add llvm-tools-preview
         python ./resources/coverage/run_coverage.py --require-percent 60
 
+    - name: Build alpine wheel via docker
+      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
+      run: |
+        cd resources/alpine && docker build -t clvm-tools-rs-alpine .
+        docker run -v ${GITHUB_WORKSPACE}:/root/clvm_tools_rs -t clvm-tools-rs-alpine sh /root/build-alpine.sh
+
     - name: Upload artifacts
       uses: actions/upload-artifact@v3
       with:
         name: wheels
         path: ./target/wheels/
 
-    - name: Test for secrets access
-      id: check_secrets
-      shell: bash
-      run: |
-        unset HAS_AWS_SECRET
-
-        if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi
-        echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT
-      env:
-        AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}"
+  upload:
+    name: Upload to PyPI
+    runs-on: ubuntu-latest
+    needs: build_wheels
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
 
     - name: Set Env
       uses: Chia-Network/actions/setjobenv@main
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: publish (PyPi)
+    - name: Install python
+      uses: Chia-Network/actions/setup-python@main
+      with:
+        python-version: "3.10"
+
+    - name: Download artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: wheels
+        path: ./target/wheels/
+
+    - name: Check for incompatible wheels and bail if found
+      run: |
+        set -e
+        # Check for a wheel with -linux_x86_64 platform tag, which will
+        # fail uploading to pypi.
+        find target -name \*-linux_x86_64.whl -exec /bin/false '{}' '+'
+
+    - name: Publish distribution to PyPI
       if: env.RELEASE == 'true'
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         packages-dir: target/wheels/
         skip-existing: true
 
-    - name: publish (Test PyPi)
+    - name: Publish distribution to Test PyPI
       if: env.PRE_RELEASE == 'true'
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         repository-url: https://test.pypi.org/legacy/
         packages-dir: target/wheels/
         skip-existing: true
 
+    - name: Test for secrets access
+      id: check_secrets
+      shell: bash
+      run: |
+        unset HAS_AWS_SECRET
+
+        if [ -n "$AWS_SECRET" ]; then HAS_AWS_SECRET='true' ; fi
+        echo HAS_AWS_SECRET=${HAS_AWS_SECRET} >>$GITHUB_OUTPUT
+      env:
+        AWS_SECRET: "${{ secrets.CHIA_AWS_ACCOUNT_ID }}"
+
     - name: Configure AWS credentials
       if: steps.check_secrets.outputs.HAS_AWS_SECRET
       uses: aws-actions/configure-aws-credentials@v2
@@ -278,12 +336,6 @@ jobs:
           aws --no-progress s3 cp "$file" "s3://download.chia.net/simple-dev/clvm-tools-rs/$filename"
         done <<< "$FILES"
 
-    - name: Build alpine wheel via docker
-      if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python, '3.8')
-      run: |
-        cd resources/alpine && docker build -t clvm-tools-rs-alpine .
-        docker run -v ${GITHUB_WORKSPACE}:/root/clvm_tools_rs -t clvm-tools-rs-alpine sh /root/build-alpine.sh
-
   fmt:
     runs-on: ubuntu-20.04
     name: cargo fmt
@@ -340,6 +392,7 @@ jobs:
             toolchain: stable
       - name: Run for coverage
         run: |
+          set -e
           sudo apt-get update
           sudo apt-get install lcov -y
           rustup component add llvm-tools-preview
@@ -354,9 +407,13 @@ jobs:
           pip install ./clvm_tools
           pip install maturin pytest
           maturin develop --release
+          echo 'PYTEST'
           (cd resources/tests/cmdline/tests && pytest)
+          echo 'PYTEST COMPLETE'
           grcov . --binary-path target -s . --branch --ignore-not-existing --ignore='*/.cargo/*' --ignore='*/tests/*' -o rust_cov.info
+          echo 'GRCOV RAN'
           python -c 'with open("rust_cov.info") as f: lines = [l for l in f if not (l.startswith("DA:") and int(l.split(",")[1].strip()) >= 2**63)]; open("lcov.info", "w").writelines(lines)'
+          echo 'COVERAGE STEP DONE'
       - name: Upload to Coveralls
         uses: coverallsapp/github-action@v2
         if: always()

.github/workflows/dependency-review.yml

@@ -0,0 +1,24 @@
+# Managed by repo-content-updater
+# Dependency Review Action
+#
+# This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging.
+#
+# Source repository: https://github.com/actions/dependency-review-action
+# Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement
+name: '🚨 Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v4
+
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v4
+        with:
+          deny-licenses: AGPL-1.0-only, AGPL-1.0-or-later, AGPL-1.0-or-later, AGPL-3.0-or-later, GPL-1.0-only, GPL-1.0-or-later, GPL-2.0-only, GPL-2.0-or-later, GPL-3.0-only, GPL-3.0-or-later