Merge unblob-native #12
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| merge_group: | |
| permissions: | |
| contents: read | |
| env: | |
| SCCACHE_GHA_ENABLED: "true" | |
| RUSTC_WRAPPER: "sccache" | |
| DOCKER_IMAGE: ghcr.io/onekey-sec/unblob | |
| jobs: | |
| check_pre_commit: | |
| name: Check - pre-commit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| install-test-deps: "false" | |
| - name: Setup Nix | |
| uses: cachix/install-nix-action@v30 | |
| with: | |
| install_url: https://releases.nixos.org/nix/nix-2.18.8/install | |
| - name: Check pre-commit hook | |
| uses: pre-commit/[email protected] | |
| check_pyright: | |
| name: Check - pyright | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| - name: Check - pyright | |
| run: uv run pyright . | |
| run_python_tests: | |
| name: Run tests (Python) | |
| needs: [check_pre_commit, check_pyright] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup git lfs | |
| uses: ./.github/actions/setup-git-lfs | |
| - name: Run pytest | |
| run: uv run pytest -vvv | |
| run_rust_tests: | |
| name: Run tests (Rust) | |
| needs: [check_pre_commit] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Run cargo test | |
| run: cargo test | |
| build_linux_wheels: | |
| name: Build wheels (linux) | |
| needs: [check_pre_commit] | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - manylinux: manylinux2014 | |
| target: x86_64 | |
| - manylinux: manylinux2014 | |
| target: aarch64 | |
| - manylinux: musllinux_1_1 | |
| target: x86_64 | |
| - manylinux: musllinux_1_1 | |
| target: aarch64 | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.x | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.platform.target }} | |
| args: --release --out dist | |
| sccache: "true" | |
| manylinux: ${{ matrix.platform.manylinux }} | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-${{ matrix.platform.manylinux }}-${{ matrix.platform.target }} | |
| path: dist | |
| - name: Check wheel (x86-manylinux) | |
| if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux') | |
| shell: bash | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install unblob --find-links dist --force-reinstall | |
| unblob --help | |
| - name: Check wheel (cross-manylinux) | |
| if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux') | |
| uses: uraimo/run-on-arch-action@v2 | |
| with: | |
| arch: ${{ matrix.platform.target }} | |
| distro: ubuntu22.04 | |
| githubToken: ${{ github.token }} | |
| install: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3 python3-pip libmagic1 | |
| pip3 install -U pip pytest | |
| run: | | |
| set -e | |
| pip3 install unblob --find-links dist --force-reinstall | |
| unblob --help | |
| - name: Check wheel (x86-musllinux) | |
| if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux') | |
| uses: addnab/docker-run-action@v3 | |
| with: | |
| image: alpine:latest | |
| options: -v ${{ github.workspace }}:/io -w /io | |
| run: | | |
| set -e | |
| apk add py3-pip libmagic gcc lz4 musl-dev python3-dev | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install unblob --no-index dist/*whl | |
| unblob --help | |
| - name: Check wheel (cross-musllinux) | |
| if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux') | |
| uses: uraimo/run-on-arch-action@v2 | |
| with: | |
| arch: ${{ matrix.platform.target }} | |
| distro: alpine_latest | |
| githubToken: ${{ github.token }} | |
| install: | | |
| apk add py3-pip libmagic gcc lz4 musl-dev python3-dev | |
| run: | | |
| set -e | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install unblob dist/*.whl | |
| unblob --help | |
| build_macos_wheels: | |
| name: Build wheels (macos) | |
| needs: [check_pre_commit] | |
| runs-on: ${{ matrix.platform.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - runner: macos-13 | |
| target: x86_64 | |
| - runner: macos-14 | |
| target: aarch64 | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| install-test-deps: false | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.platform.target }} | |
| args: --release --out dist | |
| sccache: "true" | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-macos-${{ matrix.platform.target }} | |
| path: dist | |
| - name: Check wheel | |
| run: | | |
| set -e | |
| brew install libmagic | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install unblob --find-links dist --force-reinstall | |
| unblob --help | |
| build_sdist: | |
| name: Build sdist | |
| needs: [run_python_tests, run_rust_tests] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@v4 | |
| - name: Setup 3rd party dependencies | |
| uses: ./.github/actions/setup-dependencies | |
| with: | |
| install-test-deps: false | |
| - name: Build sdist | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| - name: Upload sdist | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheels-sdist | |
| path: dist | |
| release: | |
| name: Release | |
| runs-on: ubuntu-latest | |
| if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} | |
| needs: | |
| - build_linux_wheels | |
| - build_macos_wheels | |
| - build_sdist | |
| permissions: | |
| # Use to sign the release artifacts | |
| id-token: write | |
| # Used to upload release artifacts | |
| contents: write | |
| # Used to generate artifact attestation | |
| attestations: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: wheel-* | |
| path: dist | |
| merge-multiple: true | |
| - name: Generate artifact attestation | |
| uses: actions/attest-build-provenance@v1 | |
| with: | |
| subject-path: dist/* | |
| - name: Publish to PyPI | |
| if: ${{ startsWith(github.ref, 'refs/tags/') }} | |
| uses: PyO3/maturin-action@v1 | |
| env: | |
| MATURIN_PYPI_TOKEN: ${{secrets.POETRY_PYPI_TOKEN_PYPI}} | |
| with: | |
| command: upload | |
| args: --non-interactive --skip-existing dist/* | |
| build-image: | |
| if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') | |
| needs: [build_linux_wheels] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - docker: linux/amd64 | |
| wheel: wheels-manylinux2014-x86_64 | |
| - docker: linux/arm64 | |
| wheel: wheels-manylinux2014-aarch64 | |
| steps: | |
| - name: Download wheel | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.platform.wheel }} | |
| path: dist | |
| - name: Prepare | |
| run: | | |
| platform=${{ matrix.platform.docker }} | |
| echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE }} | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=semver,pattern={{version}} | |
| type=sha | |
| - name: Set up QEMU | |
| if: matrix.platform.docker != 'linux/amd64' | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push by digest | |
| id: build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.platform.docker }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| tags: ${{ steps.meta.output.tags }} | |
| outputs: type=image,name=${{ env.DOCKER_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.repository_owner == 'onekey-sec' && 'true' || 'false' }} | |
| - name: Docker container vulnerability scan | |
| id: scan | |
| uses: anchore/scan-action@v6 | |
| with: | |
| image: ${{ env.DOCKER_IMAGE }} | |
| fail-build: false | |
| severity-cutoff: critical | |
| only-fixed: true | |
| - name: Upload SARIF report | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: ${{ steps.scan.outputs.sarif }} | |
| - name: Check unblob - help | |
| run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --help | |
| - name: Check unblob - show-external-dependencies | |
| run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --show-external-dependencies | |
| - name: Check unblob - run for a file with --verbose | |
| run: docker run --rm -v "$(pwd)"/tests/integration/archive/zip/regular:/test ${{ env.DOCKER_IMAGE }}:latest -v -e /tmp /test/__input__/apple.zip | |
| - name: Export digest | |
| run: | | |
| mkdir -p /tmp/digests | |
| digest="${{ steps.build.outputs.digest }}" | |
| touch "/tmp/digests/${digest#sha256:}" | |
| - name: Upload digest | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: digest-${{ env.PLATFORM_PAIR }} | |
| path: /tmp/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| merge-and-push-image: | |
| if: github.repository_owner == 'onekey-sec' && github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-image | |
| steps: | |
| - name: Download digests | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: /tmp/digests | |
| pattern: digest-* | |
| merge-multiple: true | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.DOCKER_IMAGE }} | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=ref,event=branch | |
| type=semver,pattern={{version}} | |
| type=sha | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create manifest list and push | |
| working-directory: /tmp/digests | |
| run: | | |
| docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ | |
| $(printf '${{ env.DOCKER_IMAGE }}@sha256:%s ' *) | |
| - name: Inspect image | |
| run: | | |
| docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }} |