Release candidate langsmith-pyo3 Python package (#1254)

Release candidate of a `langsmith-pyo3` Python package aimed to speed up
trace processing and submissions. Currently about 50% faster than the
pure Python implementation, and likely that's just a starting point for
further performance tuning.

Includes wheels for all operating systems and major platforms, and
trusted publishing to PyPI with attestations.

.github/workflows/build_langsmith_pyo3_wheels.yml

@@ -1,16 +1,238 @@
-name: Build langsmith_pyo3 wheels
+# This file is based on a starter autogenerated by maturin v1.7.4,
+# which can be optained by running `maturin generate-ci github`.
+#
+# The autogenerated file makes some assumptions that don't work for this repo.
+# For example, it assumes that the PyO3 project is the only piece of code in the repo.
+# This is why the file has been hand-edited after generation.
+#
+# To find the changes applied on top of the autogenerated contents, diff this file
+# versus commit `c746f00fadb0c84d769be117643683d60eb07ba3` which has the raw autogenerated output.
+name: Build langsmith-pyo3 wheels
 
+# Our wheels build depends on both the `langsmith-pyo3` crate itself and all its dependencies:
+# - The Rust workspace-level dependency specification in `rust/Cargo.toml` and `rust/Cargo.lock`.
+# - Other local Rust crates, like `langsmith-tracing-client`.
+# - The vendored `orjson` and `pyo3` workspaces.
 on:
   push:
     branches:
       - main
+    tags:
+      - 'langsmith-pyo3==*'
+    paths:
+      - "rust/**"
+      - "vendor/orjson/**"
+      - "vendor/pyo3/**"
   pull_request:
-    branches:
-      - main
+    paths:
+      - "rust/**"
+      - "vendor/orjson/**"
+      - "vendor/pyo3/**"
   workflow_dispatch:
 
+permissions:
+  contents: read
+
+env:
+  RUST_VERSION: '1.82'  # Be careful, "stable" gets you "whatever GitHub ships", which is quite old.
+  SUPPORTED_PYTHON_VERSIONS: 'python3.8 python3.9 python3.10 python3.11 python3.12'
+  WORKING_DIRECTORY: rust/crates/langsmith-pyo3
+
 jobs:
-  hello-world:
-    runs-on: ubuntu-20.04
+  linux:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-latest
+            target: x86_64
+          - runner: ubuntu-latest
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+          rust-toolchain: ${{ env.RUST_VERSION }}
+          target: ${{ matrix.platform.target }}
+          args: '--release --out dist --interpreter ${{ env.SUPPORTED_PYTHON_VERSIONS }}'
+          sccache: 'true'
+          manylinux: '2_28'  # The default is 'auto' AKA '2014', which is too old for us.
+          # Workaround for missing `pip` in manylinux_2_28:
+          # https://github.com/PyO3/maturin-action/issues/249
+          before-script-linux: '(python3 -m pip --version || python3 -m ensurepip)'
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-${{ matrix.platform.target }}
+          path: ${{ env.WORKING_DIRECTORY }}/dist
+          if-no-files-found: error
+
+  musllinux:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-latest
+            target: x86_64
+          - runner: ubuntu-latest
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+          rust-toolchain: ${{ env.RUST_VERSION }}
+          target: ${{ matrix.platform.target }}
+          args: '--release --out dist --interpreter ${{ env.SUPPORTED_PYTHON_VERSIONS }}'
+          sccache: 'true'
+          manylinux: musllinux_1_2
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-musllinux-${{ matrix.platform.target }}
+          path: ${{ env.WORKING_DIRECTORY }}/dist
+          if-no-files-found: error
+
+  windows:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: windows-latest
+            target: x64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+          architecture: ${{ matrix.platform.target }}
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+          rust-toolchain: ${{ env.RUST_VERSION }}
+          target: ${{ matrix.platform.target }}
+          args: '--release --out dist --interpreter ${{ env.SUPPORTED_PYTHON_VERSIONS }}'
+          sccache: 'true'
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-windows-${{ matrix.platform.target }}
+          path: ${{ env.WORKING_DIRECTORY }}/dist
+          if-no-files-found: error
+
+  macos:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: macos-14
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+          rust-toolchain: ${{ env.RUST_VERSION }}
+          target: ${{ matrix.platform.target }}
+          args: '--release --out dist --interpreter ${{ env.SUPPORTED_PYTHON_VERSIONS }}'
+          sccache: 'true'
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-${{ matrix.platform.target }}
+          path: ${{ env.WORKING_DIRECTORY }}/dist
+          if-no-files-found: error
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          working-directory: ${{ env.WORKING_DIRECTORY }}
+          rust-toolchain: ${{ env.RUST_VERSION }}
+          command: sdist
+          args: --out dist
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-sdist
+          path: ${{ env.WORKING_DIRECTORY }}/dist
+          if-no-files-found: error
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/langsmith-pyo3==') || github.event_name == 'workflow_dispatch' }}
+    needs: [linux, musllinux, windows, macos, sdist]
+    permissions:
+      # Use to sign the release artifacts
+      id-token: write
+      # Used to upload release artifacts
+      contents: write
+      # Used to generate artifact attestation
+      attestations: write
     steps:
-      - run: echo 'hello world'
+      - uses: actions/download-artifact@v4
+        with:
+          path: ${{ env.WORKING_DIRECTORY }}
+      - name: Move wheels to 'dist' dir
+        run: |
+          set -euxo pipefail
+
+          # Show what wheels got built.
+          cd "$WORKING_DIRECTORY"
+          ls -alh wheels-*
+
+          # Wipe the `dist` directory if it already existed, so we don't accidentally publish
+          # something we didn't intend to. Then, move the wheels into a fresh `dist` directory.
+          rm -rf dist
+          mkdir dist
+          mv wheels-*/* dist/
+          ls -alh dist/
+      - name: Ensure the tagged version matches the wheel versions
+        if: startsWith(github.ref, 'refs/tags/langsmith-pyo3==')
+        env:
+          REF_NAME: ${{ github.ref }}
+        run: |
+          cd "$WORKING_DIRECTORY"
+
+          # - Look up the first wheel file
+          # - Select just the filename,
+          #   for example `langsmith_pyo3-0.1.0rc1-cp313-cp313-manylinux_2_34_x86_64.whl`
+          # - Split on dashes, and extract the version field (the second field).
+          EXPECTED_VERSION="$(find dist/ -name 'langsmith_pyo3-*.whl' -printf '%f\n' | head -1 | cut -d- -f2)"
+          EXPECTED_REF="refs/tags/langsmith-pyo3==${EXPECTED_VERSION}"
+
+          if [[ "$REF_NAME" != "$EXPECTED_REF" ]]; then
+            echo 'Current tag does not match the expected tag for the wheel versions being published!'
+            echo "Expected ref name: ${EXPECTED_REF}"
+            echo "Actual ref name:   ${REF_NAME}"
+            echo ''
+            echo 'Something is wrong, so refusing to publish.'
+            exit 1
+          fi
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-path: ${{ env.WORKING_DIRECTORY }}/dist/*
+      - name: Publish package distributions to PyPI
+        if: startsWith(github.ref, 'refs/tags/langsmith-pyo3==')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: ${{ env.WORKING_DIRECTORY }}/dist

.github/workflows/link-check.yml

@@ -18,4 +18,8 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
       - name: Check links in Markdown files
-        uses: gaurav-nelson/github-action-markdown-link-check@v1
+        uses: gaurav-nelson/github-action-markdown-link-check@v1
+        with:
+          # Do not check the `vendor/` directory.
+          folder-path: .github,js,openapi,python,rust
+          file-path: ./README.md

python/bench/.gitignore

@@ -0,0 +1,2 @@
+# Profiler output files.
+*.json