Fixing release check and test docs in concrete-python CI

.github/workflows/concrete_python_release_cpu.yml

@@ -38,25 +38,6 @@ jobs:
           backend: aws
           profile: release
 
-  release-checks:
-    needs: setup-instance
-    runs-on: ${{ needs.setup-instance.outputs.runner-name }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          submodules: recursive
-          fetch-depth: 0
-      - name: Check python api doc is up to date
-        run: ci/scripts/make_apidocs.sh
-      - name: Slack Notification
-        if: ${{ failure() }}
-        continue-on-error: true
-        uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
-        env:
-          SLACK_COLOR: ${{ job.status }}
-          SLACK_MESSAGE: "release-checks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
-
   build-linux-x86:
     strategy:
       matrix:
@@ -138,6 +119,48 @@ jobs:
           SLACK_COLOR: ${{ job.status }}
           SLACK_MESSAGE: "build-linux-x86 finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
 
+  release-checks:
+    strategy:
+      matrix:
+        python-version: ["3.8"]
+    needs: [ build-linux-x86 ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - name: Setup Python
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Download wheels
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }}
+          path: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }}
+      - name: Check documentation
+        run: |
+          WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-linux-x86', matrix.python-version) }}
+          CONCRETE_WHEEL="${WHEEL_DIR}/*.whl" .github/workflows/scripts/make_apidocs.sh
+      - name: Upload docs.patch if failure
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: docs-artifacts
+          path: |
+            docs.patch
+            docs/dev/api
+          retention-days: 3
+      - name: Slack Notification
+        if: ${{ failure() }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "release-checks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
+
   build-macos:
     strategy:
       matrix:

.github/workflows/concrete_python_tests_linux.yml

@@ -80,28 +80,42 @@ jobs:
             set -e
 
             rustup toolchain install nightly-2024-09-30
-            dnf -y install graphviz graphviz-devel
+            dnf -y install graphviz graphviz-devel python3.12 python3.12-pip
+            pip3 install numpy
 
             cd /concrete/frontends/concrete-python
-            make venv
+            make PYTHON=python3 venv
             source .venv/bin/activate
 
             cd /concrete/compilers/concrete-compiler/compiler
             make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) python-bindings
 
             echo "Debug: ccache statistics (after the build):"
             ccache -s
-      - name: Create artifact archive
-        run: |
-            cd build
-            tar czvf artifacts.tgz lib/libConcretelangRuntime.so tools/concretelang/python_packages
+
+            for f in $(find /build/tools/concretelang/python_packages/concretelang_core/ -type l);
+            do
+              cp --remove-destination $(readlink -e $f) $f
+            done
+
+            cd /concrete/frontends/concrete-python
+            export COMPILER_BUILD_DIRECTORY="/build"
+            make whl
       - name: Upload concrete-compiler python-bindings
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: concrete-compiler-python-bindings
           include-hidden-files: true
           retention-days: 3
-          path: build/artifacts.tgz
+          path: |
+            ./build/lib/libConcretelangRuntime.so
+            ./build/tools/concretelang/python_packages/
+      - name: Upload wheel
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: concrete-python-wheel
+          path: frontends/concrete-python/dist/*manylinux*.whl
+          retention-days: 3
       - name: Slack Notification
         if: ${{ failure() && github.ref == 'refs/heads/main' }}
         continue-on-error: true
@@ -110,6 +124,48 @@ jobs:
           SLACK_COLOR: ${{ job.status }}
           SLACK_MESSAGE: "build-python-bindings finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
 
+  test-apidoc-up-to-date:
+    needs: [build-python-bindings]
+    runs-on: ubuntu-latest
+    env:
+      python-version: "3.10"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - name: Setup Python
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: ${{ env.python-version }}
+      - name: Download concrete-compiler python-bindings
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: concrete-python-wheel
+          path: wheel
+      - name: Check documentation
+        run: |
+          export WHEEL_DIR=${{ github.workspace }}/wheel
+          export CONCRETE_WHEEL="${WHEEL_DIR}/*.whl"
+          .github/workflows/scripts/make_apidocs.sh
+      - name: Upload docs.patch if failure
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: docs-artifacts
+          path: |
+            docs.patch
+            docs/dev/api
+          retention-days: 3
+      - name: Slack Notification
+        if: ${{ failure() && github.ref == 'refs/heads/main' }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_MESSAGE: "test-apidoc-up-to-date finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
+
   test-pytest:
     needs: [setup-instance, build-python-bindings]
     runs-on: ${{ needs.setup-instance.outputs.runner-name }}
@@ -119,10 +175,6 @@ jobs:
         with:
           name: concrete-compiler-python-bindings
           path: compiler-artifacts
-      - name: Extract artifacts archive
-        run: |
-          cd compiler-artifacts
-          tar xzvf artifacts.tgz
       - name: Run pytest
         uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
         with:
@@ -159,10 +211,6 @@ jobs:
         with:
           name: concrete-compiler-python-bindings
           path: compiler-artifacts
-      - name: Extract artifacts archive
-        run: |
-          cd compiler-artifacts
-          tar xzvf artifacts.tgz
       - name: Run pytest
         uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
         with:

.github/workflows/scripts/make_apidocs.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+set -e
+
+PYTHON=${PYTHON:-python}
+PIP=${PIP:-${PYTHON} -m pip}
+VENV_DIR=${PWD}/.venv-docs
+
+# Remove old documentation
+rm -rf docs/dev/api/*
+
+# Create virtual env and install concrete and docs tools
+${PYTHON} -m venv "${VENV_DIR}"
+source "${VENV_DIR}"/bin/activate
+if [ -z "${CONCRETE_WHEEL}" ]; then 
+    echo "You must specify the CONCRETE_WHEEL environment variable"
+    exit 1
+fi
+${PIP} install ${CONCRETE_WHEEL}
+${PIP} install lazydocs
+
+# Generate the API doc
+lazydocs --output-path="./docs/dev/api" --overview-file="README.md" --src-base-url="../../" --no-watermark concrete
+
+
+# Update documentation paths
+SED_OPT='-i'
+if [ $(uname) == "Darwin" ]; then
+    SED_OPT='-i ""'
+fi
+
+# Fixing the path issues, to point on files in GitHub
+WHICH_PYTHON_VERSION=$(${PYTHON} --version | cut -f 2 -d " " | cut -f 1-2 -d ".")
+sed $SED_OPT -e "s@../../.venv-docs/lib.*/python$WHICH_PYTHON_VERSION/site-packages/@../../../compilers/concrete-compiler/compiler/lib/Bindings/Python/@g" docs/dev/api/concrete.compiler*.md docs/dev/api/concrete.lang*.md
+sed $SED_OPT -e "s@../../.venv-docs/lib.*/python$WHICH_PYTHON_VERSION/site-packages/@../../frontends/concrete-python/@g" docs/dev/api/concrete.fhe*.md
+
+# Fixing the links in README.md, which fails (missing .'s for some reason): remove the #headers
+sed $SED_OPT -e "[email protected]#module-.*)@.md)@g" docs/dev/api/README.md
+sed $SED_OPT -e "[email protected]#function-.*)@.md)@g" docs/dev/api/README.md
+sed $SED_OPT -e "[email protected]#class-.*)@.md)@g" docs/dev/api/README.md
+
+# Removed the "object addresses" and "function addresses", since they are not constant
+sed $SED_OPT -e "s@object at 0x[a-zA-Z0-9]*@object at ADDRESS@g" docs/*.md
+sed $SED_OPT -e "s@object at 0x[a-zA-Z0-9]*@object at ADDRESS@g" docs/*/*.md
+sed $SED_OPT -e "s@object at 0x[a-zA-Z0-9]*@object at ADDRESS@g" docs/*/*/*.md
+
+sed $SED_OPT -e "s@function Int at 0x[a-zA-Z0-9]*@function Int at ADDRESS@g" docs/*.md
+sed $SED_OPT -e "s@function Int at 0x[a-zA-Z0-9]*@function Int at ADDRESS@g" docs/*/*.md
+sed $SED_OPT -e "s@function Int at 0x[a-zA-Z0-9]*@function Int at ADDRESS@g" docs/*/*/*.md
+
+# FIXME: remove this once the PR has been merged once
+sed $SED_OPT -e "s@https://github.com/zama-ai/concrete-compiler-internal/blob/main/LICENSE.txt@https://github.com/zama-ai/concrete/blob/main/LICENSE.txt@g" ./docs/dev/api/concrete.lang.dialects.md ./docs/dev/api/concrete.compiler.md ./docs/dev/api/concrete.lang.md
+
+# Create the patch file
+git add -N docs/dev/api/*
+git diff docs &> docs.patch
+
+# Was there changes?
+if [ ! -s docs.patch ]; then
+    echo "The documentation us up to date, congrats."
+    exit 0
+else
+    echo "There is a difference in the docs, please commit the changes, here the change:"
+    cat docs.patch
+    exit 1
+fi
+