Create release workflow

.github/workflows/_release.yml

@@ -3,31 +3,36 @@ on:
   push:
     branches:
       - main
+      - release
+  pull_request:
+    branches: [ release ]
 
 env:
   PYTHON_VERSION: "3.10"
   POETRY_VERSION: "1.7.1"
 
 jobs:
-  release-please:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-    outputs:
-      releases_created: ${{ steps.release.outputs.release_created }}
-      paths_released: ${{ steps.release.outputs.paths_released }}
-      prs_created: ${{ steps.release.outputs.prs_created }}
-      prs: ${{ steps.release.outputs.prs }}
-    steps:
-      - uses: google-github-actions/release-please-action@v4
-        id: release
-        with:
-          manifest-file: ".release-please-manifest.json"
-          config-file: "release-please-config.json"
+  prebuild-checks:
+    uses: ./.github/workflows/prebuild.yml
+    secrets: inherit
+  # release-please:
+  #   runs-on: ubuntu-latest
+  #   permissions:
+  #     contents: write
+  #     pull-requests: write
+  #   outputs:
+  #     releases_created: ${{ steps.release.outputs.release_created }}
+  #     paths_released: ${{ steps.release.outputs.paths_released }}
+  #     prs_created: ${{ steps.release.outputs.prs_created }}
+  #     prs: ${{ steps.release.outputs.prs }}
+  #   steps:
+  #     - uses: google-github-actions/release-please-action@v4
+  #       id: release
+  #       with:
+  #         manifest-file: ".release-please-manifest.json"
+  #         config-file: "release-please-config.json"
   build:
-    needs: release-please
-    if: ${{ needs.release-please.outputs.releases_created }}
+    needs: prebuild-checks
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -56,171 +61,168 @@ jobs:
           echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
           echo version="$(poetry version --short)" >> $GITHUB_OUTPUT
 
-      # We want to keep this build stage *separate* from the release stage,
-      # so that there's no sharing of permissions between them.
-      # The release stage has trusted publishing and GitHub repo contents write access,
-      # and we want to keep the scope of that access limited just to the release job.
-      # Otherwise, a malicious `build` step (e.g. via a compromised dependency)
-      # could get access to our GitHub or PyPI credentials.
-      #
-      # Per the trusted publishing GitHub Action:
-      # > It is strongly advised to separate jobs for building [...]
-      # > from the publish job.
-      # https://github.com/pypa/gh-action-pypi-publish#non-goals
+  #     # We want to keep this build stage *separate* from the release stage,
+  #     # so that there's no sharing of permissions between them.
+  #     # The release stage has trusted publishing and GitHub repo contents write access,
+  #     # and we want to keep the scope of that access limited just to the release job.
+  #     # Otherwise, a malicious `build` step (e.g. via a compromised dependency)
+  #     # could get access to our GitHub or PyPI credentials.
+  #     #
+  #     # Per the trusted publishing GitHub Action:
+  #     # > It is strongly advised to separate jobs for building [...]
+  #     # > from the publish job.
+  #     # https://github.com/pypa/gh-action-pypi-publish#non-goals
 
   test-pypi-publish:
     needs:
-      - build
+      - prebuild-checks
     uses:
       ./.github/workflows/_test_release.yml
     secrets: inherit
 
-  pre-release-checks:
-    needs:
-      - build
-      - test-pypi-publish
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
+  # pre-release-checks:
+  #   needs:
+  #     - build
+  #     - test-pypi-publish
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
 
-      # We explicitly *don't* set up caching here. This ensures our tests are
-      # maximally sensitive to catching breakage.
-      #
-      # For example, here's a way that caching can cause a falsely-passing test:
-      # - Make the langchain package manifest no longer list a dependency package
-      #   as a requirement. This means it won't be installed by `pip install`,
-      #   and attempting to use it would cause a crash.
-      # - That dependency used to be required, so it may have been cached.
-      #   When restoring the venv packages from cache, that dependency gets included.
-      # - Tests pass, because the dependency is present even though it wasn't specified.
-      # - The package is published, and it breaks on the missing dependency when
-      #   used in the real world.
+  #     # We explicitly *don't* set up caching here. This ensures our tests are
+  #     # maximally sensitive to catching breakage.
+  #     #
+  #     # For example, here's a way that caching can cause a falsely-passing test:
+  #     # - Make the langchain package manifest no longer list a dependency package
+  #     #   as a requirement. This means it won't be installed by `pip install`,
+  #     #   and attempting to use it would cause a crash.
+  #     # - That dependency used to be required, so it may have been cached.
+  #     #   When restoring the venv packages from cache, that dependency gets included.
+  #     # - Tests pass, because the dependency is present even though it wasn't specified.
+  #     # - The package is published, and it breaks on the missing dependency when
+  #     #   used in the real world.
 
-      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
-        uses: "./.github/actions/poetry_setup"
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          poetry-version: ${{ env.POETRY_VERSION }}
-          working-directory: .
+  #     - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+  #       uses: "./.github/actions/poetry_setup"
+  #       with:
+  #         python-version: ${{ env.PYTHON_VERSION }}
+  #         poetry-version: ${{ env.POETRY_VERSION }}
+  #         working-directory: .
 
-      - name: Import published package
-        shell: bash
-        working-directory: .
-        env:
-          PKG_NAME: ${{ needs.build.outputs.pkg-name }}
-          VERSION: ${{ needs.build.outputs.version }}
-        # Here we use:
-        # - The default regular PyPI index as the *primary* index, meaning 
-        #   that it takes priority (https://pypi.org/simple)
-        # - The test PyPI index as an extra index, so that any dependencies that
-        #   are not found on test PyPI can be resolved and installed anyway.
-        #   (https://test.pypi.org/simple). This will include the PKG_NAME==VERSION
-        #   package because VERSION will not have been uploaded to regular PyPI yet.
-        # - attempt install again after 5 seconds if it fails because there is
-        #   sometimes a delay in availability on test pypi
-        run: |
-          poetry run pip install \
-            --extra-index-url https://test.pypi.org/simple/ \
-            "$PKG_NAME==$VERSION" || \
-          ( \
-            sleep 5 && \
-            poetry run pip install \
-              --extra-index-url https://test.pypi.org/simple/ \
-              "$PKG_NAME==$VERSION" \
-          )
+  #     - name: Import published package
+  #       shell: bash
+  #       working-directory: .
+  #       env:
+  #         PKG_NAME: ${{ needs.build.outputs.pkg-name }}
+  #         VERSION: ${{ needs.build.outputs.version }}
+  #       # Here we use:
+  #       # - The default regular PyPI index as the *primary* index, meaning 
+  #       #   that it takes priority (https://pypi.org/simple)
+  #       # - The test PyPI index as an extra index, so that any dependencies that
+  #       #   are not found on test PyPI can be resolved and installed anyway.
+  #       #   (https://test.pypi.org/simple). This will include the PKG_NAME==VERSION
+  #       #   package because VERSION will not have been uploaded to regular PyPI yet.
+  #       # - attempt install again after 5 seconds if it fails because there is
+  #       #   sometimes a delay in availability on test pypi
+  #       run: |
+  #         poetry run pip install \
+  #           --extra-index-url https://test.pypi.org/simple/ \
+  #           "$PKG_NAME==$VERSION" || \
+  #         ( \
+  #           sleep 5 && \
+  #           poetry run pip install \
+  #             --extra-index-url https://test.pypi.org/simple/ \
+  #             "$PKG_NAME==$VERSION" \
+  #         )
 
-          # Replace all dashes in the package name with underscores,
-          # since that's how Python imports packages with dashes in the name.
-          IMPORT_NAME="$(echo "$PKG_NAME" | sed s/-/_/g)"
+  #         # Replace all dashes in the package name with underscores,
+  #         # since that's how Python imports packages with dashes in the name.
+  #         IMPORT_NAME="$(echo "$PKG_NAME" | sed s/-/_/g)"
 
-          poetry run python -c "import $IMPORT_NAME; print(dir($IMPORT_NAME))"
+  #         poetry run python -c "import $IMPORT_NAME; print(dir($IMPORT_NAME))"
 
-      - name: Import test dependencies
-        run: poetry install --with test,test_integration
-        working-directory: .
+  #     - name: Import test dependencies
+  #       run: poetry install --with test,test_integration
+  #       working-directory: .
 
-      # Overwrite the local version of the package with the test PyPI version.
-      - name: Import published package (again)
-        working-directory: .
-        shell: bash
-        env:
-          PKG_NAME: ${{ needs.build.outputs.pkg-name }}
-          VERSION: ${{ needs.build.outputs.version }}
-        run: |
-          poetry run pip install \
-            --extra-index-url https://test.pypi.org/simple/ \
-            "$PKG_NAME==$VERSION"
+  #     # Overwrite the local version of the package with the test PyPI version.
+  #     - name: Import published package (again)
+  #       working-directory: .
+  #       shell: bash
+  #       env:
+  #         PKG_NAME: ${{ needs.build.outputs.pkg-name }}
+  #         VERSION: ${{ needs.build.outputs.version }}
+  #       run: |
+  #         poetry run pip install \
+  #           --extra-index-url https://test.pypi.org/simple/ \
+  #           "$PKG_NAME==$VERSION"
 
-      - name: Run unit tests
-        run: make tests
-        working-directory: .
+  #     - name: Run unit tests
+  #       run: make tests
+  #       working-directory: .
 
-      - name: 'Authenticate to Google Cloud'
-        id: 'auth'
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+  #     - name: 'Authenticate to Google Cloud'
+  #       id: 'auth'
+  #       uses: google-github-actions/auth@v2
+  #       with:
+  #         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
 
-      - name: Run integration tests
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        run: make integration_tests
-        working-directory: .
+  #     - name: Run integration tests
+  #       env:
+  #         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  #       run: make integration_tests
+  #       working-directory: .
 
-      # - name: Run unit tests with minimum dependency versions
-      #   if: ${{ (inputs.working-directory == 'libs/langchain') || (inputs.working-directory == 'libs/community') || (inputs.working-directory == 'libs/experimental') }}
-      #   run: |
-      #     poetry run pip install -r _test_minimum_requirements.txt
-      #     make tests
-      #   working-directory: .
+  #     # - name: Run unit tests with minimum dependency versions
+  #     #   if: ${{ (inputs.working-directory == 'libs/langchain') || (inputs.working-directory == 'libs/community') || (inputs.working-directory == 'libs/experimental') }}
+  #     #   run: |
+  #     #     poetry run pip install -r _test_minimum_requirements.txt
+  #     #     make tests
+  #     #   working-directory: .
 
-  publish:
-    needs:
-      - build
-      - test-pypi-publish
-      - pre-release-checks
-    runs-on: ubuntu-latest
-    permissions:
-      # This permission is used for trusted publishing:
-      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
-      #
-      # Trusted publishing has to also be configured on PyPI for each package:
-      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
-      id-token: write
+  # publish:
+  #   needs:
+  #     - build
+  #     - test-pypi-publish
+  #   runs-on: ubuntu-latest
+  #   permissions:
+  #     # This permission is used for trusted publishing:
+  #     # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
+  #     #
+  #     # Trusted publishing has to also be configured on PyPI for each package:
+  #     # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+  #     id-token: write
 
-    defaults:
-      run:
-        working-directory: .
+  #   defaults:
+  #     run:
+  #       working-directory: .
 
-    steps:
-      - uses: actions/checkout@v4
+  #   steps:
+  #     - uses: actions/checkout@v4
 
-      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
-        uses: "./.github/actions/poetry_setup"
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          poetry-version: ${{ env.POETRY_VERSION }}
-          working-directory: .
-          cache-key: release
+  #     - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+  #       uses: "./.github/actions/poetry_setup"
+  #       with:
+  #         python-version: ${{ env.PYTHON_VERSION }}
+  #         poetry-version: ${{ env.POETRY_VERSION }}
+  #         working-directory: .
+  #         cache-key: release
 
-      - uses: actions/download-artifact@v3
-        with:
-          name: dist
-          path: ./dist/
+  #     - uses: actions/download-artifact@v3
+  #       with:
+  #         name: dist
+  #         path: ./dist/
 
-      - name: Publish package distributions to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages-dir: ./dist/
-          verbose: true
-          print-hash: true
+  #     - name: Publish package distributions to PyPI
+  #       uses: pypa/gh-action-pypi-publish@release/v1
+  #       with:
+  #         packages-dir: ./dist/
+  #         verbose: true
+  #         print-hash: true
 
   mark-release:
     needs:
+      # - publish
       - build
-      - test-pypi-publish
-      - pre-release-checks
-      - publish
     runs-on: ubuntu-latest
     permissions:
       # This permission is needed by `ncipollo/release-action` to
@@ -247,13 +249,13 @@ jobs:
           name: dist
           path: ./dist/
 
-      # - name: Create Release
-      #   uses: ncipollo/release-action@v1
-      #   if: ${{ inputs.working-directory == 'libs/langchain' }}
-      #   with:
-      #     artifacts: "dist/*"
-      #     token: ${{ secrets.GITHUB_TOKEN }}
-      #     draft: false
-      #     generateReleaseNotes: true
-      #     tag: v${{ needs.build.outputs.version }}
-      #     commit: master
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        # if: ${{ inputs.working-directory == 'libs/langchain-weaviate' }}
+        with:
+          artifacts: "dist/*"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: false
+          generateReleaseNotes: true
+          tag: v${{ needs.build.outputs.version }}
+          commit: release

.github/workflows/_test_release.yml

@@ -9,7 +9,7 @@ env:
 
 jobs:
   build:
-    if: github.ref == 'refs/heads/main'
+    # if: github.ref == 'refs/heads/main' # not necessary since we only release on tags pushed to main
     runs-on: ubuntu-latest
 
     outputs: