Merge branch 'main' into add_pinecone

.github/ISSUE_TEMPLATE/new-integration-proposal.md

@@ -9,23 +9,25 @@ assignees: ''
 
 ## Summary and motivation
 
-Briefly explain the feature request: why do we need this feature? What use cases does it support?
-
-## Alternatives
-
-A clear and concise description of any alternative solutions or features you've considered.
+Briefly explain the request: why do we need this integration? What use cases does it support?
 
 ## Detailed design
 
-Explain the design in enough detail for somebody familiar with Haystack to understand, and for somebody familiar with the implementation to implement. Get into specifics and corner-cases, and include examples of how the feature is used. Also, if there's any new terminology involved, define it here.
+Explain the design in enough detail for somebody familiar with Haystack to understand, and for somebody familiar with
+the implementation to implement. Get into specifics and corner-cases, and include examples of how the feature is used.
+Also, if there's any new terminology involved, define it here.
 
 ## Checklist
 
-If the feature request is accepted, ensure the following checklist is complete before closing the issue.
+If the request is accepted, ensure the following checklist is complete before closing this issue.
 
-- [ ] The package has been released on PyPI
+- [ ] The code was merged in the `main` branch
+- [ ] Docs are published at https://docs.haystack.deepset.ai/
 - [ ] There is a Github workflow running the tests for the integration nightly and at every PR
 - [ ] A label named like `integration:<your integration name>` has been added to this repo
 - [ ] The [labeler.yml](https://github.com/deepset-ai/haystack-core-integrations/blob/main/.github/labeler.yml) file has been updated
+- [ ] The package has been released on PyPI
 - [ ] An integration tile has been added to https://github.com/deepset-ai/haystack-integrations
 - [ ] The integration has been listed in the [Inventory section](https://github.com/deepset-ai/haystack-core-integrations#inventory) of this repo README
+- [ ] There is an example available to demonstrate the feature
+- [ ] The feature was announced through social media

.github/labeler.yml

@@ -1,24 +1,67 @@
 # Integrations
 integration:chroma:
-- integrations/chroma/**/*
+  - changed-files:
+      - any-glob-to-any-file: "integrations/chroma/**/*"
+      - any-glob-to-any-file: ".github/workflows/chroma.yml"
+
+integration:cohere:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/cohere/**/*"
+      - any-glob-to-any-file: ".github/workflows/cohere.yml"
+
 integration:elasticsearch:
-- integrations/elasticsearch/**/*
+  - changed-files:
+      - any-glob-to-any-file: "integrations/elasticsearch/**/*"
+      - any-glob-to-any-file: ".github/workflows/elasticsearch.yml"
+
+integration:google-vertex:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/google-vertex/**/*"
+      - any-glob-to-any-file: ".github/workflows/google_vertex.yml"
+
 integration:gradient:
-- integrations/gradient/**/*
+  - changed-files:
+      - any-glob-to-any-file: "integrations/gradient/**/*"
+      - any-glob-to-any-file: ".github/workflows/gradient.yml"
+
 integration:instructor-embedders:
-- integrations/instructor-embedders/**/*
+  - changed-files:
+      - any-glob-to-any-file: "integrations/instructor-embedders/**/*"
+      - any-glob-to-any-file: ".github/workflows/instructor_embedders.yml"
+
+integration:jina:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/jina/**/*"
+      - any-glob-to-any-file: ".github/workflows/jina.yml"
+
 integration:opensearch:
-- integrations/opensearch/**/*
+  - changed-files:
+      - any-glob-to-any-file: "integrations/opensearch/**/*"
+      - any-glob-to-any-file: ".github/workflows/opensearch.yml"
+
+integration:qdrant:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/qdrant/**/*"
+      - any-glob-to-any-file: ".github/workflows/qdrant.yml"
+
 integration:unstructured-fileconverter:
-- integrations/unstructured/fileconverter/**/*
+  - changed-files:
+      - any-glob-to-any-file: "integrations/unstructured/fileconverter/**/*"
+      - any-glob-to-any-file: ".github/workflows/unstructured_fileconverter.yml"
+
 # Topics
 topic:CI:
-- .github/*
-- .github/**/*
+  - changed-files:
+      - any-glob-to-any-file: [".github/*", ".github/**/*"]
 topic:DX:
-- CONTRIBUTING.md
-- .pre-commit-config.yaml
-- .gitignore
-- requirements.txt
+  - changed-files:
+      - any-glob-to-any-file:
+          [
+            "CONTRIBUTING.md",
+            ".pre-commit-config.yaml",
+            ".gitignore",
+            "requirements.txt",
+          ]
 topic:security:
-- SECURITY.md
+  - changed-files:
+      - any-glob-to-any-file: ["SECURITY.md"]

.github/workflows/chroma.yml

@@ -41,7 +41,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/cohere.yml

@@ -41,7 +41,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/elasticsearch.yml

@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/google_vertex.yml

@@ -0,0 +1,56 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / google-vertex
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - "integrations/google-vertex/**"
+      - ".github/workflows/google-vertex.yml"
+
+defaults:
+  run:
+    working-directory: integrations/google-vertex
+
+concurrency:
+  group: google-vertex-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.9", "3.10"]
+
+    steps:
+      - name: Support longpaths
+        if: matrix.os == 'windows-latest'
+        working-directory: .
+        run: git config --system core.longpaths true
+
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Hatch
+        run: pip install --upgrade hatch
+
+      - name: Lint
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run lint:all
+
+      - name: Run tests
+        run: hatch run cov

.github/workflows/gradient.yml

@@ -41,7 +41,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/instructor_embedders.yml

@@ -20,23 +20,15 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Setup Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
 
-    - name: Ruff
-      uses: chartboost/ruff-action@v1
-      with:
-        src: integrations/instructor-embedders
-
-    - name: Install instructor-embedders
-      run: |
-        pip install -e .[dev]
+    - name: Install Hatch
+      run: pip install --upgrade hatch
 
-    - name: Run unit tests
-      run: |
-        pytest -v -m unit
+    - name: Lint
+      run: hatch run lint:all
 
-    - name: Run integration tests
-      run: |
-        pytest -v -m integration
+    - name: Run tests
+      run: hatch run cov

.github/workflows/jina.yml

@@ -0,0 +1,56 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / jina
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - 'integrations/jina/**'
+      - '.github/workflows/jina.yml'
+
+defaults:
+  run:
+    working-directory: integrations/jina
+
+concurrency:
+  group: jina-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.9', '3.10']
+
+    steps:
+    - name: Support longpaths
+      if: matrix.os == 'windows-latest'
+      working-directory: .
+      run: git config --system core.longpaths true
+
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Hatch
+      run: pip install --upgrade hatch
+
+    - name: Lint
+      if: matrix.python-version == '3.9' && runner.os == 'Linux'
+      run: hatch run lint:all
+
+    - name: Run tests
+      run: hatch run cov

.github/workflows/labeler.yml

@@ -10,6 +10,6 @@ jobs:
   triage:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@v4
+    - uses: actions/labeler@v5
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/nodes_text2speech.yml

@@ -18,7 +18,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Setup Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
 

.github/workflows/opensearch.yml

@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/qdrant.yml

@@ -0,0 +1,56 @@
+# This workflow comes from https://github.com/ofek/hatch-mypyc
+# https://github.com/ofek/hatch-mypyc/blob/5a198c0ba8660494d02716cfc9d79ce4adfb1442/.github/workflows/test.yml
+name: Test / qdrant
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  pull_request:
+    paths:
+      - 'integrations/qdrant/**'
+      - '.github/workflows/qdrant.yml'
+
+defaults:
+  run:
+    working-directory: integrations/qdrant
+
+concurrency:
+  group: qdrant-${{ github.head_ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONUNBUFFERED: "1"
+  FORCE_COLOR: "1"
+
+jobs:
+  run:
+    name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.9', '3.10']
+
+    steps:
+    - name: Support longpaths
+      if: matrix.os == 'windows-latest'
+      working-directory: .
+      run: git config --system core.longpaths true
+
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Hatch
+      run: pip install --upgrade hatch
+
+    - name: Lint
+      if: matrix.python-version == '3.9' && runner.os == 'Linux'
+      run: hatch run lint:all
+
+    - name: Run tests
+      run: hatch run cov

.github/workflows/unstructured_fileconverter.yml

@@ -36,13 +36,13 @@ jobs:
           --health-cmd "curl --fail http://localhost:8000/healthcheck || exit 1"
           --health-interval 10s
           --health-timeout 1s
-          --health-retries 10                  
+          --health-retries 10
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}