Nightly - run unit tests with Haystack main branch (#580)

* first refactorings

* squash

* retry

* remove quotes

* try failure

* another try

* try notification

* progress

* update all integrations

* run CI

* retry fastembed

* update all integrations

* add missing generate docs to cohere

* remove one notification step

* retry

* fix wrong syntax

* try a better workflow

* another try

* ternary op

* make it work

* rm condition

.github/workflows/amazon_bedrock.yml

@@ -68,4 +68,22 @@ jobs:
           role-to-assume: ${{ secrets.AWS_CI_ROLE_ARN }}
 
       - name: Run tests
+        id: tests
         run: hatch run cov
+
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+     
+      - name: Send event to Datadog for nightly failures
+        if: failure() && github.event_name == 'schedule'
+        uses: ./.github/actions/send_failure
+        with:
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/amazon_sagemaker.yml

@@ -57,4 +57,22 @@ jobs:
         run: hatch run docs        
 
       - name: Run tests
+        id: tests
         run: hatch run cov
+
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
+      - name: Send event to Datadog for nightly failures
+        if: failure() && github.event_name == 'schedule'
+        uses: ./.github/actions/send_failure
+        with:
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/anthropic.yml

@@ -56,9 +56,19 @@ jobs:
       - name: Run tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: github.event_name == 'schedule' && failure()
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
-          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/astra.yml

@@ -61,11 +61,22 @@ jobs:
       env:
         ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_API_ENDPOINT }}
         ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_TOKEN }}
+      id: tests
       run: hatch run cov
 
+    - name: Nightly - run unit tests with Haystack main branch
+      if: github.event_name == 'schedule'
+      id: nightly-haystack-main
+      run: |
+        hatch run pip install git+https://github.com/deepset-ai/haystack.git
+        hatch run test -m "not integration"
+    
     - name: Send event to Datadog for nightly failures
-      if: github.event_name == 'schedule' && failure()
+      if: failure() && github.event_name == 'schedule'
       uses: ./.github/actions/send_failure
       with:
-        title: "core-integrations nightly failure: ${{ github.workflow }}"
+        title: |
+          core-integrations failure: 
+          ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+            - ${{ github.workflow }}
         api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/chroma.yml

@@ -57,11 +57,22 @@ jobs:
       run: hatch run docs
 
     - name: Run tests
+      id: tests
       run: hatch run cov
 
+    - name: Nightly - run unit tests with Haystack main branch
+      if: github.event_name == 'schedule'
+      id: nightly-haystack-main
+      run: |
+        hatch run pip install git+https://github.com/deepset-ai/haystack.git
+        hatch run test -m "not integration"
+    
     - name: Send event to Datadog for nightly failures
-      if: github.event_name == 'schedule' && failure()
+      if: failure() && github.event_name == 'schedule'
       uses: ./.github/actions/send_failure
       with:
-        title: "core-integrations nightly failure: ${{ github.workflow }}"
+        title: |
+          core-integrations failure: 
+          ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+            - ${{ github.workflow }}
         api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/cohere.yml

@@ -53,12 +53,27 @@ jobs:
         if: matrix.python-version == '3.9' && runner.os == 'Linux'
         run: hatch run lint:all
 
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs        
+
       - name: Run tests
+        id: tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: github.event_name == 'schedule' && failure()
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
           api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/deepeval.yml

@@ -57,11 +57,22 @@ jobs:
         run: hatch run docs
 
       - name: Run tests
+        id: tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: github.event_name == 'schedule' && failure()
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
-          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/elasticsearch.yml

@@ -55,11 +55,22 @@ jobs:
         run: hatch run docs
 
       - name: Run tests
+        id: tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: github.event_name == 'schedule' && failure()
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
-          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/fastembed.yml

@@ -42,11 +42,22 @@ jobs:
         run: hatch run docs          
 
       - name: Run tests
+        id: tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: failure() #github.event_name == 'schedule' && 
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
-          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}          
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/google_ai.yml

@@ -58,11 +58,22 @@ jobs:
         run: hatch run docs
 
       - name: Run tests
+        id: tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: github.event_name == 'schedule' && failure()
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
-          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/google_vertex.yml

@@ -57,11 +57,22 @@ jobs:
         run: hatch run docs
 
       - name: Run tests
+        id: tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: github.event_name == 'schedule' && failure()
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
-          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/gradient.yml

@@ -57,11 +57,22 @@ jobs:
       run: hatch run docs
 
     - name: Run tests
+      id: tests
       run: hatch run cov
 
+    - name: Nightly - run unit tests with Haystack main branch
+      if: github.event_name == 'schedule'
+      id: nightly-haystack-main
+      run: |
+        hatch run pip install git+https://github.com/deepset-ai/haystack.git
+        hatch run test -m "not integration"
+    
     - name: Send event to Datadog for nightly failures
-      if: github.event_name == 'schedule' && failure()
+      if: failure() && github.event_name == 'schedule'
       uses: ./.github/actions/send_failure
       with:
-        title: "core-integrations nightly failure: ${{ github.workflow }}"
+        title: |
+          core-integrations failure: 
+          ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+            - ${{ github.workflow }}
         api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/instructor_embedders.yml

@@ -35,11 +35,22 @@ jobs:
         run: hatch run docs
 
       - name: Run tests
+        id: tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: github.event_name == 'schedule' && failure()
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
-          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/jina.yml

@@ -57,11 +57,22 @@ jobs:
         run: hatch run docs
 
       - name: Run tests
+        id: tests
         run: hatch run cov
 
+      - name: Nightly - run unit tests with Haystack main branch
+        if: github.event_name == 'schedule'
+        id: nightly-haystack-main
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git
+          hatch run test -m "not integration"
+      
       - name: Send event to Datadog for nightly failures
-        if: github.event_name == 'schedule' && failure()
+        if: failure() && github.event_name == 'schedule'
         uses: ./.github/actions/send_failure
         with:
-          title: "core-integrations nightly failure: ${{ github.workflow }}"
-          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
+          title: |
+            core-integrations failure: 
+            ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+             - ${{ github.workflow }}
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/llama_cpp.yml

@@ -57,11 +57,22 @@ jobs:
       run: hatch run docs
 
     - name: Run tests
+      id: tests
       run: hatch run cov
 
+    - name: Nightly - run unit tests with Haystack main branch
+      if: github.event_name == 'schedule'
+      id: nightly-haystack-main
+      run: |
+        hatch run pip install git+https://github.com/deepset-ai/haystack.git
+        hatch run test -m "not integration"
+    
     - name: Send event to Datadog for nightly failures
-      if: github.event_name == 'schedule' && failure()
+      if: failure() && github.event_name == 'schedule'
       uses: ./.github/actions/send_failure
       with:
-        title: "core-integrations nightly failure: ${{ github.workflow }}"
+        title: |
+          core-integrations failure: 
+          ${{ (steps.tests.conclusion == 'nightly-haystack-main') && 'nightly-haystack-main' || 'tests' }}
+            - ${{ github.workflow }}
         api-key: ${{ secrets.CORE_DATADOG_API_KEY }}