Merge branch 'master' into fix_dynamic_outgoing_edges

.github/CODEOWNERS

@@ -0,0 +1,11 @@
+# See https://help.github.com/articles/about-codeowners/ for syntax
+
+# Core Engineering will be the default owners for everything
+# in the repo. Unless a later match takes precedence,
+# @deepset-ai/core-engineering will be requested for review
+# when someone opens a pull request.
+*            @deepset-ai/core-engineering
+
+# Documentation
+*.md         @deepset-ai/documentation @deepset-ai/core-engineering
+/tutorials/  @deepset-ai/documentation @deepset-ai/core-engineering

.github/utils/convert_notebooks_into_webpages.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import re
 
 from nbconvert import MarkdownExporter
@@ -142,7 +144,7 @@
 id: "tutorial17md"
 --->""",
     18: """<!---
-title: "Tutorial 18"    
+title: "Tutorial 18"
 metaTitle: "GPL Domain Adaptation"
 metaDescription: ""
 slug: "/docs/tutorial18"

.github/utils/generate_json_schema.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import sys
 import logging
 from pathlib import Path

.github/utils/generate_openapi_specs.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import json
 from pathlib import Path
 import os

.github/utils/pydoc-markdown.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e   # Fails on any error in the following loop
+cd docs/_src/api/api/
+for file in ../pydoc/* ; do
+    pydoc-markdown "$file"
+done

.github/utils/tutorials.sh

@@ -7,10 +7,13 @@ python_path=$1
 files_changed=$2
 exclusion_list=$3
 make_python_path_editable=$4
+containers_policy=$5
 no_got_tutorials='4_FAQ_style_QA 5_Evaluation 7_RAG_Generator 8_Preprocessing 10_Knowledge_Graph 15_TableQA 16_Document_Classifier_at_Index_Time'
 
 echo "Files changed in this PR: $files_changed"
 echo "Excluding: $exclusion_list"
+echo "Python path is editable: $make_python_path_editable"
+echo "Containers policy: $containers_policy"
 
 # Collect the tutorials to run
 scripts_to_run=""
@@ -33,9 +36,15 @@ for script in $files_changed; do
     scripts_to_run="$scripts_to_run $script"
 done
 
+
+# Run the containers
+docker run -d -p 9200:9200 --name elasticsearch -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms128m -Xmx256m" elasticsearch:7.9.2
+docker run -d -p 9998:9998 --name tika -e "TIKA_CHILD_JAVA_OPTS=-JXms128m" -e "TIKA_CHILD_JAVA_OPTS=-JXmx128m" apache/tika:1.24.1
+
+
 failed=""
 for script in $scripts_to_run; do
- 
+
     echo ""
     echo "##################################################################################"
     echo "##################################################################################"
@@ -73,15 +82,31 @@ for script in $scripts_to_run; do
     fi
 
     if [[ "$script" == *".py" ]]; then
-        time python $script
+        output=$(time python $script)
     else
-        sudo $python_path/bin/ipython -c "%run $script"
+        output=$(sudo $python_path/bin/ipython -c "%run $script")
     fi
 
-    if [ ! $? -eq 0 ]; then
+    echo $output > $script-output.txt
+    if [ $? -eq 0 ]; then
+        echo "Execution completed successfully."
+    else
+        echo "===================================================="
+        echo "|  $script FAILED!"
+        echo "===================================================="
+        echo "Output of the execution: "
+        echo $output
         failed=$failed" "$script
     fi
 
+    # Restart the necessary containers
+    # Note: Tika does not store data and therefore can be left running
+    if [[ "$make_python_path_editable" == "RESTART" ]]; then
+        docker stop elasticsearch
+        docker rm elasticsearch        
+        docker run -d -p 9200:9200 --name elasticsearch -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms128m -Xmx256m" elasticsearch:7.9.2
+    fi
+
     # Clean up datasets and SQLite DBs to avoid crashing the next tutorial
     git clean -f
 

.github/workflows/black.yml

@@ -0,0 +1,47 @@
+name: Black
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  pull_request:
+
+jobs:
+
+  black:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: ./.github/actions/python_cache/
+
+      - name: Install Haystack
+        run: |
+          pip install --upgrade pip
+          pip install .[dev]
+
+      - name: Check status
+        run: |
+          if ! black . --check; then
+            git status
+            echo "###################################################################################################"
+            echo "# "
+            echo "# CHECK FAILED! Black found issues with your code formatting."
+            echo "# "
+            echo "# Either:"
+            echo "# 1. Run Black locally before committing:"
+            echo "# "
+            echo "#     pip install black==22.6.0"
+            echo "#     black ."
+            echo "# "
+            echo "# 2. Install the pre-commit hook:"
+            echo "# "
+            echo "#     pre-commit install --hook-type pre-push"
+            echo "# "
+            echo "# 3. See https://github.com/deepset-ai/haystack/blob/master/CONTRIBUTING.md for help."
+            echo "# "
+            echo "# If you have further problems, please open an issue: https://github.com/deepset-ai/haystack/issues"
+            echo "# "
+            echo "##################################################################################################"
+            exit 1
+          fi

.github/workflows/documentation.yml

@@ -0,0 +1,49 @@
+name: Documentation
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  pull_request:
+
+jobs:
+
+  api-docs:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: ./.github/actions/python_cache/
+
+      - name: Install Haystack
+        run: |
+          pip install --upgrade pip
+          pip install -U .[dev]
+
+      - name: Update API documentation
+        run: .github/utils/pydoc-markdown.sh
+
+      - name: Check status
+        run: |
+          if [[ `git status --porcelain` ]]; then
+            git status
+            echo "###################################################################################################"
+            echo "# "
+            echo "# CHECK FAILED! The API docs were not updated."
+            echo "# "
+            echo "# Either:"
+            echo "# 1. Generate the new API docs locally before committing:"
+            echo "# "
+            echo "#     .github/utils/pydoc-markdown.sh"
+            echo "# "
+            echo "# 2. Install the pre-commit hook:"
+            echo "# "
+            echo "#     pre-commit install --hook-type pre-push"
+            echo "# "
+            echo "# 3. See https://github.com/deepset-ai/haystack/blob/master/CONTRIBUTING.md for help."
+            echo "# "
+            echo "# If you have further problems, please open an issue: https://github.com/deepset-ai/haystack/issues"
+            echo "# "
+            echo "###################################################################################################"
+            exit 1
+          fi

.github/workflows/schemas.yml

@@ -0,0 +1,96 @@
+name: Schemas
+
+on:
+  workflow_dispatch: # Activate this workflow manually
+  pull_request:
+
+jobs:
+
+  openapi:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: ./.github/actions/python_cache/
+
+      - name: Install Haystack
+        run: |
+          pip install --upgrade pip
+          pip install .[dev]
+          pip install -U rest_api/
+
+      - name: Update OpenAPI specs
+        run: python .github/utils/generate_openapi_specs.py
+
+      - name: Check status
+        run: |
+          if [[ `git status --porcelain` ]]; then
+            git status
+            echo "##################################################################################################"
+            git diff
+            echo "##################################################################################################"
+            echo "# "
+            echo "# CHECK FAILED! OpenAPI specs were not updated."
+            echo "# "
+            echo "# Please generate the new OpenAPI specs locally:"
+            echo "# "
+            echo "#     python .github/utils/generate_openapi_specs.py"
+            echo "# "
+            echo "# You need a FULL INSTALLATION of Haystack for this step to work."
+            echo "# "
+            echo "#     pip install -e .[all]"
+            echo "# "
+            echo "# If you cannot do that for any reason, please disable the hook, push "
+            echo "# and let us know in the PR description."
+            echo "# "
+            echo "# See https://github.com/deepset-ai/haystack/blob/master/CONTRIBUTING.md for instructions."
+            echo "# "
+            echo "# If you have further problems, please open an issue: https://github.com/deepset-ai/haystack/issues"
+            echo "# "
+            echo "###################################################################################################"
+            exit 1
+          fi
+
+  pipeline-yaml:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: ./.github/actions/python_cache/
+
+      - name: Install sndfile
+        run: sudo apt update && sudo apt-get install libsndfile1 ffmpeg
+
+      - name: Install Haystack
+        run: |
+          pip install --upgrade pip
+          pip install -U .[all]
+
+      - name: Update pipeline YAML schemas
+        run: python .github/utils/generate_json_schema.py
+
+      - name: Check status
+        run: |
+          if [[ `git status --porcelain` ]]; then
+            git status
+            echo "##################################################################################################"
+            git diff
+            echo "##################################################################################################"
+            echo "# "
+            echo "# CHECK FAILED! The YAML schemas for pipelines were not updated."
+            echo "# "
+            echo "# Please generate the new schemas locally:"
+            echo "# "
+            echo "#     python .github/utils/generate_json_schema.py"
+            echo "# "
+            echo "# Or see https://github.com/deepset-ai/haystack/blob/master/CONTRIBUTING.md for help."
+            echo "# "
+            echo "# If you have further problems, please open an issue: https://github.com/deepset-ai/haystack/issues"
+            echo "# "
+            echo "##################################################################################################"
+            exit 1
+          fi