merge

SciPhi-AI · Oct 22, 2024 · e9f97ff · e9f97ff
1 parent 91c420e
commit e9f97ff
Show file tree

Hide file tree

Showing 146 changed files with 5,538 additions and 3,714 deletions.
diff --git a/.github/actions/run-cli-graphrag-tests/action.yml b/.github/actions/run-cli-graphrag-tests/action.yml
@@ -3,22 +3,27 @@ description: 'Runs CLI graphrag tests for R2R'
 runs:
   using: "composite"
   steps:
-    - name: Ingest sample file (SDK)
+    - name: Ingest sample file (CLI)
       working-directory: ./py
       shell: bash
       run: poetry run python tests/integration/runner_cli.py test_ingest_sample_file_2_cli
 
-    - name: Create the graph (SDK)
+    - name: Create the graph (CLI)
       working-directory: ./py
       shell: bash
       run: poetry run python tests/integration/runner_cli.py test_kg_create_graph_sample_file_cli
 
-    - name: Enrich the graph (SDK)
+    - name: Deduplicate entities (CLI)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_cli.py test_kg_deduplicate_entities_sample_file_cli
+
+    - name: Enrich the graph (CLI)
       working-directory: ./py
       shell: bash
       run: poetry run python tests/integration/runner_cli.py test_kg_enrich_graph_sample_file_cli
 
-    - name: Search over the graph (SDK)
+    - name: Search over the graph (CLI)
       working-directory: ./py
       shell: bash
       run: poetry run python tests/integration/runner_cli.py test_kg_search_sample_file_cli
diff --git a/.github/actions/run-sdk-graphrag-tests/action.yml b/.github/actions/run-sdk-graphrag-tests/action.yml
@@ -13,6 +13,11 @@ runs:
       shell: bash
       run: poetry run python tests/integration/runner_sdk.py test_kg_create_graph_sample_file_sdk
 
+    - name: Deduplicate entities (SDK)
+      working-directory: ./py
+      shell: bash
+      run: poetry run python tests/integration/runner_sdk.py test_kg_deduplicate_entities_sample_file_sdk
+
     - name: Enrich the graph (SDK)
       working-directory: ./py
       shell: bash

diff --git a/.github/actions/setup-postgres-ext/action.yml b/.github/actions/setup-postgres-ext/action.yml
@@ -32,39 +32,60 @@ runs:
 
     - name: Setup PostgreSQL on Windows
       if: inputs.os == 'windows-latest'
-      shell: pwsh
+      shell: cmd
       run: |
-        choco install postgresql15 --params '/Password:postgres' --force
 
-        $env:PATH += ";C:\Program Files\PostgreSQL\15\bin"
-        $env:PGPASSWORD = 'postgres'
+        echo Starting PostgreSQL setup and pgvector installation...
+
+        echo Installing PostgreSQL...
+        choco install postgresql15 --params "/Password:postgres" --force
+
+        echo Updating PATH and setting PGPASSWORD...
+        set PATH=%PATH%;C:\Program Files\PostgreSQL\15\bin
+        set PGPASSWORD=postgres
+        echo PATH updated and PGPASSWORD set.
+
+        echo Altering PostgreSQL user password...
         psql -U postgres -c "ALTER USER postgres PASSWORD 'postgres';"
+        echo PostgreSQL user password altered.
 
-        # Install Visual Studio Build Tools
+        echo Installing Visual Studio Build Tools...
         choco install visualstudio2022buildtools --package-parameters "--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --norestart"
+        echo Visual Studio Build Tools installed.
 
-        # Set up environment for building pgvector
-        $vcvars64Path = "C:\Program Files\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
-        cmd.exe /c "call `"$vcvars64Path`" && set > %temp%\vcvars.txt"
-        Get-Content "$env:temp\vcvars.txt" | Foreach-Object {
-          if ($_ -match "^(.*?)=(.*)$") {
-            Set-Content "env:\$($matches[1])" $matches[2]
-          }
-        }
-
-        # Clone and build pgvector
-        $env:PGROOT = "C:\Program Files\PostgreSQL\15"
-        Set-Location -Path $env:TEMP
+        echo Setting up Visual Studio environment...
+        call "C:\Program Files\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
+        echo Visual Studio environment set up.
+
+        echo Cloning and building pgvector...
+        set PGROOT=C:\Program Files\PostgreSQL\15
+        cd /d %TEMP%
         git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
-        Set-Location -Path "$env:TEMP\pgvector"
+        cd pgvector
+        echo pgvector cloned.
+
+        echo Creating vector extension...
+        psql -U postgres -c "CREATE EXTENSION vector;"
+        echo Vector extension created.
+
+        echo Building pgvector...
         nmake /F Makefile.win
+        echo pgvector built.
+
+        echo Installing pgvector...
         nmake /F Makefile.win install
+        echo pgvector installed.
 
-        psql -U postgres -c "CREATE EXTENSION vector;"
+        echo Setting max_connections to 1024...
+        echo max_connections = 1024 >> "C:\Program Files\PostgreSQL\15\data\postgresql.conf"
+        echo max_connections set.
 
-        # Set max_connections to 1024
-        Add-Content -Path "C:\Program Files\PostgreSQL\15\data\postgresql.conf" -Value "max_connections = 1024"
-        Restart-Service postgresql-x64-15
+        echo Restarting PostgreSQL service...
+        net stop postgresql-x64-15
+        net start postgresql-x64-15
+        echo PostgreSQL service restarted.
+
+        echo Setup complete!
 
     - name: Setup PostgreSQL on macOS
       if: inputs.os == 'macos-latest'

diff --git a/.github/actions/setup-python-full/action.yml b/.github/actions/setup-python-full/action.yml
@@ -26,8 +26,9 @@ runs:
 
     - name: Install Poetry and dependencies on Windows
       if: inputs.os == 'windows-latest'
-      shell: pwsh
+      shell: cmd
       run: |
-        (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python -
-        $env:PATH += ";$env:USERPROFILE\AppData\Roaming\Python\Scripts"
-        cd py; poetry install -E core -E ingestion-bundle
+        python -c "import urllib.request; print(urllib.request.urlopen('https://install.python-poetry.org').read().decode())" > install-poetry.py
+        python install-poetry.py
+        echo %USERPROFILE%\AppData\Roaming\Python\Scripts >> %GITHUB_PATH%
+        cd py && poetry install -E core -E ingestion-bundle
diff --git a/.github/actions/setup-python-light/action.yml b/.github/actions/setup-python-light/action.yml
@@ -21,8 +21,9 @@ runs:
 
     - name: Install Poetry and dependencies on Windows
       if: inputs.os == 'windows-latest'
-      shell: pwsh
+      shell: cmd
       run: |
-        (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python -
-        $env:PATH += ";$env:USERPROFILE\AppData\Roaming\Python\Scripts"
-        cd py; poetry install -E core -E ingestion-bundle
+        python -c "import urllib.request; print(urllib.request.urlopen('https://install.python-poetry.org').read().decode())" > install-poetry.py
+        python install-poetry.py
+        echo %USERPROFILE%\AppData\Roaming\Python\Scripts >> %GITHUB_PATH%
+        cd py && poetry install -E core -E ingestion-bundle
diff --git a/.github/actions/start-r2r-full/action.yml b/.github/actions/start-r2r-full/action.yml
@@ -12,4 +12,4 @@ runs:
     shell: bash
     run: |
       cd py
-      poetry run r2r serve --docker --full --config-name=full
+      poetry run r2r serve --docker --full --config-name=full --build --image=r2r-local
diff --git a/.github/workflows/r2r-full-py-integration-tests-graphrag.yml b/.github/workflows/r2r-full-py-integration-tests-graphrag.yml
@@ -0,0 +1,80 @@
+name: R2R Full Python Integration Test (ubuntu)
+
+on:
+  push:
+    branches:
+      - dev
+      - dev-minor
+  pull_request:
+    branches:
+      - dev
+      - dev-minor
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        test_category:
+          - cli-graphrag
+          - sdk-graphrag
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TELEMETRY_ENABLED: 'false'
+      R2R_PROJECT_NAME: r2r_default
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python and install dependencies
+        uses: ./.github/actions/setup-python-full
+        with:
+          os: ${{ matrix.os }}
+
+      - name: Setup and start Docker
+        uses: ./.github/actions/setup-docker
+
+      - name: Login Docker
+        uses: ./.github/actions/login-docker
+        with:
+          docker_username: ${{ secrets.RAGTORICHES_DOCKER_UNAME }}
+          docker_password: ${{ secrets.RAGTORICHES_DOCKER_TOKEN }}
+
+      - name: Start R2R Full server
+        uses: ./.github/actions/start-r2r-full
+
+      - name: Run CLI Ingestion Tests
+        if: matrix.test_category == 'cli-ingestion'
+        uses: ./.github/actions/run-cli-ingestion-tests
+
+      - name: Run CLI Retrieval Tests
+        if: matrix.test_category == 'cli-retrieval'
+        uses: ./.github/actions/run-cli-retrieval-tests
+
+      - name: Run CLI GraphRAG Tests
+        if: matrix.test_category == 'cli-graphrag'
+        uses: ./.github/actions/run-cli-graphrag-tests
+
+      - name: Run SDK Ingestion Tests
+        if: matrix.test_category == 'sdk-ingestion'
+        uses: ./.github/actions/run-sdk-ingestion-tests
+
+      - name: Run SDK Retrieval Tests
+        if: matrix.test_category == 'sdk-retrieval'
+        uses: ./.github/actions/run-sdk-retrieval-tests
+
+      - name: Run SDK Auth Tests
+        if: matrix.test_category == 'sdk-auth'
+        uses: ./.github/actions/run-sdk-auth-tests
+
+      - name: Run SDK Collections Tests
+        if: matrix.test_category == 'sdk-collections'
+        uses: ./.github/actions/run-sdk-collections-tests
+
+      - name: Run SDK GraphRAG Tests
+        if: matrix.test_category == 'sdk-graphrag'
+        uses: ./.github/actions/run-sdk-graphrag-tests
diff --git a/.github/workflows/r2r-full-py-integration-tests-mac-and-windows.yml b/.github/workflows/r2r-full-py-integration-tests-mac-and-windows.yml
@@ -79,4 +79,4 @@ jobs:
 
       - name: Run SDK GraphRAG Tests
         if: matrix.test_category == 'sdk-graphrag'
-        uses: ./.github/actions/run-cli-graphrag-tests
+        uses: ./.github/actions/run-sdk-graphrag-tests
diff --git a/.github/workflows/r2r-full-py-integration-tests.yml b/.github/workflows/r2r-full-py-integration-tests.yml
@@ -21,12 +21,10 @@ jobs:
         test_category:
           - cli-ingestion
           - cli-retrieval
-          - cli-graphrag
           - sdk-ingestion
           - sdk-retrieval
           - sdk-auth
           - sdk-collections
-          - sdk-graphrag
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       TELEMETRY_ENABLED: 'false'
@@ -83,4 +81,4 @@ jobs:
 
       - name: Run SDK GraphRAG Tests
         if: matrix.test_category == 'sdk-graphrag'
-        uses: ./.github/actions/run-cli-graphrag-tests
+        uses: ./.github/actions/run-sdk-graphrag-tests
diff --git a/.github/workflows/r2r-light-py-integration-tests-graphrag.yml b/.github/workflows/r2r-light-py-integration-tests-graphrag.yml
@@ -0,0 +1,83 @@
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
+
+name: R2R Light Python Integration Test (ubuntu)
+
+on:
+  push:
+    branches:
+      - dev
+      - dev-minor
+  pull_request:
+    branches:
+      - dev
+      - dev-minor
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        test_category:
+          - cli-graphrag
+          - sdk-graphrag
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      TELEMETRY_ENABLED: 'false'
+      R2R_POSTGRES_HOST: localhost
+      R2R_POSTGRES_DBNAME: postgres
+      R2R_POSTGRES_PORT: '5432'
+      R2R_POSTGRES_PASSWORD: postgres
+      R2R_POSTGRES_USER: postgres
+      R2R_PROJECT_NAME: r2r_default
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python and install dependencies
+        uses: ./.github/actions/setup-python-light
+        with:
+          os: ${{ matrix.os }}
+
+      - name: Setup and start PostgreSQL
+        uses: ./.github/actions/setup-postgres-ext
+        with:
+          os: ${{ matrix.os }}
+
+      - name: Start R2R Light server
+        uses: ./.github/actions/start-r2r-light
+
+      - name: Run CLI Ingestion Tests
+        if: matrix.test_category == 'cli-ingestion'
+        uses: ./.github/actions/run-cli-ingestion-tests
+
+      - name: Run CLI Retrieval Tests
+        if: matrix.test_category == 'cli-retrieval'
+        uses: ./.github/actions/run-cli-retrieval-tests
+
+      - name: Run CLI GraphRAG Tests
+        if: matrix.test_category == 'cli-graphrag'
+        uses: ./.github/actions/run-cli-graphrag-tests
+
+      - name: Run SDK Ingestion Tests
+        if: matrix.test_category == 'sdk-ingestion'
+        uses: ./.github/actions/run-sdk-ingestion-tests
+
+      - name: Run SDK Retrieval Tests
+        if: matrix.test_category == 'sdk-retrieval'
+        uses: ./.github/actions/run-sdk-retrieval-tests
+
+      - name: Run SDK Auth Tests
+        if: matrix.test_category == 'sdk-auth'
+        uses: ./.github/actions/run-sdk-auth-tests
+
+      - name: Run SDK Collections Tests
+        if: matrix.test_category == 'sdk-collections'
+        uses: ./.github/actions/run-sdk-collections-tests
+
+      - name: Run SDK GraphRAG Tests
+        if: matrix.test_category == 'sdk-graphrag'
+        uses: ./.github/actions/run-sdk-graphrag-tests
diff --git a/.github/workflows/r2r-light-py-integration-tests-mac-and-windows.yml b/.github/workflows/r2r-light-py-integration-tests-mac-and-windows.yml
@@ -82,4 +82,4 @@ jobs:
 
       - name: Run SDK GraphRAG Tests
         if: matrix.test_category == 'sdk-graphrag'
-        uses: ./.github/actions/run-cli-graphrag-tests
+        uses: ./.github/actions/run-sdk-graphrag-tests
diff --git a/.github/workflows/r2r-light-py-integration-tests.yml b/.github/workflows/r2r-light-py-integration-tests.yml
@@ -19,16 +19,14 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest]
         test_category:
           - cli-ingestion
           - cli-retrieval
-          - cli-graphrag
           - sdk-ingestion
           - sdk-retrieval
           - sdk-auth
           - sdk-collections
-          - sdk-graphrag
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       TELEMETRY_ENABLED: 'false'
@@ -86,4 +84,4 @@ jobs:
 
       - name: Run SDK GraphRAG Tests
         if: matrix.test_category == 'sdk-graphrag'
-        uses: ./.github/actions/run-cli-graphrag-tests
+        uses: ./.github/actions/run-sdk-graphrag-tests
diff --git a/docs/api-reference/endpoint/completion.mdx b/docs/api-reference/endpoint/completion.mdx
@@ -0,0 +1,4 @@
+---
+title: 'Completion'
+openapi: 'GET /v2/completion'
+---
diff --git a/docs/api-reference/endpoint/deduplicate_entities.mdx b/docs/api-reference/endpoint/deduplicate_entities.mdx
@@ -0,0 +1,4 @@
+---
+title: Deduplicate Entities
+openapi: 'POST /v2/deduplicate_entities'
+---
diff --git a/docs/api-reference/endpoint/delete_entities_and_triples.mdx b/docs/api-reference/endpoint/delete_entities_and_triples.mdx
@@ -0,0 +1,4 @@
+---
+title: 'Delete graph for collection'
+openapi: 'DELETE /v2/delete_graph_for_collection'
+---
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json