Merge branch 'main' of https://github.com/neo4j/neo4j-graphrag-python …

…into feature/improved-schema-definition # Conflicts: # src/neo4j_graphrag/utils.py
neo4j · Nov 5, 2024 · f02fa10 · f02fa10
2 parents 9cd5acc + 18e8e2a
commit f02fa10
Show file tree

Hide file tree

Showing 165 changed files with 8,540 additions and 3,359 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @neo4j/team-gen-ai
diff --git a/.github/workflows/pr-e2e-tests.yaml b/.github/workflows/pr-e2e-tests.yaml
@@ -16,10 +16,8 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.9', '3.12']
-        neo4j-version:
-          - 5
-        neo4j-edition:
-          - enterprise
+        neo4j-tag:
+          - 'latest'
     services:
       t2v-transformers:
         image: cr.weaviate.io/semitechnologies/transformers-inference:sentence-transformers-all-MiniLM-L6-v2-onnx
@@ -37,14 +35,18 @@ jobs:
           - 8080:8080
           - 50051:50051
       neo4j:
-        image: neo4j:${{ matrix.neo4j-version }}-${{ matrix.neo4j-edition }}
+        image: neo4j:${{ matrix.neo4j-tag }}
         env:
           NEO4J_AUTH: neo4j/password
           NEO4J_ACCEPT_LICENSE_AGREEMENT: 'eval'
           NEO4J_PLUGINS: '["apoc"]'
         ports:
           - 7687:7687
           - 7474:7474
+      qdrant:
+        image: qdrant/qdrant
+        ports:
+          - 6333:6333
 
     steps:
       - name: Install graphviz package
@@ -75,7 +77,7 @@ jobs:
           key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-cache --with dev
+        run: poetry install --no-interaction --no-cache --with dev --all-extras
       - name: Clear Poetry cache
         run: poetry cache clear --all .
       - name: Show disk usage after Poetry installation
@@ -89,7 +91,7 @@ jobs:
       - name: Run tests
         shell: bash
         run: |
-          if [[ "${{ matrix.neo4j-edition }}" == "community" ]]; then
+          if [[ "${{ matrix.neo4j-tag }}" == "latest" || "${{ matrix.neo4j-tag }}" == *-community ]]; then
               poetry run pytest -m 'not enterprise_only' ./tests/e2e
           else
               poetry run pytest ./tests/e2e

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -32,14 +32,13 @@ jobs:
           key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --extras external_clients
+        run: poetry install --no-interaction --all-extras
       - name: Check format and linting
         run: |
-          poetry run ruff check --select I .
           poetry run ruff check .
           poetry run ruff format --check .
       - name: Run strict mypy check
-        run: poetry run mypy --strict --ignore-missing-imports --allow-subclassing-any --allow-untyped-calls .
+        run: poetry run mypy .
       - name: Run unit tests and check coverage
         run: |
           poetry run coverage run -m pytest tests/unit

diff --git a/.github/workflows/scheduled-e2e-tests.yaml b/.github/workflows/scheduled-e2e-tests.yaml
@@ -13,11 +13,10 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.9', '3.10', '3.11', '3.12']
-        neo4j-version:
-          - 5
-        neo4j-edition:
-          - community
-          - enterprise
+        neo4j-tag:
+          - '5-community'
+          - '5-enterprise'
+          - 'latest'
     services:
       t2v-transformers:
         image: cr.weaviate.io/semitechnologies/transformers-inference:sentence-transformers-all-MiniLM-L6-v2-onnx
@@ -41,7 +40,7 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       neo4j:
-        image: neo4j:${{ matrix.neo4j-version }}-${{ matrix.neo4j-edition }}
+        image: neo4j:${{ matrix.neo4j-tag }}
         env:
           NEO4J_AUTH: neo4j/password
           NEO4J_ACCEPT_LICENSE_AGREEMENT: 'eval'
@@ -52,6 +51,10 @@ jobs:
         credentials:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      qdrant:
+        image: qdrant/qdrant
+        ports:
+          - 6333:6333
 
     steps:
       - name: Install graphviz package
@@ -82,7 +85,7 @@ jobs:
           key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-cache --with dev
+        run: poetry install --no-interaction --no-cache --with dev --all-extras
       - name: Clear Poetry cache
         run: poetry cache clear --all .
       - name: Show disk usage after Poetry installation
@@ -96,7 +99,7 @@ jobs:
       - name: Run tests
         shell: bash
         run: |
-          if [[ "${{ matrix.neo4j-edition }}" == "community" ]]; then
+          if [[ "${{ matrix.neo4j-tag }}" == "latest" || "${{ matrix.neo4j-tag }}" == *-community ]]; then
               poetry run pytest -m 'not enterprise_only' ./tests/e2e
           else
               poetry run pytest ./tests/e2e

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,33 +4,31 @@ repos:
     hooks:
     - id: trailing-whitespace
     - id: end-of-file-fixer
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.10.0
-    hooks:
-      - id: mypy
-        name: Mypy Type Check
-        entry: mypy
-        language: python
-        types: [ python ]
-        stages: [ commit, push ]
-        args: [--strict, --ignore-missing-imports, --allow-subclassing-any, --allow-untyped-calls]
-        additional_dependencies:
-          - pytest
-          - pydantic
-          - neo4j
-          - langchain_community
-          - langchain_openai
   - repo: local
     hooks:
       - id: ruff-lint-isort
         name: Ruff Lint Sort Imports
-        entry: poetry run ruff check --select I .
+        entry: poetry run ruff check .
         language: system
         types: [ python ]
         stages: [ commit, push ]
       - id: ruff-lint
         name: Ruff Lint Check
-        entry: poetry run ruff format --check
+        entry: poetry run ruff format --check .
+        language: system
+        types: [ python ]
+        stages: [ commit, push ]
+      - id: mypy
+        name: Mypy Type Check
+        entry: mypy .
         language: system
         types: [ python ]
         stages: [ commit, push ]
+        pass_filenames: false
+        args: [
+          --strict,
+          --ignore-missing-imports,
+          --allow-untyped-calls,
+          --allow-subclassing-any,
+          --exclude='./docs/'
+        ]
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,35 @@
 
 ## Next
 
+### Added
+- Introduced optional lexical graph configuration for SimpleKGPipeline, enhancing flexibility in customizing node labels and relationship types in the lexical graph.
+
+## 1.2.0
+
+### Added
+- Made `relations` and `potential_schema` optional in `SchemaBuilder`.
+- Added a check to prevent the use of deprecated Cypher syntax for Neo4j versions 5.23.0 and above.
+- Added a `LexicalGraphBuilder` component to enable the import of the lexical graph (document, chunks) without performing entity and relation extraction.
+- Added a `Neo4jChunkReader` component to be able to read chunk text from the database.
+
+### Changed
+- Vector and Hybrid retrievers used with `return_properties` now also return the node labels (`nodeLabels`) and the node's element ID (`id`).
+- `HybridRetriever` now filters out the embedding property index in `self.vector_index_name` from the retriever result by default.
+- Removed support for neo4j.AsyncDriver in the KG creation pipeline, affecting Neo4jWriter and related components.
+- Updated examples and unit tests to reflect the removal of async driver support.
+
+### Fixed
+- Resolved issue with `AzureOpenAIEmbeddings` incorrectly inheriting from `OpenAIEmbeddings`, now inherits from `BaseOpenAIEmbeddings`.
+
+## 1.1.0
+
+### Added
+- Introduced a `fail_if_exist` option to index creation functions to control behavior when an index already exists.
+- Added Qdrant retriever in neo4j_graphrag.retrievers.
+
+### Changed
+- Comprehensive rewrite of the README to improve clarity and provide detailed usage examples.
+
 ## 1.0.0
 
 ### Fixed
@@ -35,6 +64,7 @@
 - Added support for Cohere LLM and embeddings - added optional dependency to `cohere`.
 - Added support for Anthropic LLM - added optional dependency to `anthropic`.
 - Added support for MistralAI LLM - added optional dependency to `mistralai`.
+- Added support for Qdrant - added optional dependency to `qdrant-client`.
 
 ### Fixed
 - Resolved import issue with the Vertex AI Embeddings class.
@@ -81,7 +111,8 @@
 ### IMPORTANT NOTICE
 - The `neo4j-genai` package is now deprecated. Users are advised to switch to the new package `neo4j-graphrag`.
 ### Added
-- Ability to visualise pipeline with `my_pipeline.draw("pipeline.png")`
+- Ability to visualise pipeline with `my_pipeline.draw("pipeline.png")`.
+- `LexicalGraphBuilder` component to create the lexical graph without entity-relation extraction.
 
 ### Fixed
 - Pipelines now return correct results when the same pipeline is run in parallel.