Dev minor (#1536)

* improve ci/cd runtime (#1535) * improve ci/cd runtime * update prompt tests * Support Python ^3.10 (#1534) * Patch/add azure to workflows (#1537) * improve ci/cd runtime * update prompt tests * add azure * up * up * revert latest ingestion changes (#1539) * revert latest ingestion changes * revert latest ingestion changes * Update pyproject.toml --------- Co-authored-by: Nolan Tremelling <[email protected]>
SciPhi-AI · Oct 31, 2024 · f1868c7 · f1868c7
1 parent 946743e
commit f1868c7
Show file tree

Hide file tree

Showing 36 changed files with 1,006 additions and 517 deletions.
diff --git a/.github/actions/run-sdk-auth-tests/action.yml b/.github/actions/run-sdk-auth-tests/action.yml
@@ -6,44 +6,44 @@ runs:
     - name: User registration and login
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_registration_and_login
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_registration_and_login
 
     - name: Duplicate user registration
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_duplicate_user_registration
+      run: poetry run python tests/integration/runner_sdk_basic.py test_duplicate_user_registration
 
     - name: Token refresh
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_token_refresh
+      run: poetry run python tests/integration/runner_sdk_basic.py test_token_refresh
 
     - name: User document management
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_document_management
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_document_management
 
     - name: User search and RAG
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_search_and_rag
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_search_and_rag
 
     - name: User password management
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_password_management
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_password_management
 
     - name: User profile management
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_profile_management
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_profile_management
 
     - name: User overview
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_overview
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_overview
 
     - name: User logout
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_logout
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_logout
diff --git a/.github/actions/run-sdk-collections-tests/action.yml b/.github/actions/run-sdk-collections-tests/action.yml
@@ -6,104 +6,104 @@ runs:
     - name: Ingest sample file (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_ingest_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_ingest_sample_file_sdk
 
     - name: User creates collection
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_creates_collection
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_creates_collection
 
     - name: User updates collection
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_updates_collection
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_updates_collection
 
     - name: User lists collections
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_lists_collections
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_lists_collections
 
     - name: User collection document management
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_collection_document_management
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_collection_document_management
 
     - name: User removes document from collection
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_removes_document_from_collection
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_removes_document_from_collection
 
     - name: User lists documents in collection
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_lists_documents_in_collection
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_lists_documents_in_collection
 
     - name: Pagination and filtering
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_pagination_and_filtering
+      run: poetry run python tests/integration/runner_sdk_basic.py test_pagination_and_filtering
 
     - name: Advanced collection management
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_advanced_collection_management
+      run: poetry run python tests/integration/runner_sdk_basic.py test_advanced_collection_management
 
     - name: User gets collection details
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_gets_collection_details
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_gets_collection_details
 
     - name: User adds user to collection
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_adds_user_to_collection
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_adds_user_to_collection
 
     - name: User removes user from collection
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_removes_user_from_collection
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_removes_user_from_collection
 
     - name: User lists users in collection
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_lists_users_in_collection
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_lists_users_in_collection
 
     - name: User gets collections for user
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_gets_collections_for_user
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_gets_collections_for_user
 
     - name: User gets collections for document
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_gets_collections_for_document
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_gets_collections_for_document
 
     - name: User permissions
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_user_permissions
+      run: poetry run python tests/integration/runner_sdk_basic.py test_user_permissions
 
     - name: Ingest chunks
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_ingest_chunks
+      run: poetry run python tests/integration/runner_sdk_basic.py test_ingest_chunks
 
     - name: Update chunks
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_update_chunks
+      run: poetry run python tests/integration/runner_sdk_basic.py test_update_chunks
 
     - name: Delete chunks
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_delete_chunks
+      run: poetry run python tests/integration/runner_sdk_basic.py test_delete_chunks
 
     - name: Get all prompts
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_get_all_prompts
+      run: poetry run python tests/integration/runner_sdk_basic.py test_get_all_prompts
 
     - name: Get prompt
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_get_prompt
+      run: poetry run python tests/integration/runner_sdk_basic.py test_get_prompt
diff --git a/.github/actions/run-sdk-graphrag-deduplication-tests/action.yml b/.github/actions/run-sdk-graphrag-deduplication-tests/action.yml
@@ -6,34 +6,34 @@ runs:
     - name: Ingest sample file (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_ingest_sample_file_2_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_ingest_sample_file_2_sdk
 
     - name: Create the graph (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_create_graph_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_create_graph_sample_file_sdk
 
     - name: Deduplicate entities (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_deduplicate_entities_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_deduplicate_entities_sample_file_sdk
 
     - name: Enrich the graph (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_enrich_graph_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_enrich_graph_sample_file_sdk
 
     - name: Search over the graph (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_search_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_search_sample_file_sdk
 
     - name: Delete the graph (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_delete_graph_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_delete_graph_sample_file_sdk
 
     - name: Delete the graph with cascading (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_delete_graph_with_cascading_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_delete_graph_with_cascading_sample_file_sdk
diff --git a/.github/actions/run-sdk-graphrag-tests/action.yml b/.github/actions/run-sdk-graphrag-tests/action.yml
@@ -6,29 +6,29 @@ runs:
     - name: Ingest sample file (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_ingest_sample_file_2_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_ingest_sample_file_2_sdk
 
     - name: Create the graph (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_create_graph_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_create_graph_sample_file_sdk
 
     - name: Enrich the graph (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_enrich_graph_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_enrich_graph_sample_file_sdk
 
     - name: Search over the graph (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_search_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_search_sample_file_sdk
 
     - name: Delete the graph (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_delete_graph_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_delete_graph_sample_file_sdk
 
     - name: Delete the graph with cascading (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_kg_delete_graph_with_cascading_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_kg_delete_graph_with_cascading_sample_file_sdk
diff --git a/.github/actions/run-sdk-ingestion-tests/action.yml b/.github/actions/run-sdk-ingestion-tests/action.yml
@@ -6,29 +6,29 @@ runs:
     - name: Ingest sample file (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_ingest_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_ingest_sample_file_sdk
 
     - name: Reingest sample file (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_reingest_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_reingest_sample_file_sdk
 
     - name: Document overview (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_document_overview_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_document_overview_sample_file_sdk
 
     - name: Document chunks (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_document_chunks_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_document_chunks_sample_file_sdk
 
     - name: Delete and reingest (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_delete_and_reingest_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_delete_and_reingest_sample_file_sdk
 
     - name: Ingest sample file with config (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_ingest_sample_file_with_config_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_ingest_sample_file_with_config_sdk
diff --git a/.github/actions/run-sdk-prompt-management-tests/action.yml b/.github/actions/run-sdk-prompt-management-tests/action.yml
@@ -7,36 +7,36 @@ runs:
     - name: Add prompt test (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_add_prompt
+      run: poetry run python tests/integration/runner_sdk_basic.py test_add_prompt
 
     - name: Get prompt test (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_get_prompt
+      run: poetry run python tests/integration/runner_sdk_basic.py test_get_prompt
 
     - name: Get all prompts test (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_get_all_prompts
+      run: poetry run python tests/integration/runner_sdk_basic.py test_get_all_prompts
 
     - name: Update prompt test (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_update_prompt
+      run: poetry run python tests/integration/runner_sdk_basic.py test_update_prompt
 
     # Then run error handling and access control tests
     - name: Prompt error handling test (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_prompt_error_handling
+      run: poetry run python tests/integration/runner_sdk_basic.py test_prompt_error_handling
 
     - name: Prompt access control test (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_prompt_access_control
+      run: poetry run python tests/integration/runner_sdk_basic.py test_prompt_access_control
 
     # Finally run deletion test
     - name: Delete prompt test (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_delete_prompt
+      run: poetry run python tests/integration/runner_sdk_basic.py test_delete_prompt
diff --git a/.github/actions/run-sdk-retrieval-tests/action.yml b/.github/actions/run-sdk-retrieval-tests/action.yml
@@ -7,25 +7,25 @@ runs:
     - name: Ingest sample file (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_ingest_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_ingest_sample_file_sdk
 
     - name: Vector search sample file filter (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_vector_search_sample_file_filter_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_vector_search_sample_file_filter_sdk
 
     - name: Hybrid search sample file filter (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_hybrid_search_sample_file_filter_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_hybrid_search_sample_file_filter_sdk
 
     - name: RAG response sample file (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_rag_response_sample_file_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_rag_response_sample_file_sdk
 
 
     - name: Agent response sample file (SDK)
       working-directory: ./py
       shell: bash
-      run: poetry run python tests/integration/runner_sdk.py test_conversation_history_sdk
+      run: poetry run python tests/integration/runner_sdk_basic.py test_conversation_history_sdk
diff --git a/.github/actions/start-r2r-full/action.yml b/.github/actions/start-r2r-full/action.yml
@@ -12,4 +12,4 @@ runs:
     shell: bash
     run: |
       cd py
-      poetry run r2r serve --docker --full --config-name=full --build --image=r2r-local
+      poetry run r2r serve --docker --full --config-name=full_azure --build --image=r2r-local
diff --git a/.github/actions/start-r2r-light/action.yml b/.github/actions/start-r2r-light/action.yml
@@ -7,6 +7,6 @@ runs:
       shell: bash
       run: |
         cd py
-        poetry run r2r serve &
+        poetry run r2r serve --config-name=r2r_azure &
         echo "Waiting for services to start..."
         sleep 30
diff --git a/.github/workflows/r2r-full-integration-deep-dive-tests.yml b/.github/workflows/r2r-full-integration-deep-dive-tests.yml
@@ -13,6 +13,9 @@ jobs:
     runs-on: "ubuntu-latest"
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+      AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+      AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
       TELEMETRY_ENABLED: 'false'
       R2R_PROJECT_NAME: r2r_default