From c340edbc45e8147d37d8b5c480c03771f6f5ab33 Mon Sep 17 00:00:00 2001 From: Alex Thomas Date: Tue, 10 Sep 2024 16:43:57 +0100 Subject: [PATCH 1/3] Refactored E2E test GitHub Actions --- .github/workflows/e2e-tests.yaml | 61 +++++++++++++++++++ .github/workflows/pr-e2e-tests.yaml | 53 ++-------------- .../workflows/{pr.yaml => pr-unit-tests.yaml} | 2 +- 3 files changed, 67 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/e2e-tests.yaml rename .github/workflows/{pr.yaml => pr-unit-tests.yaml} (97%) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml new file mode 100644 index 00000000..4f9b3f90 --- /dev/null +++ b/.github/workflows/e2e-tests.yaml @@ -0,0 +1,61 @@ +name: Run E2E tests +on: + workflow_call: + inputs: + python-version: + required: true + description: 'The version of Python to use' + type: string + neo4j-edition: + required: true + description: "The Neo4j edition to use, either 'community' or 'enterprise'" + type: string + +jobs: + e2e-tests: + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Docker Prune + run: | + docker system prune -af + docker volume prune -f + - name: Set up Python ${{ python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ python-version }} + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + - name: Set Python version for Poetry + run: poetry env use python${{ python-version }} + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: ${{ runner.os }}-venv-${{ python-version }}-${{ hashFiles('**/poetry.lock') }} + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-cache --with dev + - name: Clear Poetry cache + run: poetry cache clear --all . + - name: Show disk usage after Poetry installation + run: | + df -h + - name: Wait for Weaviate to start + shell: bash + run: | + set +e + count=0; until curl -s --fail localhost:8080/v1/.well-known/ready; do ((count++)); [ $count -ge 10 ] && echo "Reached maximum retry limit" && exit 1; sleep 15; done + - name: Run tests + shell: bash + run: | + if [[ "${{ neo4j-edition }}" == "community" ]]; then + poetry run pytest -m 'not enterprise_only' ./tests/e2e + else + poetry run pytest ./tests/e2e + fi diff --git a/.github/workflows/pr-e2e-tests.yaml b/.github/workflows/pr-e2e-tests.yaml index 8a34bd2c..aa83e85a 100644 --- a/.github/workflows/pr-e2e-tests.yaml +++ b/.github/workflows/pr-e2e-tests.yaml @@ -1,4 +1,4 @@ -name: 'Neo4j-GenAI PR E2E Tests' +name: Neo4j-GenAI PR E2E Tests on: pull_request: @@ -13,6 +13,7 @@ concurrency: jobs: e2e-tests: runs-on: ubuntu-latest + uses: ./.github/workflows/e2e-tests.yaml strategy: matrix: python-version: ['3.8', '3.12'] @@ -20,6 +21,9 @@ jobs: - 5 neo4j-edition: - enterprise + with: + target: ${{ matrix.python-version }} + python-version: ${{ neo4j-edition }} services: t2v-transformers: image: cr.weaviate.io/semitechnologies/transformers-inference:sentence-transformers-all-MiniLM-L6-v2-onnx @@ -45,50 +49,3 @@ jobs: ports: - 7687:7687 - 7474:7474 - - steps: - - name: Check out repository code - uses: actions/checkout@v4 - - name: Docker Prune - run: | - docker system prune -af - docker volume prune -f - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - name: Set Python version for Poetry - run: poetry env use python${{ matrix.python-version }} - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-cache --with dev - - name: Clear Poetry cache - run: poetry cache clear --all . - - name: Show disk usage after Poetry installation - run: | - df -h - - name: Wait for Weaviate to start - shell: bash - run: | - set +e - count=0; until curl -s --fail localhost:8080/v1/.well-known/ready; do ((count++)); [ $count -ge 10 ] && echo "Reached maximum retry limit" && exit 1; sleep 15; done - - name: Run tests - shell: bash - run: | - if [[ "${{ matrix.neo4j-edition }}" == "community" ]]; then - poetry run pytest -m 'not enterprise_only' ./tests/e2e - else - poetry run pytest ./tests/e2e - fi diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr-unit-tests.yaml similarity index 97% rename from .github/workflows/pr.yaml rename to .github/workflows/pr-unit-tests.yaml index 315bf3e6..5f1e8146 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr-unit-tests.yaml @@ -1,4 +1,4 @@ -name: neo4j_genai PR +name:name: Neo4j-GenAI Scheduled Unit Tests on: pull_request jobs: From 818d81097ed5cec1d522162406e71ed8189ff2af Mon Sep 17 00:00:00 2001 From: Alex Thomas Date: Tue, 10 Sep 2024 16:48:50 +0100 Subject: [PATCH 2/3] GitHub Actions fixes --- .github/workflows/e2e-tests.yaml | 2 +- .github/workflows/pr-e2e-tests.yaml | 2 +- .github/workflows/pr-unit-tests.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 4f9b3f90..b025ead4 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -1,4 +1,4 @@ -name: Run E2E tests +name: 'Run E2E tests' on: workflow_call: inputs: diff --git a/.github/workflows/pr-e2e-tests.yaml b/.github/workflows/pr-e2e-tests.yaml index aa83e85a..0d149d33 100644 --- a/.github/workflows/pr-e2e-tests.yaml +++ b/.github/workflows/pr-e2e-tests.yaml @@ -1,4 +1,4 @@ -name: Neo4j-GenAI PR E2E Tests +name: 'Neo4j-GenAI PR E2E Tests' on: pull_request: diff --git a/.github/workflows/pr-unit-tests.yaml b/.github/workflows/pr-unit-tests.yaml index 5f1e8146..9dea762d 100644 --- a/.github/workflows/pr-unit-tests.yaml +++ b/.github/workflows/pr-unit-tests.yaml @@ -1,4 +1,4 @@ -name:name: Neo4j-GenAI Scheduled Unit Tests +name:name: 'Neo4j-GenAI Scheduled Unit Tests' on: pull_request jobs: From 8b248e9fecf7b9dbbbdc8205bc10b7fcf13c332e Mon Sep 17 00:00:00 2001 From: Alex Thomas Date: Tue, 10 Sep 2024 17:00:29 +0100 Subject: [PATCH 3/3] GitHub Actions fixes --- .github/workflows/pr-e2e-tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr-e2e-tests.yaml b/.github/workflows/pr-e2e-tests.yaml index 0d149d33..6cee8147 100644 --- a/.github/workflows/pr-e2e-tests.yaml +++ b/.github/workflows/pr-e2e-tests.yaml @@ -12,7 +12,6 @@ concurrency: jobs: e2e-tests: - runs-on: ubuntu-latest uses: ./.github/workflows/e2e-tests.yaml strategy: matrix: @@ -23,7 +22,7 @@ jobs: - enterprise with: target: ${{ matrix.python-version }} - python-version: ${{ neo4j-edition }} + python-version: ${{ matrix.neo4j-edition }} services: t2v-transformers: image: cr.weaviate.io/semitechnologies/transformers-inference:sentence-transformers-all-MiniLM-L6-v2-onnx