From 963dc71b1edd65650e9be45cc48ae899721cb475 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 17 Dec 2024 12:20:30 +1000 Subject: [PATCH 1/3] [QOLDEV-1003] fix Solr container versions - CKAN 2.11 needs Solr 9 container, older versions need Solr 8 --- .github/workflows/{ci.yml => test.yml} | 72 +++++++++++--------------- scripts/test.sh | 2 +- 2 files changed, 31 insertions(+), 43 deletions(-) rename .github/workflows/{ci.yml => test.yml} (67%) diff --git a/.github/workflows/ci.yml b/.github/workflows/test.yml similarity index 67% rename from .github/workflows/ci.yml rename to .github/workflows/test.yml index cb3afd0..0633f53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/test.yml @@ -11,89 +11,77 @@ jobs: code_quality: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - timeout-minutes: 2 - - - name: Setup Python - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - timeout-minutes: 5 - - - name: Install flake8 + - name: Install requirements run: | python -m pip install --upgrade pip pip install flake8 - timeout-minutes: 5 - - - name: Lint with flake8 - run: | - flake8 + - name: Check syntax + run: flake8 timeout-minutes: 5 test: - runs-on: ubuntu-latest needs: code_quality strategy: matrix: - ckan-version: ['2.11', '2.10', '2.9'] + solr-version: [9] experimental: [false] - ckan-git-org: ['ckan'] include: - - ckan-version: '2.9' - ckan-git-version: 'ckan-2.9.9-qgov.4' + - ckan-version: '2.11' + ckan-git-version: 'ckan-2.11.1' + ckan-git-org: 'ckan' + - ckan-version: '2.10' + ckan-git-version: 'ckan-2.10.6' + ckan-git-org: 'ckan' + solr-version: '8' + - ckan-version: '2.10' + ckan-git-version: 'ckan-2.10.5-qgov.4' ckan-git-org: 'qld-gov-au' + solr-version: '8' - ckan-version: '2.9' - ckan-git-version: 'qgov-master-2.9.9' + ckan-git-version: 'ckan-2.9.9-qgov.4' ckan-git-org: 'qld-gov-au' + solr-version: '8' - ckan-version: 'master' ckan-git-org: 'ckan' ckan-git-version: 'master' - experimental: true #master is unstable, good to know if we are compatible or not + experimental: true # master is unstable, good to know if we are compatible or not fail-fast: false name: CKAN ${{ matrix.ckan-version }} ${{ matrix.ckan-git-org }} ${{ matrix.ckan-git-version }} + runs-on: ubuntu-latest container: image: ckan/ckan-dev:${{ matrix.ckan-version }} - services: - postgresql: + solr: + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr${{ matrix.solr-version }} + postgres: image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 ports: - 5432:5432 - redis: image: redis - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 ports: - 6379:6379 - moto: image: motoserver/moto ports: - "5000" - solr: - image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 - env: - CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgresql/ckan_test - CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgresql/datastore_test - CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgresql/datastore_test + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test CKAN_SOLR_URL: http://solr:8983/solr/ckan CKAN_REDIS_URL: redis://redis:6379/1 CKAN_VERSION: ${{ matrix.ckan-version }} @@ -103,7 +91,7 @@ jobs: steps: - uses: actions/checkout@v4 - timeout-minutes: 2 + timeout-minutes: 1 - name: Install requirements continue-on-error: ${{ matrix.experimental }} @@ -125,5 +113,5 @@ jobs: - name: Test Summary uses: test-summary/action@v2 with: - paths: "/tmp/artifacts/junit/results.xml" + paths: "/tmp/junit/results.xml" if: always() diff --git a/scripts/test.sh b/scripts/test.sh index 7618047..9db3070 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -pytest -vv --ckan-ini=test.ini --cov=ckanext.s3filestore --junit-xml=/tmp/artifacts/junit/results.xml +pytest -vv --ckan-ini=test.ini --cov=ckanext.s3filestore --junit-xml=/tmp/junit/results.xml From 9a532c4225361938fa39dc4ea4fcff24eabf7e3b Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 17 Dec 2024 12:34:06 +1000 Subject: [PATCH 2/3] [QOLDEV-1003] run test container as root - Work around https://github.com/ckan/ckan-docker-base/issues/86 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0633f53..0b24ccf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,6 +56,7 @@ jobs: runs-on: ubuntu-latest container: image: ckan/ckan-dev:${{ matrix.ckan-version }} + options: --user root services: solr: image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr${{ matrix.solr-version }} From 1b6b88c3d20dd1f7e6a549dcb59d8b50b47ced1d Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 17 Dec 2024 12:40:03 +1000 Subject: [PATCH 3/3] [QOLDEV-1003] fix Solr container pinning syntax --- .github/workflows/test.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b24ccf..dbf9a67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,27 +27,31 @@ jobs: needs: code_quality strategy: matrix: - solr-version: [9] - experimental: [false] include: - ckan-version: '2.11' ckan-git-version: 'ckan-2.11.1' ckan-git-org: 'ckan' + solr-version: 9 + experimental: false - ckan-version: '2.10' ckan-git-version: 'ckan-2.10.6' ckan-git-org: 'ckan' - solr-version: '8' + solr-version: 8 + experimental: false - ckan-version: '2.10' ckan-git-version: 'ckan-2.10.5-qgov.4' ckan-git-org: 'qld-gov-au' - solr-version: '8' + solr-version: 8 + experimental: false - ckan-version: '2.9' ckan-git-version: 'ckan-2.9.9-qgov.4' ckan-git-org: 'qld-gov-au' - solr-version: '8' + solr-version: 8 + experimental: false - ckan-version: 'master' ckan-git-org: 'ckan' ckan-git-version: 'master' + solr-version: 9 experimental: true # master is unstable, good to know if we are compatible or not fail-fast: false