Merge pull request #127 from qld-gov-au/develop

Develop to master - prepare for CKAN 2.11

.github/workflows/test.yml

@@ -0,0 +1,122 @@
+
+name: CI
+
+on:
+  push:
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  code_quality:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8
+      - name: Check syntax
+        run: flake8
+        timeout-minutes: 5
+
+  test:
+    needs: code_quality
+    strategy:
+      matrix:
+        include:
+          - ckan-version: '2.11'
+            ckan-git-version: 'ckan-2.11.1'
+            ckan-git-org: 'ckan'
+            solr-version: 9
+            experimental: false
+          - ckan-version: '2.10'
+            ckan-git-version: 'ckan-2.10.6'
+            ckan-git-org: 'ckan'
+            solr-version: 8
+            experimental: false
+          - ckan-version: '2.10'
+            ckan-git-version: 'ckan-2.10.5-qgov.4'
+            ckan-git-org: 'qld-gov-au'
+            solr-version: 8
+            experimental: false
+          - ckan-version: '2.9'
+            ckan-git-version: 'ckan-2.9.9-qgov.4'
+            ckan-git-org: 'qld-gov-au'
+            solr-version: 8
+            experimental: false
+          - ckan-version: 'master'
+            ckan-git-org: 'ckan'
+            ckan-git-version: 'master'
+            solr-version: 9
+            experimental: true  # master is unstable, good to know if we are compatible or not
+
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }} ${{ matrix.ckan-git-org }} ${{ matrix.ckan-git-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: ckan/ckan-dev:${{ matrix.ckan-version }}
+      options: --user root
+    services:
+      solr:
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr${{ matrix.solr-version }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+        ports:
+          - 5432:5432
+      redis:
+        image: redis
+        options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
+        ports:
+          - 6379:6379
+      moto:
+        image: motoserver/moto
+        ports:
+          - "5000"
+
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+      CKAN_VERSION: ${{ matrix.ckan-version }}
+      CKAN_GIT_ORG: ${{ matrix.ckan-git-org }}
+      CKAN_GIT_VERSION: ${{ matrix.ckan-git-version }}
+
+
+    steps:
+      - uses: actions/checkout@v4
+        timeout-minutes: 1
+
+      - name: Install requirements
+        continue-on-error: ${{ matrix.experimental }}
+        run: |
+          chmod u+x ./scripts/*
+          ./scripts/build.sh
+        timeout-minutes: 15
+
+      - name: Setup CKAN
+        continue-on-error: ${{ matrix.experimental }}
+        run: ./scripts/init.sh
+        timeout-minutes: 15
+
+      - name: Run tests
+        continue-on-error: ${{ matrix.experimental }}
+        run: ./scripts/test.sh
+        timeout-minutes: 30
+
+      - name: Test Summary
+        uses: test-summary/action@v2
+        with:
+          paths: "/tmp/junit/results.xml"
+        if: always()

README.rst

@@ -215,7 +215,7 @@ apt-get install sudo systemd postgresql-10 git python python-pip
 export PGVERSION=10 && export CKAN_BRANCH=qgov-master && export CKAN_GIT_REPO=qld-gov-au/ckan
 cd /build
 bash bin/travis-build.bash
-nosetests --ckan  --with-pylons=subdir/test.ini --with-coverage --cover-package=ckanext.s3filestore --cover-inclusive --cover-erase --cover-tests
+pytest --ckan-ini=test.ini --cov=ckanext.s3filestore
 
 ---------------------------------------
 Registering ckanext-s3filestore on PyPI

ckanext/s3filestore/plugin.py

@@ -123,6 +123,9 @@ def after_update_resource_list_update(self, visibility_level, pkg_id, pkg_dict):
 
         LOG.debug("after_update_resource_list_update: Package %s has been updated, notifying resources", pkg_id)
         for resource in pkg_dict['resources']:
+            if 'id' not in resource:
+                # skip new resources as they would already have correct visibility
+                continue
             uploader = get_resource_uploader(resource)
             if hasattr(uploader, 'update_visibility'):
                 uploader.update_visibility(

ckanext/s3filestore/tests/__init__.py

@@ -1,7 +1,5 @@
 # encoding: utf-8
 
-from ckan.tests import helpers
-
 
 def _get_status_code(response):
     """ Get the status code from a HTTP response.
@@ -20,7 +18,3 @@ def _get_response_body(response):
         return response.text
     else:
         return response.body
-
-
-def teardown_function(self=None):
-    helpers.reset_db()

ckanext/s3filestore/tests/test_controller.py

@@ -6,7 +6,7 @@
 import requests
 import six
 
-from nose.tools import (with_setup)
+import pytest
 
 from werkzeug.datastructures import FileStorage as FlaskFileStorage
 
@@ -18,19 +18,11 @@
 
 from ckanext.s3filestore import uploader
 
-from . import _get_status_code, _get_response_body, teardown_function
+from . import _get_status_code, _get_response_body
 
 log = logging.getLogger(__name__)
 
 
-def setup_function(self):
-    self.sysadmin = factories.Sysadmin(apikey="my-test-key")
-
-    assert config.get('ckanext.s3filestore.signature_version') == 's3v4'
-    self.bucket_name = config.get(u'ckanext.s3filestore.aws_bucket_name')
-    uploader.BaseS3Uploader().get_s3_bucket(self.bucket_name)
-
-
 def _test_org():
     try:
         return helpers.call_action('organization_show', id='test-org')
@@ -47,9 +39,16 @@ def _test_org():
             image_upload=FlaskFileStorage(six.BytesIO(b"\0\0\0"), u"image.png"))
 
 
-@with_setup(setup_function, teardown_function)
+@pytest.mark.usefixtures("clean_db", "with_plugins")
 class TestS3Controller(object):
 
+    def setup_method(self, test_method):
+        self.sysadmin = factories.Sysadmin(apikey="my-test-key")
+
+        assert config.get('ckanext.s3filestore.signature_version') == 's3v4'
+        self.bucket_name = config.get(u'ckanext.s3filestore.aws_bucket_name')
+        uploader.BaseS3Uploader().get_s3_bucket(self.bucket_name)
+
     def _upload_resource(self):
         dataset = factories.Dataset(name="my-dataset")
 

ckanext/s3filestore/tests/test_plugin.py

@@ -1,6 +1,9 @@
 # encoding: utf-8
 
-import mock
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 from parameterized import parameterized
 
 import ckantoolkit as toolkit
@@ -11,7 +14,7 @@
 
 class TestS3Plugin():
 
-    def setup(self):
+    def setup_method(self, test_method):
         self.plugin = S3FileStorePlugin()
 
     def test_update_config(self):