Merge branch 'develop' of https://github.com/qld-gov-au/ckanext-s3fil…

…estore into snyk-fix-6edad2ad2714d4666b99961d704d1e3e

.github/workflows/ci.yml

@@ -11,11 +11,11 @@ jobs:
   code_quality:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         timeout-minutes: 2
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.x'
         timeout-minutes: 5
@@ -36,24 +36,26 @@ jobs:
     needs: code_quality
     strategy:
       matrix:
-        ckan-version: ['2.10', '2.9', '2.9-py2', '2.8', '2.7']
+        ckan-version: ['2.11', '2.10', '2.9']
+        experimental: [false]
         ckan-git-org: ['ckan']
         include:
-          - ckan-version: '2.8'
-            ckan-git-version: 'ckan-2.8.8-qgov.5'
-            ckan-git-org: 'qld-gov-au'
           - ckan-version: '2.9'
-            ckan-git-version: 'ckan-2.9.5-qgov.10'
+            ckan-git-version: 'ckan-2.9.9-qgov.4'
             ckan-git-org: 'qld-gov-au'
           - ckan-version: '2.9'
-            ckan-git-version: 'qgov-master-2.9.7'
+            ckan-git-version: 'qgov-master-2.9.9'
             ckan-git-org: 'qld-gov-au'
+          - ckan-version: 'master'
+            ckan-git-org: 'ckan'
+            ckan-git-version: 'master'
+            experimental: true  #master is unstable, good to know if we are compatible or not
 
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }} ${{ matrix.ckan-git-org }} ${{ matrix.ckan-git-version }}
     container:
-      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+      image: ckan/ckan-dev:${{ matrix.ckan-version }}
 
     services:
       postgresql:
@@ -86,7 +88,7 @@ jobs:
           - "5000"
 
       solr:
-        image: ckan/ckan-solr:${{ matrix.ckan-version }}
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
 
     env:
       CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgresql/ckan_test
@@ -100,21 +102,28 @@ jobs:
 
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         timeout-minutes: 2
 
       - name: Install requirements
+        continue-on-error: ${{ matrix.experimental }}
         run: |
           chmod u+x ./scripts/*
           ./scripts/build.sh
         timeout-minutes: 15
 
       - name: Setup CKAN
-        run: |
-          ./scripts/init.sh
+        continue-on-error: ${{ matrix.experimental }}
+        run: ./scripts/init.sh
         timeout-minutes: 15
 
       - name: Run tests
-        run: |
-          ./scripts/test.sh
+        continue-on-error: ${{ matrix.experimental }}
+        run: ./scripts/test.sh
         timeout-minutes: 30
+
+      - name: Test Summary
+        uses: test-summary/action@v2
+        with:
+          paths: "/tmp/artifacts/junit/results.xml"
+        if: always()

README.rst

@@ -37,10 +37,17 @@ To install ckanext-s3filestore:
 
      . /usr/lib/ckan/default/bin/activate
 
-2. Install the ckanext-s3filestore Python package into your virtual environment::
+2. Install the ckanext-s3filestore Python package into your virtual environment:
+
+ 2.1 From PyPI::
 
      pip install ckanext-s3filestore
 
+ 2.2 Alternatively, install from source::
+
+    pip install -e https://github.com/qld-gov-au/ckanext-s3filestore.git#egg=ckanext-s3filestore
+    pip install -r /usr/lib/ckan/default/src/ckanext-s3filestore/requirements.txt
+
 3. Add ``s3filestore`` to the ``ckan.plugins`` setting in your CKAN
    config file (by default the config file is located at
    ``/etc/ckan/default/production.ini``).
@@ -208,7 +215,7 @@ apt-get install sudo systemd postgresql-10 git python python-pip
 export PGVERSION=10 && export CKAN_BRANCH=qgov-master && export CKAN_GIT_REPO=qld-gov-au/ckan
 cd /build
 bash bin/travis-build.bash
-nosetests --ckan  --with-pylons=subdir/test.ini --with-coverage --cover-package=ckanext.s3filestore --cover-inclusive --cover-erase --cover-tests
+pytest --ckan-ini=test.ini --cov=ckanext.s3filestore
 
 ---------------------------------------
 Registering ckanext-s3filestore on PyPI

ckanext/s3filestore/plugin.py

@@ -81,7 +81,12 @@ def get_uploader(self, upload_to, old_filename=None):
 
     # IPackageController
 
+    # CKAN < 2.10
     def after_update(self, context, pkg_dict):
+        return self.after_dataset_update(context, pkg_dict)
+
+    # CKAN >= 2.10
+    def after_dataset_update(self, context, pkg_dict):
         ''' Update the access of each S3 object to match the package.
         '''
         pkg_id = pkg_dict['id']
@@ -118,6 +123,9 @@ def after_update_resource_list_update(self, visibility_level, pkg_id, pkg_dict):
 
         LOG.debug("after_update_resource_list_update: Package %s has been updated, notifying resources", pkg_id)
         for resource in pkg_dict['resources']:
+            if 'id' not in resource:
+                # skip new resources as they would already have correct visibility
+                continue
             uploader = get_resource_uploader(resource)
             if hasattr(uploader, 'update_visibility'):
                 uploader.update_visibility(

ckanext/s3filestore/tests/__init__.py

@@ -1,7 +1,5 @@
 # encoding: utf-8
 
-from ckan.tests import helpers
-
 
 def _get_status_code(response):
     """ Get the status code from a HTTP response.
@@ -20,7 +18,3 @@ def _get_response_body(response):
         return response.text
     else:
         return response.body
-
-
-def teardown_function(self=None):
-    helpers.reset_db()

ckanext/s3filestore/tests/test_controller.py

@@ -6,7 +6,7 @@
 import requests
 import six
 
-from nose.tools import (with_setup)
+import pytest
 
 from werkzeug.datastructures import FileStorage as FlaskFileStorage
 
@@ -18,19 +18,11 @@
 
 from ckanext.s3filestore import uploader
 
-from . import _get_status_code, _get_response_body, teardown_function
+from . import _get_status_code, _get_response_body
 
 log = logging.getLogger(__name__)
 
 
-def setup_function(self):
-    self.sysadmin = factories.Sysadmin(apikey="my-test-key")
-
-    assert config.get('ckanext.s3filestore.signature_version') == 's3v4'
-    self.bucket_name = config.get(u'ckanext.s3filestore.aws_bucket_name')
-    uploader.BaseS3Uploader().get_s3_bucket(self.bucket_name)
-
-
 def _test_org():
     try:
         return helpers.call_action('organization_show', id='test-org')
@@ -47,9 +39,16 @@ def _test_org():
             image_upload=FlaskFileStorage(six.BytesIO(b"\0\0\0"), u"image.png"))
 
 
-@with_setup(setup_function, teardown_function)
+@pytest.mark.usefixtures("clean_db", "with_plugins")
 class TestS3Controller(object):
 
+    def setup_method(self, test_method):
+        self.sysadmin = factories.Sysadmin(apikey="my-test-key")
+
+        assert config.get('ckanext.s3filestore.signature_version') == 's3v4'
+        self.bucket_name = config.get(u'ckanext.s3filestore.aws_bucket_name')
+        uploader.BaseS3Uploader().get_s3_bucket(self.bucket_name)
+
     def _upload_resource(self):
         dataset = factories.Dataset(name="my-dataset")
 

ckanext/s3filestore/tests/test_plugin.py

@@ -1,6 +1,9 @@
 # encoding: utf-8
 
-import mock
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 from parameterized import parameterized
 
 import ckantoolkit as toolkit
@@ -11,7 +14,7 @@
 
 class TestS3Plugin():
 
-    def setup(self):
+    def setup_method(self, test_method):
         self.plugin = S3FileStorePlugin()
 
     def test_update_config(self):