diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4d6153f3..3b44198d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -32,6 +32,7 @@ on: jobs: lint: + if: github.repository == 'ckan/ckanext-xloader' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -47,13 +48,22 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["2.11", "2.10", 2.9] + include: #ckan-image see https://github.com/ckan/ckan-docker-base, ckan-version controls other image tags + - ckan-version: "2.11" + ckan-image: "2.11-py3.10" + - ckan-version: "2.10" + ckan-image: "2.10-py3.10" + - ckan-version: "2.9" + ckan-image: "2.9-py3.9" + #- ckan-version: "master" Publish does not care about master + # ckan-image: "master" fail-fast: false name: CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: - image: ckan/ckan-dev:${{ matrix.ckan-version }} + image: ckan/ckan-dev:${{ matrix.ckan-image }} + options: --user root services: solr: image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 @@ -93,6 +103,12 @@ jobs: - name: Run tests run: pytest --ckan-ini=test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests + publishSkipped: + if: github.repository != 'ckan/ckanext-xloader' + steps: + - run: | + echo "Skipping PyPI publish on downstream repository" + publish: needs: test permissions: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a65aed65..53ec0c5f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,13 +23,26 @@ jobs: needs: lint strategy: matrix: - ckan-version: ["2.11", "2.10", 2.9] + include: #ckan-image see https://github.com/ckan/ckan-docker-base, ckan-version controls other image tags + - ckan-version: "2.11" + ckan-image: "2.11-py3.10" + experimental: false + - ckan-version: "2.10" + ckan-image: "2.10-py3.10" + experimental: false + - ckan-version: "2.9" + ckan-image: "2.9-py3.9" + experimental: false + - ckan-version: "master" + ckan-image: "master" + experimental: true # master is unstable, good to know if we are compatible or not fail-fast: false name: CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: - image: ckan/ckan-dev:${{ matrix.ckan-version }} + image: ckan/ckan-dev:${{ matrix.ckan-image }} + options: --user root services: solr: image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 @@ -53,9 +66,15 @@ jobs: steps: - uses: actions/checkout@v4 - - if: ${{ matrix.ckan-version == 2.9 }} + continue-on-error: ${{ matrix.experimental }} + + - name: Pin setuptools for ckan 2.9 only + if: ${{ matrix.ckan-version == 2.9 }} run: pip install "setuptools>=44.1.0,<71" + continue-on-error: ${{ matrix.experimental }} + - name: Install requirements + continue-on-error: ${{ matrix.experimental }} run: | pip install -r requirements.txt pip install -r dev-requirements.txt @@ -63,8 +82,19 @@ jobs: pip install -U requests[security] # Replace default path to CKAN core config file with the one on the container sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini - - name: Setup extension (CKAN >= 2.9) + + - name: Setup extension + continue-on-error: ${{ matrix.experimental }} run: | ckan -c test.ini db init + - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests + continue-on-error: ${{ matrix.experimental }} + run: pytest --ckan-ini=test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests --junit-xml=/tmp/artifacts/junit/results.xml + + - name: Test Summary + uses: test-summary/action@v2 + continue-on-error: ${{ matrix.experimental }} + with: + paths: "/tmp/artifacts/junit/*.xml" + if: always() \ No newline at end of file diff --git a/ckanext/xloader/config_declaration.yaml b/ckanext/xloader/config_declaration.yaml index 6d85e896..9487999d 100644 --- a/ckanext/xloader/config_declaration.yaml +++ b/ckanext/xloader/config_declaration.yaml @@ -128,6 +128,23 @@ groups: to True. type: bool required: false + - key: ckanext.xloader.validation.requires_successful_report + default: False + example: True + description: | + Resources are required to pass Validation from the ckanext-validation + plugin to be able to get XLoadered. + type: bool + required: false + - key: ckanext.xloader.validation.enforce_schema + default: True + example: False + description: | + Resources are expected to have a Validation Schema, or use the default ones if not. + If this option is set to `False`, Resources that do not have + a Validation Schema will be treated like they do not require Validation. + See https://github.com/frictionlessdata/ckanext-validation?tab=readme-ov-file#data-schema + for more details. - key: ckanext.xloader.clean_datastore_tables default: False example: True diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index 3ac8ebba..85c51936 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -124,6 +124,7 @@ def xloader_data_into_datastore(input): if tries < MAX_RETRIES: tries = tries + 1 log.info("Job %s failed due to temporary error [%s], retrying", job_id, e) + logger.info("Job failed due to temporary error [%s], retrying", e) job_dict['status'] = 'pending' job_dict['metadata']['tries'] = tries enqueue_job( @@ -245,7 +246,12 @@ def tabulator_load(): logger.info("'use_type_guessing' mode is: %s", use_type_guessing) try: if use_type_guessing: - tabulator_load() + try: + tabulator_load() + except JobError as e: + logger.warning('Load using tabulator failed: %s', e) + logger.info('Trying again with direct COPY') + direct_load() else: try: direct_load() diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index 8c913e0a..46814181 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -118,8 +118,8 @@ def _clear_datastore_resource(resource_id): ''' engine = get_write_engine() with engine.begin() as conn: - conn.execute("SET LOCAL lock_timeout = '5s'") - conn.execute('TRUNCATE TABLE "{}"'.format(resource_id)) + conn.execute("SET LOCAL lock_timeout = '15s'") + conn.execute('TRUNCATE TABLE "{}" RESTART IDENTITY'.format(resource_id)) def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): @@ -339,6 +339,18 @@ def create_column_indexes(fields, resource_id, logger): logger.info('...column indexes created.') +def _save_type_overrides(headers_dicts): + # copy 'type' to 'type_override' if it's not the default type (text) + # and there isn't already an override in place + for h in headers_dicts: + if h['type'] != 'text': + if 'info' in h: + if 'type_override' not in h['info']: + h['info']['type_override'] = h['type'] + else: + h['info'] = {'type_override': h['type']} + + def load_table(table_filepath, resource_id, mimetype='text/csv', logger=None): '''Loads an Excel file (or other tabular data recognized by tabulator) into Datastore and creates indexes. @@ -399,7 +411,14 @@ def load_table(table_filepath, resource_id, mimetype='text/csv', logger=None): }.get(existing_info.get(h, {}).get('type_override'), t) for t, h in zip(types, headers)] - headers = [header.strip()[:MAX_COLUMN_LENGTH] for header in headers if header.strip()] + # Strip leading and trailing whitespace, then truncate to maximum length, + # then strip again in case the truncation exposed a space. + headers = [ + header.strip()[:MAX_COLUMN_LENGTH].strip() + for header in headers + if header and header.strip() + ] + header_count = len(headers) type_converter = TypeConverter(types=types) with UnknownEncodingStream(table_filepath, file_format, decoding_result, @@ -409,6 +428,17 @@ def row_iterator(): for row in stream: data_row = {} for index, cell in enumerate(row): + # Handle files that have extra blank cells in heading and body + # eg from Microsoft Excel adding lots of empty cells on export. + # Blank header cells won't generate a column, + # so row length won't match column count. + if index >= header_count: + # error if there's actual data out of bounds, otherwise ignore + if cell: + raise LoaderError("Found data in column %s but resource only has %s header(s)", + index + 1, header_count) + else: + continue data_row[headers[index]] = cell yield data_row result = row_iterator() @@ -426,6 +456,9 @@ def row_iterator(): if type_override in list(_TYPE_MAPPING.values()): h['type'] = type_override + # preserve any types that we have sniffed unless told otherwise + _save_type_overrides(headers_dicts) + logger.info('Determined headers and types: %s', headers_dicts) ''' diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 07af8db7..d916cc54 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -7,11 +7,16 @@ from ckan.model.domain_object import DomainObjectOperation from ckan.model.resource import Resource -from ckan.model.package import Package from . import action, auth, helpers as xloader_helpers, utils from ckanext.xloader.utils import XLoaderFormats +try: + from ckanext.validation.interfaces import IPipeValidation + HAS_IPIPE_VALIDATION = True +except ImportError: + HAS_IPIPE_VALIDATION = False + try: config_declarations = toolkit.blanket.config_declarations except AttributeError: @@ -34,6 +39,8 @@ class xloaderPlugin(plugins.SingletonPlugin): plugins.implements(plugins.IResourceController, inherit=True) plugins.implements(plugins.IClick) plugins.implements(plugins.IBlueprint) + if HAS_IPIPE_VALIDATION: + plugins.implements(IPipeValidation) # IClick def get_commands(self): @@ -69,6 +76,21 @@ def configure(self, config_): ) ) + # IPipeValidation + + def receive_validation_report(self, validation_report): + if utils.requires_successful_validation_report(): + res_dict = toolkit.get_action('resource_show')({'ignore_auth': True}, + {'id': validation_report.get('resource_id')}) + if (toolkit.asbool(toolkit.config.get('ckanext.xloader.validation.enforce_schema', True)) + or res_dict.get('schema', None)) and validation_report.get('status') != 'success': + # A schema is present, or required to be present + return + # if validation is running in async mode, it is running from the redis workers. + # thus we need to do sync=True to have Xloader put the job at the front of the queue. + sync = toolkit.asbool(toolkit.config.get(u'ckanext.validation.run_on_update_async', True)) + self._submit_to_xloader(res_dict, sync=sync) + # IDomainObjectModification def notify(self, entity, operation): @@ -96,7 +118,16 @@ def notify(self, entity, operation): if _should_remove_unsupported_resource_from_datastore(resource_dict): toolkit.enqueue_job(fn=_remove_unsupported_resource_from_datastore, args=[entity.id]) - if not getattr(entity, 'url_changed', False): + if utils.requires_successful_validation_report(): + # If the resource requires validation, stop here if validation + # has not been performed or did not succeed. The Validation + # extension will call resource_patch and this method should + # be called again. However, url_changed will not be in the entity + # once Validation does the patch. + log.debug("Deferring xloading resource %s because the " + "resource did not pass validation yet.", resource_dict.get('id')) + return + elif not getattr(entity, 'url_changed', False): # do not submit to xloader if the url has not changed. return @@ -105,6 +136,11 @@ def notify(self, entity, operation): # IResourceController def after_resource_create(self, context, resource_dict): + if utils.requires_successful_validation_report(): + log.debug("Deferring xloading resource %s because the " + "resource did not pass validation yet.", resource_dict.get('id')) + return + self._submit_to_xloader(resource_dict) def before_resource_show(self, resource_dict): diff --git a/ckanext/xloader/templates/package/resource_read.html b/ckanext/xloader/templates/package/resource_read.html index c99dcec2..8ce87bc1 100644 --- a/ckanext/xloader/templates/package/resource_read.html +++ b/ckanext/xloader/templates/package/resource_read.html @@ -4,13 +4,13 @@ {% block resource_read_url %} {% set badge = h.xloader_badge(res) %} {% if badge %} - {{ badge }}

+ {{ badge }}
{% asset 'ckanext-xloader/main-css' %} {% endif %} {{ super() }} {% endblock %} -{% block action_manage_inner %} +{% block action_manage %} {{ super() }} {% if h.is_resource_supported_by_xloader(res) %}
  • {% link_for _('DataStore'), named_route='xloader.resource_data', id=pkg.name, resource_id=res.id, class_='btn btn-light', icon='cloud-upload' %}
  • @@ -23,5 +23,3 @@ {% endif %} {{ super() }} {% endblock %} - - diff --git a/ckanext/xloader/tests/samples/sample_with_blanks.csv b/ckanext/xloader/tests/samples/sample_with_blanks.csv index b53b25db..2b7c415c 100644 --- a/ckanext/xloader/tests/samples/sample_with_blanks.csv +++ b/ckanext/xloader/tests/samples/sample_with_blanks.csv @@ -1,4 +1,4 @@ -Funding agency,Program title,Opening date,Service ID -DTIS,Visitor First Experiences Fund,23/03/2023,63039 -DTIS,First Nations Sport and Recreation Program Round 2,22/03/2023,63040 -,,,63041 +Funding agency,Program title,Opening date,Service ID +DTIS,Visitor First Experiences Fund,23/03/2023,63039 +DTIS,First Nations Sport and Recreation Program Round 2,22/03/2023,63040 +,,,63041 diff --git a/ckanext/xloader/tests/samples/sample_with_extra_blank_cells.csv b/ckanext/xloader/tests/samples/sample_with_extra_blank_cells.csv new file mode 100644 index 00000000..8be1d7de --- /dev/null +++ b/ckanext/xloader/tests/samples/sample_with_extra_blank_cells.csv @@ -0,0 +1,2 @@ +Agency (Dept or Stat Body),Agency address,Contract description/name,Award contract date,Contract value,Supplier name,Supplier address,Variation to contract (Yes/No),Specific confidentiality provision used,Procurement method,Reason for Limited tender,Form of contract,Number of offers sought,Evaluation criteria and weightings,Deliverables,Contract milestones,Contract performance management,,,,,,,,,,,,,,, +State-wide Operations,"111 Easy St, Duckburg, 40000",con_12345-Social services,01/01/1970,"$123,456",LexCorp,123 Example St ELEMENT CITY 4444,No,No,Selective,,,,,,,,,,,,,,,,,,,,,, diff --git a/ckanext/xloader/tests/test_loader.py b/ckanext/xloader/tests/test_loader.py index e8816a13..5cc080a0 100644 --- a/ckanext/xloader/tests/test_loader.py +++ b/ckanext/xloader/tests/test_loader.py @@ -961,6 +961,31 @@ def test_simple(self, Session): u"numeric", u"text", ] + # Check that the sniffed types have been recorded as overrides + rec = p.toolkit.get_action("datastore_search")( + None, {"resource_id": resource_id, "limit": 0} + ) + fields = [f for f in rec["fields"] if not f["id"].startswith("_")] + assert fields[0].get("info", {}).get("type_override", "") == "timestamp" + assert fields[1].get("info", {}).get("type_override", "") == "numeric" + assert fields[2].get("info", {}).get("type_override", "") == "" + + def test_simple_large_file(self, Session): + csv_filepath = get_sample_filepath("simple-large.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert self._get_column_types(Session, resource_id) == [ + u"int4", + u"tsvector", + u"numeric", + u"text", + ] def test_simple_large_file(self, Session): csv_filepath = get_sample_filepath("simple-large.csv") @@ -1242,6 +1267,30 @@ def test_no_entries(self): logger=logger, ) + def test_with_blanks(self, Session): + csv_filepath = get_sample_filepath("sample_with_blanks.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, resource_id)) == 3 + + def test_with_empty_lines(self, Session): + csv_filepath = get_sample_filepath("sample_with_empty_lines.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, resource_id)) == 6 + def test_with_quoted_commas(self, Session): csv_filepath = get_sample_filepath("sample_with_quoted_commas.csv") resource = factories.Resource() @@ -1266,6 +1315,18 @@ def test_with_iso_8859_1(self, Session): ) assert len(self._get_records(Session, resource_id)) == 266 + def test_with_extra_blank_cells(self, Session): + csv_filepath = get_sample_filepath("sample_with_extra_blank_cells.csv") + resource = factories.Resource() + resource_id = resource['id'] + loader.load_table( + csv_filepath, + resource_id=resource_id, + mimetype="text/csv", + logger=logger, + ) + assert len(self._get_records(Session, resource_id)) == 1 + def test_with_mixed_quotes(self, Session): csv_filepath = get_sample_filepath("sample_with_mixed_quotes.csv") resource = factories.Resource() diff --git a/ckanext/xloader/tests/test_plugin.py b/ckanext/xloader/tests/test_plugin.py index 8382e68b..f22dafbd 100644 --- a/ckanext/xloader/tests/test_plugin.py +++ b/ckanext/xloader/tests/test_plugin.py @@ -60,6 +60,93 @@ def test_submit_when_url_changes(self, monkeypatch): assert func.called + @pytest.mark.ckan_config("ckanext.xloader.validation.requires_successful_report", True) + def test_require_validation(self, monkeypatch): + func = mock.Mock() + monkeypatch.setitem(_actions, "xloader_submit", func) + + mock_resource_validation_show = mock.Mock() + monkeypatch.setitem(_actions, "resource_validation_show", mock_resource_validation_show) + + dataset = factories.Dataset() + + resource = helpers.call_action( + "resource_create", + {}, + package_id=dataset["id"], + url="http://example.com/file.csv", + format="CSV", + validation_status='failure', + ) + + # TODO: test IPipeValidation + assert not func.called # because of the validation_status not being `success` + func.called = None # reset + + helpers.call_action( + "resource_update", + {}, + id=resource["id"], + package_id=dataset["id"], + url="http://example.com/file2.csv", + format="CSV", + validation_status='success', + ) + + # TODO: test IPipeValidation + assert not func.called # because of the validation_status is `success` + + @pytest.mark.ckan_config("ckanext.xloader.validation.requires_successful_report", True) + @pytest.mark.ckan_config("ckanext.xloader.validation.enforce_schema", False) + def test_enforce_validation_schema(self, monkeypatch): + func = mock.Mock() + monkeypatch.setitem(_actions, "xloader_submit", func) + + mock_resource_validation_show = mock.Mock() + monkeypatch.setitem(_actions, "resource_validation_show", mock_resource_validation_show) + + dataset = factories.Dataset() + + resource = helpers.call_action( + "resource_create", + {}, + package_id=dataset["id"], + url="http://example.com/file.csv", + schema='', + validation_status='', + ) + + # TODO: test IPipeValidation + assert not func.called # because of the schema being empty + func.called = None # reset + + helpers.call_action( + "resource_update", + {}, + id=resource["id"], + package_id=dataset["id"], + url="http://example.com/file2.csv", + schema='https://example.com/schema.json', + validation_status='failure', + ) + + # TODO: test IPipeValidation + assert not func.called # because of the validation_status not being `success` and there is a schema + func.called = None # reset + + helpers.call_action( + "resource_update", + {}, + package_id=dataset["id"], + id=resource["id"], + url="http://example.com/file3.csv", + schema='https://example.com/schema.json', + validation_status='success', + ) + + # TODO: test IPipeValidation + assert not func.called # because of the validation_status is `success` and there is a schema + @pytest.mark.parametrize("toolkit_config_value, mock_xloader_formats, url_type, datastore_active, expected_result", [ # Test1: Should pass as it is an upload with an active datastore entry but an unsupported format (True, False, 'upload', True, True), diff --git a/ckanext/xloader/utils.py b/ckanext/xloader/utils.py index db8ff06f..bcba510e 100644 --- a/ckanext/xloader/utils.py +++ b/ckanext/xloader/utils.py @@ -11,7 +11,14 @@ from decimal import Decimal import ckan.plugins as p -from ckan.plugins.toolkit import config +from ckan.plugins.toolkit import config, h, _ + +from .job_exceptions import JobError + +from logging import getLogger + + +log = getLogger(__name__) # resource.formats accepted by ckanext-xloader. Must be lowercase here. DEFAULT_FORMATS = [ @@ -46,9 +53,70 @@ def is_it_an_xloader_format(cls, format_): return format_.lower() in cls._formats +def requires_successful_validation_report(): + return p.toolkit.asbool(config.get('ckanext.xloader.validation.requires_successful_report', False)) + + +def awaiting_validation(res_dict): + """ + Checks the existence of a logic action from the ckanext-validation + plugin, thus supporting any extending of the Validation Plugin class. + Checks ckanext.xloader.validation.requires_successful_report config + option value. + Checks ckanext.xloader.validation.enforce_schema config + option value. Then checks the Resource's validation_status. + """ + if not requires_successful_validation_report(): + # validation.requires_successful_report is turned off, return right away + return False + + try: + # check for one of the main actions from ckanext-validation + # in the case that users extend the Validation plugin class + # and rename the plugin entry-point. + p.toolkit.get_action('resource_validation_show') + is_validation_plugin_loaded = True + except KeyError: + is_validation_plugin_loaded = False + + if not is_validation_plugin_loaded: + # the validation plugin is not loaded but required, log a warning + log.warning('ckanext.xloader.validation.requires_successful_report requires the ckanext-validation plugin to be activated.') + return False + + if (p.toolkit.asbool(config.get('ckanext.xloader.validation.enforce_schema', True)) + or res_dict.get('schema', None)) and res_dict.get('validation_status', None) != 'success': + + # either validation.enforce_schema is turned on or it is off and there is a schema, + # we then explicitly check for the `validation_status` report to be `success`` + return True + + # at this point, we can assume that the Resource is not waiting for Validation. + # or that the Resource does not have a Validation Schema and we are not enforcing schemas. + return False + + def resource_data(id, resource_id, rows=None): if p.toolkit.request.method == "POST": + + context = { + "ignore_auth": True, + } + resource_dict = p.toolkit.get_action("resource_show")( + context, + { + "id": resource_id, + }, + ) + + if awaiting_validation(resource_dict): + h.flash_error(_("Cannot upload resource %s to the DataStore " + "because the resource did not pass validation yet.") % resource_id) + return p.toolkit.redirect_to( + "xloader.resource_data", id=id, resource_id=resource_id + ) + try: p.toolkit.get_action("xloader_submit")( None, @@ -231,7 +299,7 @@ def type_guess(rows, types=TYPES, strict=False): else: for i, row in enumerate(rows): diff = len(row) - len(guesses) - for _ in range(diff): + for _i in range(diff): guesses.append(defaultdict(int)) for i, cell in enumerate(row): # add string guess so that we have at least one guess diff --git a/dev-requirements.txt b/dev-requirements.txt index 47fdf35d..592d0d6c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,3 +3,6 @@ mock==2.0.0 flake8 pytest-ckan pytest-cov +requests>=2.32.0 # not directly required, pinned by Snyk to avoid a vulnerability +urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/requirements.txt b/requirements.txt index fe92b6d7..484f4d2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,11 @@ ckantoolkit -requests[security]>=2.11.1 +requests>=2.31.0 six>=1.12.0 tabulator==1.53.5 Unidecode==1.0.22 python-dateutil>=2.8.2 -chardet==5.2.0 \ No newline at end of file +certifi>=2023.7.22 # not directly required, pinned by Snyk to avoid a vulnerability +chardet==5.2.0 +idna>=3.7 # not directly required, pinned by Snyk to avoid a vulnerability +urllib3>=1.26.19 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability