diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eb4efb07..1ba93356a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.10", "3.11"] steps: - name: Checkout code diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f3ae1b897..01cd6e171 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,9 +1,12 @@ Changelog ========= -v32.8.0 (unreleased) +v33.0.0 (unreleased) -------------------- +- Upgrade Django to version 5.0 and drop support for Python 3.8 and 3.9 + https://github.com/nexB/scancode.io/issues/1020 + - Refactor run_scancode to not fail on scan errors happening at the resource level, such as a timeout. Project error message are created instead. https://github.com/nexB/scancode.io/issues/1018 diff --git a/docs/custom-pipelines.rst b/docs/custom-pipelines.rst index 15cd871b6..752aa6cb4 100644 --- a/docs/custom-pipelines.rst +++ b/docs/custom-pipelines.rst @@ -240,7 +240,7 @@ the entry point to the pipeline under the ``[options.entry_points]`` section. packages=find: include_package_data = true zip_safe = false - python_requires = >=3.8 + python_requires = >=3.10 setup_requires = setuptools_scm[toml] >= 4 [options.packages.find] diff --git a/docs/installation.rst b/docs/installation.rst index 053bdc378..c01539495 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -224,7 +224,7 @@ Pre-installation Checklist Before you install ScanCode.io, make sure you have the following prerequisites: - * **Python: versions 3.8 to 3.11** found at https://www.python.org/downloads/ + * **Python: versions 3.10 to 3.11** found at https://www.python.org/downloads/ * **Git**: most recent release available at https://git-scm.com/ * **PostgreSQL**: release 11 or later found at https://www.postgresql.org/ or https://postgresapp.com/ on macOS diff --git a/etc/thirdparty/virtualenv.pyz b/etc/thirdparty/virtualenv.pyz index 867f72082..6ce8c1116 100644 Binary files a/etc/thirdparty/virtualenv.pyz and b/etc/thirdparty/virtualenv.pyz differ diff --git a/etc/thirdparty/virtualenv.pyz.ABOUT b/etc/thirdparty/virtualenv.pyz.ABOUT index 34c3cb3e7..616520c99 100644 --- a/etc/thirdparty/virtualenv.pyz.ABOUT +++ b/etc/thirdparty/virtualenv.pyz.ABOUT @@ -1,7 +1,7 @@ about_resource: virtualenv.pyz name: get-virtualenv -version: 20.24.6 -download_url: https://github.com/pypa/get-virtualenv/raw/20.24.6/public/virtualenv.pyz +version: 20.25.0 +download_url: https://github.com/pypa/get-virtualenv/raw/20.25.0/public/virtualenv.pyz description: virtualenv is a tool to create isolated Python environments. homepage_url: https://github.com/pypa/virtualenv license_expression: lgpl-2.1-plus AND (bsd-new OR apache-2.0) AND mit AND python AND bsd-new @@ -10,4 +10,4 @@ copyright: Copyright (c) The Python Software Foundation and others redistribute: yes attribute: yes track_changes: yes -package_url: pkg:github/pypa/get-virtualenv@20.24.6#public/virtualenv.pyz \ No newline at end of file +package_url: pkg:github/pypa/get-virtualenv@20.25.0#public/virtualenv.pyz \ No newline at end of file diff --git a/scanpipe/apps.py b/scanpipe/apps.py index 100f3d628..c92ab5227 100644 --- a/scanpipe/apps.py +++ b/scanpipe/apps.py @@ -23,6 +23,7 @@ import inspect import logging import sys +import warnings from importlib.machinery import SourceFileLoader from pathlib import Path @@ -74,6 +75,12 @@ def ready(self): # before its running process death. # In ASYNC mode, the cleanup is handled by the "ScanCodeIOWorker" worker. if not settings.SCANCODEIO_ASYNC and "runserver" in sys.argv: + warnings.filterwarnings( + "ignore", + message="Accessing the database during app initialization", + category=RuntimeWarning, + module="django", + ) self.sync_runs_and_jobs() def load_pipelines(self): @@ -82,9 +89,7 @@ def load_pipelines(self): pipelines Python files found at `SCANCODEIO_PIPELINES_DIRS` locations. """ entry_points = importlib_metadata.entry_points() - - # Ignore duplicated entries caused by duplicated paths in `sys.path`. - pipeline_entry_points = set(entry_points.get("scancodeio_pipelines")) + pipeline_entry_points = set(entry_points.select(group="scancodeio_pipelines")) for entry_point in sorted(pipeline_entry_points): self.register_pipeline(name=entry_point.name, cls=entry_point.load()) diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 5a189379a..bbd85e3ee 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -302,19 +302,6 @@ def get_bin_executable(filename): return str(Path(sys.executable).parent / filename) -def remove_prefix(text, prefix): - """ - Remove the `prefix` from `text`. - Note that build-in `removeprefix` was added in Python3.9 but we need to keep - this one for Python3.8 support. - https://docs.python.org/3.9/library/stdtypes.html#str.removeprefix - """ - if text.startswith(prefix): - prefix_len = len(prefix) - return text[prefix_len:] - return text - - class LoopProgress: """ A context manager for logging progress in loops. diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index aad1c6daa..aebfd1c11 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -1402,7 +1402,8 @@ def flag_whitespace_files(project): whitespace_set = set(b" \n\r\t\f\b") for resource in resources: - binary_data = open(resource.location, "rb").read() + with open(resource.location, "rb") as f: + binary_data = f.read() binary_set = set(binary_data) non_whitespace_bytes = binary_set - whitespace_set diff --git a/scanpipe/pipes/resolve.py b/scanpipe/pipes/resolve.py index 519757377..25b343d9b 100644 --- a/scanpipe/pipes/resolve.py +++ b/scanpipe/pipes/resolve.py @@ -30,7 +30,7 @@ from packagedcode import APPLICATION_PACKAGE_DATAFILE_HANDLERS from packagedcode.licensing import get_license_detections_and_expression from packageurl import PackageURL -from python_inspector.resolve_cli import resolver_api +from python_inspector.api import resolve_dependencies from scancode.api import get_package_data from scanpipe.models import DiscoveredPackage @@ -64,7 +64,7 @@ def resolve_pypi_packages(input_location): python_version = f"{sys.version_info.major}{sys.version_info.minor}" operating_system = "linux" - inspector_output = resolver_api( + inspector_output = resolve_dependencies( requirement_files=[input_location], python_version=python_version, operating_system=operating_system, diff --git a/scanpipe/templates/scanpipe/base.html b/scanpipe/templates/scanpipe/base.html index 26614aac3..0ed429d92 100644 --- a/scanpipe/templates/scanpipe/base.html +++ b/scanpipe/templates/scanpipe/base.html @@ -39,6 +39,7 @@ .is-black-link {color: #363636;} .is-grey-link {color: #7a7a7a;} .is-black-link:hover, .is-grey-link:hover {color: #3273dc; text-decoration: underline;} + .navbar button.navbar-item {font-size: 1em;} #inputs-panel .panel-block.dropdown:hover {background-color: #f5f5f5;} #inputs-panel .dropdown-menu {width: 85%;} a.panel-block {word-break: break-all;} diff --git a/scanpipe/templates/scanpipe/includes/navbar_header.html b/scanpipe/templates/scanpipe/includes/navbar_header.html index 37315f80d..39d4d14a5 100644 --- a/scanpipe/templates/scanpipe/includes/navbar_header.html +++ b/scanpipe/templates/scanpipe/includes/navbar_header.html @@ -27,9 +27,12 @@ Profile settings - - Sign out - +
+ {% csrf_token %} + +
ScanCode.io {{ SCANCODEIO_VERSION }} diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 0fa1f748d..2ddcdc720 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -963,7 +963,7 @@ def test_scanpipe_pipes_d2d_get_project_resources_qs(self): "directory1/foo.txt", ] expected_qs = self.project1.codebaseresources.filter(path__in=expected_paths) - self.assertQuerysetEqual(expected_qs, resources_qs) + self.assertQuerySetEqual(expected_qs, resources_qs) def test_scanpipe_pipes_d2d_get_from_files_related_with_not_in_package_to_files( self, @@ -971,15 +971,15 @@ def test_scanpipe_pipes_d2d_get_from_files_related_with_not_in_package_to_files( from_resource1 = make_resource_file(self.project1, "from/foo.java") to_resource1 = make_resource_file(self.project1, "to/foo.class") qs = d2d.get_from_files_related_with_not_in_package_to_files(self.project1) - self.assertQuerysetEqual([], qs) + self.assertQuerySetEqual([], qs) pipes.make_relation(from_resource1, to_resource1, "java_to_class") qs = d2d.get_from_files_related_with_not_in_package_to_files(self.project1) - self.assertQuerysetEqual([], qs) + self.assertQuerySetEqual([], qs) from_resource1.update(detected_license_expression="mit") qs = d2d.get_from_files_related_with_not_in_package_to_files(self.project1) - self.assertQuerysetEqual([from_resource1], qs) + self.assertQuerySetEqual([from_resource1], qs) def test_scanpipe_pipes_d2d_create_local_files_packages(self): from_resource1 = make_resource_file( diff --git a/scanpipe/tests/pipes/test_js.py b/scanpipe/tests/pipes/test_js.py index 4b11f607a..a3ca8f4cb 100644 --- a/scanpipe/tests/pipes/test_js.py +++ b/scanpipe/tests/pipes/test_js.py @@ -133,9 +133,11 @@ def test_scanpipe_pipes_js_get_map_sources_content(self): "resources/adaptive_media/js/main.js.map" ) ) - expected = open(expected_location, "r").read() - result = js.get_map_sources_content(to_resource) + with open(expected_location, "r") as f: + expected = f.read() + + result = js.get_map_sources_content(to_resource) self.assertEqual([expected], result) def test_scanpipe_pipes_js_get_minified_resource(self): diff --git a/scanpipe/tests/pipes/test_scancode.py b/scanpipe/tests/pipes/test_scancode.py index d7cae4a29..a48859b28 100644 --- a/scanpipe/tests/pipes/test_scancode.py +++ b/scanpipe/tests/pipes/test_scancode.py @@ -458,10 +458,10 @@ def test_scanpipe_pipes_scancode_make_results_summary(self, regen=FIXTURES_REGEN uuid = "ba110d49-b6f2-4c86-8d89-a6fd34838ca8" package.update(package_uid=f"pkg:npm/is-npm@1.0.0?uuid={uuid}") - # Patching the ``file_type`` values as those OS dependant. + # Patching the ``file_type`` and ``mime_typea` values as those are OS dependant. # Note that we cannot use proper ``mock`` as the ``scan_package`` pipeline # uses a subprocess call to run the ``scancode`` command. - project1.codebaseresources.all().update(file_type="") + project1.codebaseresources.all().update(file_type="", mime_type="text/plain") scan_output_location = self.data_location / "is-npm-1.0.0_scan_package.json" summary = scancode.make_results_summary(project1, scan_output_location) diff --git a/scanpipe/tests/test_auth.py b/scanpipe/tests/test_auth.py index a43279641..b9e05eb81 100644 --- a/scanpipe/tests/test_auth.py +++ b/scanpipe/tests/test_auth.py @@ -95,15 +95,18 @@ def test_scancodeio_auth_logged_in_navbar_header(self): self.assertContains(response, expected, html=True) expected = f'Profile settings' self.assertContains(response, expected, html=True) - expected = f'Sign out' - self.assertContains(response, expected, html=True) + expected = f'
' + self.assertContains(response, expected) def test_scancodeio_auth_logout_view(self): response = self.client.get(logout_url) + self.assertEqual(405, response.status_code) + + response = self.client.post(logout_url) self.assertRedirects(response, login_url) self.client.login(username=self.basic_user.username, password=TEST_PASSWORD) - response = self.client.get(logout_url) + response = self.client.post(logout_url) self.assertRedirects(response, login_url) def test_scancodeio_account_profile_view(self): diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 4c11be417..cc8ea8278 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -27,6 +27,7 @@ import uuid from contextlib import redirect_stdout from datetime import datetime +from datetime import timezone as tz from pathlib import Path from unittest import mock from unittest import skipIf @@ -754,11 +755,11 @@ def test_scanpipe_run_model_task_execution_time_property(self): self.assertIsNone(run1.execution_time) - run1.task_start_date = datetime(1984, 10, 10, 10, 10, 10, tzinfo=timezone.utc) + run1.task_start_date = datetime(1984, 10, 10, 10, 10, 10, tzinfo=tz.utc) run1.save() self.assertIsNone(run1.execution_time) - run1.task_end_date = datetime(1984, 10, 10, 10, 10, 35, tzinfo=timezone.utc) + run1.task_end_date = datetime(1984, 10, 10, 10, 10, 35, tzinfo=tz.utc) run1.save() self.assertEqual(25.0, run1.execution_time) @@ -771,19 +772,19 @@ def test_scanpipe_run_model_execution_time_for_display_property(self): self.assertIsNone(run1.execution_time_for_display) - run1.task_start_date = datetime(1984, 10, 10, 10, 10, 10, tzinfo=timezone.utc) + run1.task_start_date = datetime(1984, 10, 10, 10, 10, 10, tzinfo=tz.utc) run1.save() self.assertIsNone(run1.execution_time_for_display) - run1.task_end_date = datetime(1984, 10, 10, 10, 10, 35, tzinfo=timezone.utc) + run1.task_end_date = datetime(1984, 10, 10, 10, 10, 35, tzinfo=tz.utc) run1.save() self.assertEqual("25 seconds", run1.execution_time_for_display) - run1.task_end_date = datetime(1984, 10, 10, 10, 12, 35, tzinfo=timezone.utc) + run1.task_end_date = datetime(1984, 10, 10, 10, 12, 35, tzinfo=tz.utc) run1.save() self.assertEqual("145 seconds (2.4 minutes)", run1.execution_time_for_display) - run1.task_end_date = datetime(1984, 10, 10, 11, 12, 35, tzinfo=timezone.utc) + run1.task_end_date = datetime(1984, 10, 10, 11, 12, 35, tzinfo=tz.utc) run1.save() self.assertEqual("3745 seconds (1.0 hours)", run1.execution_time_for_display) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 00e5d93b6..818f56155 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -253,13 +253,16 @@ def test_scanpipe_rootfs_pipeline_extract_input_files_errors(self): self.assertEqual("Error\nError", error.description) -def sort_scanned_files_by_path(scan_data): +def sort_for_os_compatibility(scan_data): """ - Sort the ``scan_data`` files in place. Return ``scan_data``. + Sort the ``scan_data`` files and relations in place. Return ``scan_data``. """ - files = scan_data.get("files") - if files: + if files := scan_data.get("files"): files.sort(key=lambda x: x["path"]) + + if relations := scan_data.get("relations"): + relations.sort(key=lambda x: x["to_resource"]) + return scan_data @@ -292,8 +295,9 @@ class PipelinesIntegrationTest(TestCase): # system_environment differs between systems "system_environment", "file_type", - # mime type is inconsistent across systems + # mime type and is_script are inconsistent across systems "mime_type", + "is_script", "notes", "settings", "description", @@ -381,7 +385,7 @@ def assertPipelineResultEqual( result_json = json.loads(Path(result_file).read_text()) result_json = self._normalize_package_uids(result_json) result_data = self._without_keys(result_json, self.exclude_from_diff) - result_data = sort_scanned_files_by_path(result_data) + result_data = sort_for_os_compatibility(result_data) if regen: expected_file.write_text(json.dumps(result_data, indent=2)) @@ -389,7 +393,7 @@ def assertPipelineResultEqual( expected_json = json.loads(expected_file.read_text()) expected_json = self._normalize_package_uids(expected_json) expected_data = self._without_keys(expected_json, self.exclude_from_diff) - expected_data = sort_scanned_files_by_path(expected_data) + expected_data = sort_for_os_compatibility(expected_data) self.assertEqual(expected_data, result_data) @@ -766,9 +770,9 @@ def test_scanpipe_inspect_manifest_pipeline_integration_test(self): self.assertEqual(1, exitcode, msg=out) self.assertIn("No package type found for", out) - @mock.patch("scanpipe.pipes.resolve.resolver_api") + @mock.patch("scanpipe.pipes.resolve.resolve_dependencies") def test_scanpipe_inspect_manifest_pipeline_pypi_integration_test( - self, resolver_api + self, resolve_dependencies ): pipeline_name = "inspect_manifest" project1 = Project.objects.create(name="Analysis") @@ -776,13 +780,13 @@ def test_scanpipe_inspect_manifest_pipeline_pypi_integration_test( run = project1.add_pipeline(pipeline_name) pipeline = run.make_pipeline_instance() - resolver_api.return_value = mock.Mock(packages=[]) + resolve_dependencies.return_value = mock.Mock(packages=[]) project1.move_input_from(tempfile.mkstemp(suffix="requirements.txt")[1]) exitcode, out = pipeline.execute() self.assertEqual(1, exitcode, msg=out) self.assertIn("No packages could be resolved", out) - resolver_api.return_value = mock.Mock(packages=[package_data1]) + resolve_dependencies.return_value = mock.Mock(packages=[package_data1]) exitcode, out = pipeline.execute() self.assertEqual(0, exitcode, msg=out) diff --git a/setup.cfg b/setup.cfg index eb0b8f467..f2aed4380 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,8 +14,6 @@ classifiers = Intended Audience :: Legal Industry Programming Language :: Python Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Topic :: Utilities @@ -45,27 +43,27 @@ license_files = scan.NOTICE [options] -python_requires = >=3.8 +python_requires = >=3.10 packages=find: include_package_data = true zip_safe = false install_requires = - importlib-metadata==6.8.0 + importlib-metadata==7.0.0 # Django related - Django==4.2.6 + Django==5.0.0 django-environ==0.11.2 django-crispy-forms==2.1 crispy-bootstrap3==2022.1 - django-filter==23.3 + django-filter==23.5 djangorestframework==3.14.0 - django-taggit==4.0.0 + django-taggit==5.0.1 # Database - psycopg[binary]==3.1.12 + psycopg[binary]==3.1.14 # wait_for_database Django management command django-probes==1.7.0 # Task queue rq==1.15.1 - django-rq==2.8.1 + django-rq==2.9.0 redis==5.0.1 # WSGI server gunicorn==21.2.0 @@ -78,7 +76,7 @@ install_requires = # FetchCode fetchcode-container==1.2.3.210512; sys_platform == "linux" # Inspectors - python-inspector==0.9.8 + python-inspector==0.10.0 aboutcode-toolkit==10.1.0 # Utilities XlsxWriter==3.1.9 @@ -86,14 +84,14 @@ install_requires = requests==2.31.0 gitpython==3.1.40 # Profiling - pyinstrument==4.6.0 + pyinstrument==4.6.1 # SPDX jsonschema==4.19.1 # CycloneDX cyclonedx-python-lib==3.1.5 hoppr-cyclonedx-models==0.4.10 # Font Awesome - fontawesomefree==6.4.2 + fontawesomefree==6.5.1 # MatchCode-toolkit matchcode-toolkit==1.1.3 # Univers @@ -103,17 +101,17 @@ install_requires = dev = # Validation flake8==6.1.0 - black==23.10.1 - isort==5.12.0 - doc8==0.11.2 + black==23.11.0 + isort==5.13.0 + doc8==1.1.1 pydocstyle==6.3.0 # Security analyzer - bandit==1.7.5 + bandit==1.7.6 # Debug django-debug-toolbar==4.2.0 # Documentation - Sphinx==7.1.2 - sphinx-rtd-theme==1.3.0 + Sphinx==7.2.6 + sphinx-rtd-theme==2.0.0 sphinx-rtd-dark-mode==1.3.0 sphinxcontrib-django==2.5 # Release @@ -147,7 +145,7 @@ skip = lib,migrations,bin,data,Script,Lib,lib64,local,var,tmp,thirdparty,build,d [flake8] max_line_length = 88 max_complexity = 10 -exclude = lib,thirdparty,docs,bin,migrations,settings.py,data,var,build,dist +exclude = lib,thirdparty,docs,bin,migrations,settings.py,data,var,build,dist,.cache ignore = E203,W503 [pydocstyle]