global: bump python, update ci, convert to pyproject.toml

*ref: cern-sis/issues-inspire/issues/588

.github/workflows/build-and-release.yml

@@ -1,75 +1,50 @@
-name: Build and release
+name: Pull request master (lint and test)
 
 on:
-  push:
-    branches: [master]
-  pull_request:
-    branches: [master]
+    push:
+      branches: [master]
+    pull_request:
+      branches: [master]
 
-env:
-  REGISTRY: registry.cern.ch
-  IMAGE: cern-sis/inspirehep/refextract
 
 jobs:
-  test_and_publish:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install dependencies
-        run: sudo apt-get update && sudo apt-get install poppler-utils
-
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Install python dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel
-          pip install -e .[docs,tests]
-
-      - name: Show python dependencies
-        run: |
-          pip freeze
-
-      - name: Run tests
-        run: |
-          ./run-tests.sh
-
-      - name: Build package
-        if: ${{ success() && github.event_name == 'push' }}
-        run: |
-          python setup.py sdist bdist_wheel
-
-  push_to_registry_and_deploy:
-    runs-on: ubuntu-20.04
-    needs: [test_and_publish]
-    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - name: Pre-commit check
-        uses: pre-commit/[email protected]
-      - name: Build Image
-        id: build
-        uses: cern-sis/gh-workflows/.github/actions/docker-build@v6
-        with:
-          registry: ${{ env.REGISTRY }}
-          image: ${{ env.IMAGE }}
-          cache: false
-          username: ${{ secrets.HARBOR_USERNAME }}
-          password: ${{ secrets.HARBOR_PASSWORD }}
-
-      - name: Deploy QA
-        uses: cern-sis/gh-workflows/.github/actions/kubernetes-project-new-images@v6
-        with:
-          event-type: update
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE }}@${{ steps.build.outputs.image-digest }}
-          token: ${{ secrets.PAT_FIRE_EVENTS_ON_CERN_SIS_KUBERNETES }}
+    lint_and_test:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout Code
+              uses: actions/checkout@v4
+              with:
+                ref: ${{ github.ref }}
+            - name: Lint - Pre-commit check
+              uses: pre-commit/[email protected]
+            - name: Prep Build
+              uses: docker/setup-buildx-action@v3
+            - name: Build Docker image
+              run: docker build --target refextract-tests -t refextract .
+            - name: Run tests
+              run: >
+                docker run
+                --entrypoint poetry
+                refextract
+                run pytest
+    push_and_deploy:
+        runs-on: ubuntu-latest
+        needs: [lint_and_test]
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        steps:
+        - name: Build Image
+          id: build
+          uses: cern-sis/gh-workflows/.github/actions/docker-build@v6
+          with:
+            registry: ${{ env.REGISTRY }}
+            stage: refextract
+            image: ${{ env.IMAGE }}
+            cache: false
+            username: ${{ secrets.HARBOR_USERNAME }}
+            password: ${{ secrets.HARBOR_PASSWORD }}
+        - name: Deploy QA
+          uses: cern-sis/gh-workflows/.github/actions/kubernetes-project-new-images@v6
+          with:
+            event-type: update
+            images: ${{ env.REGISTRY }}/${{ env.IMAGE }}@${{ steps.build.outputs.image-digest }}
+            token: ${{ secrets.PAT_FIRE_EVENTS_ON_CERN_SIS_KUBERNETES }}

Dockerfile

@@ -1,7 +1,22 @@
-FROM python:3.8
-RUN apt update && apt install poppler-utils -y
-COPY setup.py setup.cfg README.rst ./
+FROM python:3.11.6-slim-bullseye AS refextract
+
+ARG APP_HOME=/refextract
+WORKDIR ${APP_HOME}
+
+RUN apt update && apt install poppler-utils libmagic1 -y
+COPY poetry.lock pyproject.toml setup.cfg README.rst ${APP_HOME}
+
+RUN pip install --no-cache-dir poetry
+RUN poetry config virtualenvs.create false \
+    && poetry install --only main
+
 COPY refextract refextract/
-RUN python setup.py install
+
+
 ENV PROMETHEUS_MULTIPROC_DIR='/tmp'
 ENTRYPOINT exec gunicorn -b :5000 --access-logfile - --error-logfile - refextract.app:app --timeout 650
+
+FROM refextract AS refextract-tests
+
+COPY tests tests/
+RUN poetry install

pyproject.toml

@@ -0,0 +1,53 @@
+[tool.poetry]
+name = "refextract"
+version = "0.1.0"
+description = "Small library for extracting references used in scholarly communication."
+readme = "README.rst"
+homepage = "https://github.com/inspirehep/refextract"
+license = "GPL-2.0-or-later"
+authors = [
+    "CERN <[email protected]>"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Utilities",
+]
+
+
+[tool.poetry.dependencies]
+python = ">=3.11,<4"
+PyPDF2 = ">=1.26.0,~=1.0"
+six = ">=1.10.0,~=1.0"
+unidecode = ">=1.0.22,~=1.0"
+Flask = ">=2.0.3"
+webargs = "<=5.4.0"
+prometheus-flask-exporter = ">=0.20.1,~=0.20"
+gunicorn = "^23.0.0"
+python-magic = "^0.4.27"
+inspire-utils = "^3.0.61"
+requests = "^2.32.3"
+urllib3 = "1.26.12"
+
+
+[tool.poetry.group.dev.dependencies]
+mock = "^5.1.0"
+responses ="^0.10.15"
+pytest = "^8.3.3"
+pytest-cov = "^5.0.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

refextract/references/api.py

@@ -152,7 +152,8 @@ def extract_references_from_file(path,
         extracted_texkeys_urls = extract_texkeys_and_urls_from_pdf(path)
         if len(extracted_texkeys_urls) == len(parsed_refs):
             parsed_refs_updated = []
-            for ref, ref_texkey_urls in zip(parsed_refs, extracted_texkeys_urls):
+            for ref, ref_texkey_urls in zip(parsed_refs, extracted_texkeys_urls,
+                                            strict=False):
                 update_reference_with_urls(ref, ref_texkey_urls.get('urls', []))
                 if ref.get('url'):
                     ref['url'] = dedupe_list(ref['url'])

ruff.toml

@@ -1,4 +1,4 @@
-target-version = "py38"
+target-version = "py311"
 [lint.flake8-tidy-imports]
 ban-relative-imports = "all"
 

setup.cfg

@@ -30,6 +30,3 @@ include = refextract/*.py
 
 [tool:pytest]
 addopts = --cov=refextract --cov-report=term-missing:skip-covered
-
-[flake8]
-ignore = *.py E501 FI12 FI14 FI15 FI16 FI17 FI18 FI50 FI51 FI53 W504 W605 FI10 FI11 FI13

tests/test_kbs.py

@@ -42,7 +42,7 @@ def test_get_kbs_caches_journal_dict():
     # the cache is reused, so identity of the cache elements doesn't change
     assert all(
         cached_first is cached_second for (cached_first, cached_second)
-        in zip(first_cache["journals"], second_cache["journals"])
+        in zip(first_cache["journals"], second_cache["journals"], strict=False)
     )
 
 
@@ -55,7 +55,7 @@ def test_get_kbs_invalidates_cache_if_input_changes():
     # the cache is invalidated, so identity of the cache elements changes
     assert all(
         cached_first is not cached_second for (cached_first, cached_second)
-        in zip(first_cache["journals"], second_cache["journals"])
+        in zip(first_cache["journals"], second_cache["journals"], strict=False)
     )
     assert len(second_cache["journals"]) == 3
     assert second_cache["journals"][-1] == ["JOURNAL OF TESTING", "J TEST"]