From 3c971c0c852d16941621e7edb428fff58e362149 Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 09:52:40 +0200 Subject: [PATCH 01/11] Add npm build step to pyproject.toml --- pyproject.toml | 50 +++++++++++++++++++------------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 26a6969..9136bc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling", "hatch-requirements-txt"] +requires = ["hatchling", "hatch-requirements-txt", "hatch-build-scripts"] build-backend = "hatchling.build" [project] @@ -9,23 +9,17 @@ readme = "README.md" requires-python = ">=3.9" license = "MIT" keywords = [] -authors = [ - { name = "Jelmer van der Linde", email = "jelmer@ikhoefgeen.nl" }, -] +authors = [{ name = "Jelmer van der Linde", email = "jelmer@ikhoefgeen.nl" }] classifiers = [ - "Development Status :: 4 - Beta", - "Programming Language :: Python", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", -] -dynamic = [ - "version", - "dependencies", - "optional-dependencies" + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", ] +dynamic = ["version", "dependencies", "optional-dependencies"] [project.scripts] opuscleaner-server = "opuscleaner.server:main" @@ -44,10 +38,7 @@ Source = "https://github.com/hplt-project/opuscleaner" path = "opuscleaner/__about__.py" [tool.hatch.envs.default] -dependencies = [ - "pytest", - "pytest-cov", -] +dependencies = ["pytest", "pytest-cov"] [tool.hatch.envs.default.scripts] cov = "pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=opuscleaner --cov=tests {args}" no-cov = "cov --no-cov {args}" @@ -56,13 +47,16 @@ no-cov = "cov --no-cov {args}" python = ["39", "310", "311"] [tool.hatch.build] -include = [ - "/opuscleaner", -] +include = ["/opuscleaner"] [tool.hatch.build.force-include] "frontend/dist" = "/opuscleaner/frontend" +[[tool.hatch.build.hooks.build-scripts.scripts]] +work_dir = "frontend" +commands = ["npm clean-install && npm run build"] +artifacts = ["/frontend/dist"] + [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"] @@ -72,13 +66,7 @@ all = ["requirements-all.txt"] [tool.coverage.run] branch = true parallel = true -omit = [ - "opuscleaner/__about__.py", -] +omit = ["opuscleaner/__about__.py"] [tool.coverage.report] -exclude_lines = [ - "no cov", - "if __name__ == .__main__.:", - "if TYPE_CHECKING:", -] +exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] From 659f616e820aac1ee1d858ff7a1d0b5607898aa2 Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 10:03:53 +0200 Subject: [PATCH 02/11] Simplify github workflow --- .github/workflows/release.yaml | 101 ++++++++------------------------- 1 file changed, 23 insertions(+), 78 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5835d91..25ae27f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,51 +7,7 @@ on: - published jobs: - build_frontend: - name: Build frontend Javascript code - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: Build frontend - working-directory: frontend - run: npm ci && npm run build - - - uses: actions/upload-artifact@v3 - with: - name: frontend - path: frontend/dist/ - - run_tests: - needs: [build_frontend] - name: Run tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v3 - - - uses: actions/download-artifact@v3 - with: - name: frontend - path: frontend/dist - - - name: Install - run: python3 -m pip install . - - - name: Run runtime unittest - run: python3 -m unittest discover -s test - - - name: Run filters unittest - run: python3 -m unittest discover -s opuscleaner/filters - - build_sdist: - needs: [build_frontend] + build: name: Build source distribution runs-on: ubuntu-latest steps: @@ -59,33 +15,31 @@ jobs: - uses: actions/setup-python@v3 - - uses: actions/download-artifact@v3 + - uses: actions/setup-node@v3 with: - name: frontend - path: frontend/dist + node-version: 22 - name: Set version number if: startsWith(github.ref, 'refs/tags/v') run: echo "VERSION = \"${GITHUB_REF_NAME:1}\"" > opuscleaner/__about__.py - - - name: Build sdist - run: pipx run build --sdist + + - name: Build + run: python -m build - uses: actions/upload-artifact@v3 with: name: sdist path: dist/opuscleaner-*.tar.gz - build_wheels: - needs: [build_frontend] - name: Build wheels - runs-on: ubuntu-latest - # name: Build wheels on ${{ matrix.os }} - # runs-on: ${{ matrix.os }} - # strategy: - # matrix: - # os: [ubuntu-latest, windows-2019, macos-13] + - uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist/opuscleaner-*.whl + run_tests: + needs: [build] + name: Run tests + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -93,29 +47,20 @@ jobs: - uses: actions/download-artifact@v3 with: - name: frontend - path: frontend/dist + name: wheels + path: dist - - name: Set version number - if: startsWith(github.ref, 'refs/tags/v') - run: echo "VERSION = \"${GITHUB_REF_NAME:1}\"" > opuscleaner/__about__.py - - - name: Build wheels - run: python -m pip wheel -w wheelhouse . - - # - name: Install cibuildwheel - # run: python -m pip install cibuildwheel==2.12.0 + - name: Install + run: python3 -m pip install dist/opuscleaner-*.tar.gz - # - name: Build wheels - # run: python -m cibuildwheel --output-dir wheelhouse + - name: Run runtime unittest + run: python3 -m unittest discover -s test - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: ./wheelhouse/opuscleaner-*.whl + - name: Run filters unittest + run: python3 -m unittest discover -s opuscleaner/filters upload_pypi: - needs: [build_wheels, build_sdist] + needs: [build] runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' steps: From a5d301550a35e4c277d1a82ec5e891cfbc7d7da7 Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 11:01:58 +0200 Subject: [PATCH 03/11] Apparently build is not installed by default --- .github/workflows/release.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 25ae27f..976fe4a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,6 +19,9 @@ jobs: with: node-version: 22 + - name: Install build + run: python -m pip install build + - name: Set version number if: startsWith(github.ref, 'refs/tags/v') run: echo "VERSION = \"${GITHUB_REF_NAME:1}\"" > opuscleaner/__about__.py From 9d0c45587a5f8618b1c1470dab7fc1f88ccf54ef Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 11:09:13 +0200 Subject: [PATCH 04/11] Try installing from wheels --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 976fe4a..0974197 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -54,7 +54,7 @@ jobs: path: dist - name: Install - run: python3 -m pip install dist/opuscleaner-*.tar.gz + run: python3 -m pip install dist/opuscleaner-*.whl - name: Run runtime unittest run: python3 -m unittest discover -s test From c42843585e2d55ab02a4c438ef3fe5d592fb7d27 Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 12:03:55 +0200 Subject: [PATCH 05/11] Try just sdist builds? --- .github/workflows/release.yaml | 16 +++------------- pyproject.toml | 3 ++- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0974197..06afdcd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,18 +27,13 @@ jobs: run: echo "VERSION = \"${GITHUB_REF_NAME:1}\"" > opuscleaner/__about__.py - name: Build - run: python -m build + run: python -m build --sdist - uses: actions/upload-artifact@v3 with: name: sdist path: dist/opuscleaner-*.tar.gz - - uses: actions/upload-artifact@v3 - with: - name: wheels - path: dist/opuscleaner-*.whl - run_tests: needs: [build] name: Run tests @@ -50,11 +45,11 @@ jobs: - uses: actions/download-artifact@v3 with: - name: wheels + name: sdist path: dist - name: Install - run: python3 -m pip install dist/opuscleaner-*.whl + run: python3 -m pip install dist/opuscleaner-*.tar.gz - name: Run runtime unittest run: python3 -m unittest discover -s test @@ -67,11 +62,6 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' steps: - - uses: actions/download-artifact@v3 - with: - name: wheels - path: dist - - uses: actions/download-artifact@v3 with: name: sdist diff --git a/pyproject.toml b/pyproject.toml index 9136bc2..56f7823 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ path = "opuscleaner/__about__.py" [tool.hatch.envs.default] dependencies = ["pytest", "pytest-cov"] + [tool.hatch.envs.default.scripts] cov = "pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=opuscleaner --cov=tests {args}" no-cov = "cov --no-cov {args}" @@ -52,7 +53,7 @@ include = ["/opuscleaner"] [tool.hatch.build.force-include] "frontend/dist" = "/opuscleaner/frontend" -[[tool.hatch.build.hooks.build-scripts.scripts]] +[[tool.hatch.build.target.sdist.hooks.build-scripts.scripts]] work_dir = "frontend" commands = ["npm clean-install && npm run build"] artifacts = ["/frontend/dist"] From 50c4a3bcec68be8088ed9ca854c9aa111703d20e Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 12:06:21 +0200 Subject: [PATCH 06/11] Fix typo --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 56f7823..4f83987 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ include = ["/opuscleaner"] [tool.hatch.build.force-include] "frontend/dist" = "/opuscleaner/frontend" -[[tool.hatch.build.target.sdist.hooks.build-scripts.scripts]] +[[tool.hatch.build.targets.sdist.hooks.build-scripts.scripts]] work_dir = "frontend" commands = ["npm clean-install && npm run build"] artifacts = ["/frontend/dist"] From 42a574e1bbd9cd065d151cbfef696298b9994a5a Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 14:39:41 +0200 Subject: [PATCH 07/11] Try custom script --- pyproject.toml | 10 +++++----- utils/frontend_build_hook.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 utils/frontend_build_hook.py diff --git a/pyproject.toml b/pyproject.toml index 4f83987..da4fe1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling", "hatch-requirements-txt", "hatch-build-scripts"] +requires = ["hatchling", "hatch-requirements-txt"] build-backend = "hatchling.build" [project] @@ -53,10 +53,10 @@ include = ["/opuscleaner"] [tool.hatch.build.force-include] "frontend/dist" = "/opuscleaner/frontend" -[[tool.hatch.build.targets.sdist.hooks.build-scripts.scripts]] -work_dir = "frontend" -commands = ["npm clean-install && npm run build"] -artifacts = ["/frontend/dist"] +[tool.hatch.build.hooks.custom] +path = "utils/frontend_build_hook.py" +working_dir = "frontend" +artifacts = ["frontend/dist"] [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"] diff --git a/utils/frontend_build_hook.py b/utils/frontend_build_hook.py new file mode 100644 index 0000000..0b50ea7 --- /dev/null +++ b/utils/frontend_build_hook.py @@ -0,0 +1,18 @@ +import subprocess +import os +from typing import Any + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class NpmBuildHook(BuildHookInterface): + def initialize(self, version: str, build_data: dict[str, Any]) -> None: + if all(os.path.exists(artifact) for artifact in self.config.get("artifacts", [])): + return + subprocess.check_output( + "npm install && npm run build", + shell=True, + cwd=self.config.get("working_dir", ".")) + + def clean(self, versions: list[str]) -> None: + subprocess.check_output("npm run clean", shell=True) From 6e5c6a932cadbffb1792e2c7fe73567d7f5b0ad8 Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 16:02:43 +0200 Subject: [PATCH 08/11] Retry --- .github/workflows/release.yaml | 6 ++---- pyproject.toml | 4 ++-- utils/frontend_build_hook.py | 18 ++++++++++++++++-- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 06afdcd..0f845a6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -31,8 +31,7 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: sdist - path: dist/opuscleaner-*.tar.gz + path: dist/opuscleaner-*.* run_tests: needs: [build] @@ -45,11 +44,10 @@ jobs: - uses: actions/download-artifact@v3 with: - name: sdist path: dist - name: Install - run: python3 -m pip install dist/opuscleaner-*.tar.gz + run: python3 -m pip install dist/opuscleaner-*.whl - name: Run runtime unittest run: python3 -m unittest discover -s test diff --git a/pyproject.toml b/pyproject.toml index da4fe1e..3b5c5b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,10 +50,10 @@ python = ["39", "310", "311"] [tool.hatch.build] include = ["/opuscleaner"] -[tool.hatch.build.force-include] +[tool.hatch.build.targets.sdist.force-include] "frontend/dist" = "/opuscleaner/frontend" -[tool.hatch.build.hooks.custom] +[tool.hatch.build.targets.sdist.hooks.custom] path = "utils/frontend_build_hook.py" working_dir = "frontend" artifacts = ["frontend/dist"] diff --git a/utils/frontend_build_hook.py b/utils/frontend_build_hook.py index 0b50ea7..8b71a0d 100644 --- a/utils/frontend_build_hook.py +++ b/utils/frontend_build_hook.py @@ -1,18 +1,32 @@ import subprocess import os from typing import Any +import logging from hatchling.builders.hooks.plugin.interface import BuildHookInterface +logger = logging.getLogger(__name__) + + class NpmBuildHook(BuildHookInterface): + @property + def working_dir(self): + return self.config.get("working_dir", ".") + def initialize(self, version: str, build_data: dict[str, Any]) -> None: + # if we can't build because we don't have the build dir, skip + if not os.path.exists(self.working_dir): + logger.info(f"Skipping npm build because {self.working_dir=} does not exist") + return + # if we don't need to build because we have all the artifacts, skip if all(os.path.exists(artifact) for artifact in self.config.get("artifacts", [])): + logger.info(f"Skipping npm build because all artifacts exist") return subprocess.check_output( "npm install && npm run build", shell=True, - cwd=self.config.get("working_dir", ".")) + cwd=self.working_dir) def clean(self, versions: list[str]) -> None: - subprocess.check_output("npm run clean", shell=True) + subprocess.check_output("npm run clean", shell=True, cwd=self.working_dir) From e3ec16ed5cbfb217d2acc5e0c3297015a1db8716 Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 16:04:57 +0200 Subject: [PATCH 09/11] Build the wheel again as well --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0f845a6..cc91275 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,7 +27,7 @@ jobs: run: echo "VERSION = \"${GITHUB_REF_NAME:1}\"" > opuscleaner/__about__.py - name: Build - run: python -m build --sdist + run: python -m build - uses: actions/upload-artifact@v3 with: From f2a46c409d4fe735705878f1bc24ce75754b8024 Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 16:38:07 +0200 Subject: [PATCH 10/11] Workflows wtf --- .github/workflows/release.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index cc91275..1c9b82d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -31,6 +31,7 @@ jobs: - uses: actions/upload-artifact@v3 with: + name: build-output path: dist/opuscleaner-*.* run_tests: @@ -44,6 +45,7 @@ jobs: - uses: actions/download-artifact@v3 with: + name: build-output path: dist - name: Install @@ -62,7 +64,7 @@ jobs: steps: - uses: actions/download-artifact@v3 with: - name: sdist + name: build-output path: dist - uses: pypa/gh-action-pypi-publish@v1.5.0 From c6b8195eb8be1950ab3a69145b6b61601c9abaca Mon Sep 17 00:00:00 2001 From: Jelmer van der Linde Date: Tue, 3 Sep 2024 16:44:19 +0200 Subject: [PATCH 11/11] Clean up readme --- README.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index aedce2e..cc36c9e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ If you just want to use OpusCleaner for cleaning, you can install it from PyPI, ```sh pip3 install opuscleaner -opuscleaner-server serve +opuscleaner-server ``` Then you can go to http://127.0.0.1:8000/ to show the interface. @@ -56,12 +56,9 @@ If you want to use another path, you can use the `DATA_PATH` environment variabl ### Installation for development -```sh -cd frontend -npm clean-install -npm run build -cd .. +For building from source (i.e. git, not anything downloaded from Pypi) you'll need to have node + npm installed. +```sh python3 -m venv .env bash --init-file .env/bin/activate pip install -e . @@ -105,4 +102,3 @@ To push a new release to Pypi from Github, tag a commit with a `vX.Y.Z` version # Acknowledgements This project has received funding from the European Union’s Horizon Europe research and innovation programme under grant agreement No 101070350 and from UK Research and Innovation (UKRI) under the UK government’s Horizon Europe funding guarantee [grant number 10052546] -