Release 1.0.0rc1 (#24)

* migrate to poetry (#3)

* migrate to poetry

* move poetry install first

* update lockfile

* fix script declaration

* update docs

* bump workflow versions

* Support more Python versions (#4)

* test up to py3.12

* drop py3.8 as it's nearly eol

* bump package versions

* Fix removed numpy NaN

* update py version in readme

* Change linting to Ruff (#5)

* Install ruff

* remove makefile

* ruff check fix

* manual ruff fixes

* undo .at to .loc with noqa

* update lint workflow

* delete old flake8 config

* fix CI lint commands

* convert pickle fixtures to json tables (#6)

* Remove yoast plugin requirement (#12)

* better handling without yoast plugin

* run ruff

* Better support no scrape (#13)

* Allow empty scrape properly

* ruff lint

* Integrate scraping (#15)

* incorporate wp-json-scraper module

* ruff autofix

* docconvert

* manual fixes

* fix noqa comment

* refactor cli to support subcommands

* Basic integration of downloader

* swap progress bar implementation

* ruff

* remove object display code

* add media dl support

* lint

* remove totally unused modules

* remove unused csv export code

* ruff

* implement proxy/cookies/auth on download client

* improve subparser documentation

* ruff

* add NOTICE file

* remove plugin list

* remove console util

* dl utils cleanup

* implement request robustness features

* make media dl use request session

* Add docs (#18)

* docs

* ruff

* fix broken test

* fix typing union

* add deploy dir to gitignore

* change project name to match package name

* hotfix: remove docconvert

* hotfix: fix package name for version get

* Packaging improvements (#19)

* package meta and readme updates

* relax dependency constraints

* regen lock

* readme tweaks

* put back gh markdown admonition

* rename main package for consistency (#20)

* rename main package

* ruff

* Feature/prefix consistency (#21)

* wip

* make prefix behaviour consistent between commands

* update cli docs

* Some tests for downloader (#22)

* add downloader tests

* test no prefix downloader too

* test wpapi and exporter

* ruff

* hotfix: print and wrong command name

* fix dl input directory path

* Prerelease restructure (#23)

* expose dl api a bit better

* improve docs for dl module

* enable pyupgrade rules

* enable ruff specific rules

* remove some doc noqas

* ruff reformat

* fix export decorator usage

* change dl media to use pathlib

* fix build system in pyproject

* minor cli and docs fixes

* add changelog for 1.0.0

* clean up naming of translation pickers

* set version to 1.0.0rc1

* change pytest to use importlib mode (#25)

* Change CLI to use click (#26)

* swap to click for cli

* ruff

* remove old argparse support

* use download instead of dl consistently

* update docs for cli

* ruff

* Feature/packaging meta (#27)

* add classifiers

* update license docs to match new path

* rename repo

* Remove print statements (#28)

* fix name in notice file

* replace prints with logs

* add cli version

* Ruff

* prepare release 1.0.0

* add publish workflow

* Add building to CI test

.github/workflows/lint.yml

@@ -2,9 +2,9 @@ name: Lint
 
 on:
   push:
-    branches: ["main"]
+    branches: ["main", "dev"]
   pull_request:
-    branches: ["main"]
+    branches: ["main", "dev"]
 
 permissions:
   contents: read
@@ -14,19 +14,22 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v3
+      - uses: actions/checkout@v4
+      - name: Install Poetry
+        run: pipx install poetry==1.8.3
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
-          cache: "pipenv"
-      - name: Install Pipenv
-        run: python -m pip install --upgrade pipenv wheel
+          python-version: "3.9"
+          cache: "poetry"
       - name: Install Python dependencies
-        run: pipenv install --deploy --dev
-      - name: Lint with Black
-        run: pipenv run black --check .
-      - name: Lint with Flake8
-        run: pipenv run flake8 .
-      - name: Tests
-        run: pipenv run pytest
+        run: poetry install --no-interaction --no-root
+      - name: Install Project
+        run: poetry install --no-interaction
+      - name: Ruff Format Check
+        run: poetry run ruff format --check
+        id: format
+      - name: Ruff Lint Check
+        run: poetry run ruff check --output-format=github
+        # Still run if format check fails
+        if: success() || steps.format.conclusion == 'failure'

.github/workflows/publish.yml

@@ -0,0 +1,91 @@
+name: Push to PyPI
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build Distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Poetry
+        run: pipx install poetry==1.8.3
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+          cache: "poetry"
+      - name: Install Python dependencies
+        run: poetry install --no-interaction --no-root
+      - name: Build
+        run: poetry build
+      - name: Store distribution packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: python-package-distributions
+          path: dist/
+  publish-to-pypi:
+    name: Publish to PyPI
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://pypi.org/p/wpextract
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distribution packages
+        uses: actions/download-artifact@v3
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+#  github-release:
+#    name: GitHub release
+#    needs:
+#      - publish-to-pypi
+#    runs-on: ubuntu-latest
+#
+#    permissions:
+#      contents: write
+#      id-token: write
+#
+#    steps:
+#    - name: Download distribution packages
+#      uses: actions/download-artifact@v3
+#      with:
+#        name: python-package-distributions
+#        path: dist/
+#    - name: Sign the dists with Sigstore
+#      uses: sigstore/[email protected]
+#      with:
+#        inputs: >-
+#          ./dist/*.tar.gz
+#          ./dist/*.whl
+#    - name: Create GitHub Release
+#      env:
+#        GITHUB_TOKEN: ${{ github.token }}
+#      run: >-
+#        gh release create
+#        '${{ github.ref_name }}'
+#        --repo '${{ github.repository }}'
+#        --notes ""
+#    - name: Upload artifact signatures to GitHub Release
+#      env:
+#        GITHUB_TOKEN: ${{ github.token }}
+#      # Upload to GitHub Release using the `gh` CLI.
+#      # `dist/` contains the built packages, and the
+#      # sigstore-produced signatures and certificates.
+#      run: >-
+#        gh release upload
+#        '${{ github.ref_name }}' dist/**
+#        --repo '${{ github.repository }}'

.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: Test
 
 on:
   push:
-    branches: ["main"]
+    branches: ["main", "dev"]
   pull_request:
-    branches: ["main"]
+    branches: ["main", "dev"]
 
 permissions:
   contents: read
@@ -13,19 +13,24 @@ jobs:
   test:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v3
-        with:
-          python-version: "3.8"
-          cache: "pipenv"
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - name: Install Poetry
+        run: pipx install poetry==1.8.3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
-          node-version: 16
-      - name: Install Pipenv
-        run: python -m pip install --upgrade pipenv wheel
+          python-version: ${{ matrix.python-version }}
+          cache: "poetry"
       - name: Install Python dependencies
-        run: pipenv install --deploy --dev
+        run: poetry install --no-interaction --no-root
+      - name: Install Project
+        run: poetry install --no-interaction
+      - name: Build Project
+        run: poetry build
       - name: Run tests
-        run: pipenv run pytest
+        run: poetry run pytest

.gitignore

@@ -61,4 +61,7 @@ MANIFEST
 
 ## == JAVASCRIPT ==
 
-node_modules/
+node_modules/
+
+## == DOCUMENTATION ==
+site/

CITATION.cff

@@ -10,11 +10,11 @@ authors:
   - given-names: Carolina
     family-names: Scarton
     orcid: 'https://orcid.org/0000-0002-0103-4072'
-title: "wordpress-site-extractor"
+title: "WPextract"
 version: 1.0.0
 doi: 10.5281/zenodo.10008086
 date-released: 2023-10-20
-url: "https://github.com/GateNLP/wordpress-site-extractor"
+url: "https://github.com/GateNLP/wpextract"
 preferred-citation:
   type: conference-paper
   authors:

Makefile

@@ -1,19 +1,28 @@
-.PHONY: black
-black:
-	black .
+.PHONY: format
+format:
+	poetry run ruff format
 
-.PHONY: flake8
-flake8:
-	flake8 .
+.PHONY: lint
+lint:
+	poetry run ruff check --fix
 
-.PHONY: isort
-isort:
-	isort .
+.PHONY: docdev
+docdev:
+	poetry run mkdocs serve --watch src
 
-.PHONY: lint
-lint: isort black flake8
+.PHONY: testonly
+testonly:
+	poetry run pytest
+
+.PHONY: testcov
+testcov:
+	poetry run coverage run -m pytest
 
+.PHONY: covreport
+covreport: testcov
+	coverage report
 
 .PHONY: test
-test:
-	pytest
+test: testcov
+	coverage html
+	poetry run python -m webbrowser ./htmlcov/index.html

NOTICE

@@ -0,0 +1,4 @@
+WPextract
+Copyright 2022-2024 The University of Sheffield
+
+Portions of this code are derived from WPJsonScraper, which is available under the MIT license. For details, see src/wpextract/download/LICENSE.