Add pre-commit config

.pre-commit-config.yaml

@@ -0,0 +1,52 @@
+exclude: "^docs/|/migrations/"
+default_install_hook_types: [pre-commit, pre-push]
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+
+  - repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/Riverside-Healthcare/djLint
+    rev: v1.34.0
+    hooks:
+      - id: djlint-django
+        name: djLint Reformatting for Django
+        entry: djlint --profile=django --reformat
+        types_or: [html]
+        language: python
+
+  - repo: https://github.com/Riverside-Healthcare/djLint
+    rev: v1.34.0
+    hooks:
+      - id: djlint-django
+        name: djLint Checks for Django
+        entry: djlint --profile=django
+        types_or: [html]
+        language: python
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.1.0
+    hooks:
+      - id: prettier
+        types_or: [scss, yaml, markdown, javascript, xml]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.1.6
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+
+# sets up .pre-commit-ci.yaml to ensure pre-commit dependencies stay up to date
+ci:
+  autoupdate_schedule: weekly
+  skip: []
+  submodules: false

README.md

@@ -15,7 +15,7 @@ The file layout is explained in the [project layout documentation](https://githu
 1. Install `gradle`. On MacOS, you can use `brew install gradle`.
 
 2. If you're running against anything other than development, copy `gradle-development.properties`
-to `gradle-{environment}.properties` and set the credentials and hostname for your Marklogic server.
+   to `gradle-{environment}.properties` and set the credentials and hostname for your Marklogic server.
 
 ## Deployment
 
@@ -93,10 +93,9 @@ Two gradle tasks are available for bulk management of documents in a database us
 necessary to use, but are provided in order to automate some development tasks and provide
 examples for future data migrations.
 
-
-* `gradle manageAllDocuments`: Enables version management for all documents
-* `gradle publishAllDocuments`: Sets the `published` flag for all documents
-* `gradle addAllDocumentsToJudgmentsCollection`: Adds all documents to the 'judgments' collection.
+- `gradle manageAllDocuments`: Enables version management for all documents
+- `gradle publishAllDocuments`: Sets the `published` flag for all documents
+- `gradle addAllDocumentsToJudgmentsCollection`: Adds all documents to the 'judgments' collection.
 
 ### Loading data from a backup on S3 (deprecated)
 

development_scripts/populate_from_caselaw.py

@@ -32,7 +32,8 @@
     print("got xml")
 
     response = requests.put(
-        f"http://admin:admin@localhost:8011/LATEST/documents?uri={ml_url}", data=xml
+        f"http://admin:admin@localhost:8011/LATEST/documents?uri={ml_url}",
+        data=xml,
     )
     response.raise_for_status()
     print("added to local Marklogic db")

development_scripts/populate_top_judgments_and_neighbours.py

@@ -11,11 +11,12 @@
 - Run the script.
 """
 
-import requests
+import re
 from urllib.parse import quote
 from xml.etree import ElementTree as ET
+
+import requests
 from bs4 import BeautifulSoup
-import re
 
 urls = [
     "ewca/civ/2022/1146",
@@ -27,35 +28,49 @@
     "ewca/civ/2022/1047",
     "ewfc/2023/46",
     "ewfc/2022/95",
-    "ewhc/admin/2006/815"
+    "ewhc/admin/2006/815",
 ]
 
+
 def get_judgment_xml(url):
     print("Getting judgment: %s" % url)
     response = requests.get(f"https://caselaw.nationalarchives.gov.uk/{url}/data.xml")
     response.raise_for_status()
     return response.content
 
+
 def save_judgment_xml(url, xml):
-    print("Saving judgment: %s" % url);
+    print("Saving judgment: %s" % url)
     ml_url = f"/{url}.xml"
     response = requests.put(
-        f"http://admin:admin@localhost:8011/LATEST/documents?uri={ml_url}", data=xml
+        f"http://admin:admin@localhost:8011/LATEST/documents?uri={ml_url}",
+        data=xml,
     )
     response.raise_for_status()
 
+
 def get_neighbours_for_judgment(xml):
     tree = ET.fromstring(xml)
-    ns = {'akn': 'http://docs.oasis-open.org/legaldocml/ns/akn/3.0'}
-    title = tree.find("./akn:judgment/akn:meta/akn:identification/akn:FRBRWork/akn:FRBRname", ns).attrib["value"]
+    ns = {"akn": "http://docs.oasis-open.org/legaldocml/ns/akn/3.0"}
+    title = tree.find(
+        "./akn:judgment/akn:meta/akn:identification/akn:FRBRWork/akn:FRBRname",
+        ns,
+    ).attrib["value"]
     print("Getting neighbours for judgment title: %s" % title)
-    search_url = "https://caselaw.nationalarchives.gov.uk/judgments/results?query=" + quote(title)
+    search_url = (
+        "https://caselaw.nationalarchives.gov.uk/judgments/results?query="
+        + quote(title)
+    )
     search_results = requests.get(search_url)
-    search_soup = BeautifulSoup(search_results.content, 'html.parser')
-    neighbours = list(re.sub("^\/", "", a["href"]) for a in search_soup.select(".judgment-listing__title a"))
+    search_soup = BeautifulSoup(search_results.content, "html.parser")
+    neighbours = list(
+        re.sub(r"^\/", "", a["href"])
+        for a in search_soup.select(".judgment-listing__title a")
+    )
     print("... found %s" % len(neighbours))
     return neighbours
 
+
 found = set()
 for url in urls:
     xml = get_judgment_xml(url)
@@ -70,4 +85,3 @@ def get_neighbours_for_judgment(xml):
             print("Skipping already imported judgment %s" % url2)
     print(f"**** {url} and close title matches added to local Marklogic db ****")
 print("DONE. Imported %s judgments." % len(found))
-

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.1'
+version: "3.1"
 
 services:
   marklogic:

pyproject.toml

@@ -0,0 +1,54 @@
+[tool.poetry]
+name = "ds-caselaw-marklogic"
+version = "0.1.0"
+description = ""
+authors = ["David McKee <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+saxonche = "^12.3.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+
+[tool.ruff]
+ignore = ["E501", "G004", "PLR2004", "RUF005", "RUF012", "UP040"] # long lines, fstrings in logs, magic values, consider not concat, mutable classbits, type instead of TypeAlias
+extend-select = ["W", "B", "Q", "C90", "I", "UP", "YTT", "ASYNC", "S", "BLE", "A", "COM", "C4", "DTZ", "T10", "DJ", "EM", "EXE", "FA",
+                 "ISC", "ICN", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET", "SLF", "SLOT", "SIM", "TID", "TCH", "INT", "PTH",
+                 "FIX", "PGH", "PL", "TRY", "FLY", "PERF", "RUF"]
+unfixable = ["ERA"]
+
+# things skipped:
+# N: naming, possibly good
+# D: docstrings missing throughout
+# ANN: annotations missing throughout
+# FBT: not convinced boolean trap worth auto-banning.
+# CPY: copyright at top of each file
+# G: logging warnings -- fstrings bad?
+# ARG: sometimes you need to accept arguments.
+# TD: somewhat finicky details about formatting TODOs
+# FIX: flags todos: possible to add -- skipped for now
+# ERA: lots of false positives, not a good autofix
+# PD, NPY, AIR: ignored, panda / numpy / airflow specific
+# FURB: not yet out of preview
+
+
+
+[tool.ruff.extend-per-file-ignores]
+"*" = ["RET505", # disagree with if X: return Y else: return Z being wrong
+       "T201",   # print
+       "S113",   # requests no timeout : TODO
+      ]
+"tests/*" = ["S101"]  # assert fine in tests
+"development_scripts/populate_top_judgments_and_neighbours.py" = ["S314", "C400"] # TODO
+
+
+
+[tool.ruff.isort]
+known-first-party = ["ds-caselaw-editor-ui", "config"]
+
+[tool.ruff.pycodestyle]

src/main/ml-config/security/roles/1-caselaw-nobody-role.json

@@ -2,4 +2,4 @@
   "role-name" : "caselaw-nobody",
   "description" : "Unauthenticated user",
   "role" : [ ]
-}
+}

src/main/ml-config/security/roles/2-caselaw-reader-role.json

@@ -11,4 +11,4 @@
     "action" : "http://marklogic.com/xdmp/privileges/xdbc-invoke",
     "kind" : "execute"
   } ]
-}
+}

src/main/ml-config/security/roles/3-caselaw-writer-role.json

@@ -11,4 +11,4 @@
     "action" : "http://marklogic.com/xdmp/privileges/unprotected-collections",
     "kind" : "execute"
   } ]
-}
+}

src/main/ml-config/security/roles/5-caselaw-admin-role.json

@@ -15,4 +15,4 @@
     "action" : "http://marklogic.com/xdmp/privileges/xdmp-eval-in",
     "kind" : "execute"
   } ]
-}
+}

src/main/ml-config/security/roles/6-caselaw-unpublished-reader-role.json

@@ -9,4 +9,4 @@
       "action": "https://caselaw.nationalarchives.gov.uk/custom/privileges/can-view-unpublished-documents"
     }
   ]
-}
+}

src/main/ml-config/security/roles/7-caselaw-priv-api-writer-role.json

@@ -2,4 +2,4 @@
   "role-name" : "priv-api-writer",
   "description" : "Can view documents, including unpublished documents, and edit",
   "role" : [ "caselaw-writer", "caselaw-unpublished-reader" ]
-}
+}