From 8c6636ef9a90b53ff8f429ec2e77c9055795a4b2 Mon Sep 17 00:00:00 2001
From: David McKee <david.mckee@dxw.com>
Date: Mon, 11 Dec 2023 18:28:47 +0000
Subject: [PATCH 1/5] WIP: initial stab at writing an event

---
 script/build_xquery_type_dicts              |  1 +
 smoketest/smoketest.py                      |  9 +++++++
 src/caselawclient/Client.py                 | 12 +++++++++
 src/caselawclient/models/history.py         | 18 +++++++++++++
 src/caselawclient/xquery/append_history.xqy | 30 +++++++++++++++++++++
 src/caselawclient/xquery_type_dicts.py      |  8 ++++++
 6 files changed, 78 insertions(+)
 create mode 100644 src/caselawclient/models/history.py
 create mode 100644 src/caselawclient/xquery/append_history.xqy

diff --git a/script/build_xquery_type_dicts b/script/build_xquery_type_dicts
index 19c57d04..3c253b83 100755
--- a/script/build_xquery_type_dicts
+++ b/script/build_xquery_type_dicts
@@ -28,6 +28,7 @@ DICTS_FILES_PATCH = join("src", "caselawclient", "xquery_type_dicts.py")
 ML_TYPES_TO_PYTHON_TYPES_DICT = {
     "xs:string": "str",
     "json:array": "list[Any]",
+    "json:object": "dict[Any, Any]",
     "xs:boolean": "bool",
     "xs:int": "int",
 }
diff --git a/smoketest/smoketest.py b/smoketest/smoketest.py
index c0466e31..712baf1e 100644
--- a/smoketest/smoketest.py
+++ b/smoketest/smoketest.py
@@ -5,6 +5,7 @@
 import caselawclient.Client as Client
 from caselawclient.errors import DocumentNotFoundError
 from caselawclient.models.documents import Document
+from caselawclient.models.history import HistoryEvent
 
 load_dotenv()
 env = environ.Env()
@@ -43,3 +44,11 @@ def test_get_version_annotation():
         api_client.get_version_annotation(FIRST_VERSION_URI) == "this is an annotation"
     )
     assert Document(FIRST_VERSION_URI, api_client).annotation == "this is an annotation"
+
+
+@pytest.mark.write
+def test_append_history():
+    api_client.append_history(
+        URI, HistoryEvent({"id": "1"}, ["flag"], "<kittens>1<cat/></kittens>")
+    )
+    # assert api_client.get_history() ...
diff --git a/src/caselawclient/Client.py b/src/caselawclient/Client.py
index a6d008ea..e48796ed 100644
--- a/src/caselawclient/Client.py
+++ b/src/caselawclient/Client.py
@@ -24,6 +24,7 @@
     Document,
     DocumentURIString,
 )
+from caselawclient.models.history import HistoryEvent
 from caselawclient.models.judgments import Judgment
 from caselawclient.models.press_summaries import PressSummary
 from caselawclient.models.utilities import move
@@ -951,3 +952,14 @@ def get_combined_stats_table(self) -> list[list[Any]]:
         )
 
         return results
+
+    def append_history(self, uri: DocumentURIString, history: HistoryEvent) -> None:
+        formatted_uri = self._format_uri_for_marklogic(uri)
+
+        vars: query_dicts.AppendHistoryDict = {
+            "uri": formatted_uri,
+            "attributes": history.attributes,
+            "flags": history.flags,
+            "payload": history.payload,
+        }
+        self._send_to_eval(vars, "append_history.xqy")
diff --git a/src/caselawclient/models/history.py b/src/caselawclient/models/history.py
new file mode 100644
index 00000000..18f99614
--- /dev/null
+++ b/src/caselawclient/models/history.py
@@ -0,0 +1,18 @@
+from typing import Union
+
+import lxml.etree
+
+
+class HistoryEvent:
+    def __init__(
+        self,
+        attributes: dict[str, str],
+        flags: list[str],
+        payload: Union[bytes, str, lxml.etree._Element],
+    ):
+        self.attributes = attributes
+        self.flags = flags
+        if isinstance(payload, lxml.etree._Element):
+            self.payload = lxml.etree.tostring(payload)
+        else:
+            self.payload = payload
diff --git a/src/caselawclient/xquery/append_history.xqy b/src/caselawclient/xquery/append_history.xqy
new file mode 100644
index 00000000..3c59b28f
--- /dev/null
+++ b/src/caselawclient/xquery/append_history.xqy
@@ -0,0 +1,30 @@
+xquery version "1.0-ml";
+
+import module namespace json="http://marklogic.com/xdmp/json" at "/MarkLogic/json/json.xqy";
+declare namespace basic="http://marklogic.com/xdmp/json/basic";
+
+(: let $attributes := json:transform-from-json(xdmp:unquote('{"id": "3", "type": "telemetry", "service": "ingester"}'))
+let $flags := json:transform-from-json(xdmp:unquote('["failed", "automated"]'))
+let $payload := <payload><mice levels='3'>maurice</mice>kittens</payload>:)
+declare variable $uri as xs:string external;
+declare variable $attributes as json:object external;
+declare variable $flags as json:array external;
+declare variable $payload as xs:string external;
+
+let $attributes-as-xml := json:transform-from-json($attributes)
+let $flags-as-xml := json:transform-from-json($flags)
+let $payload := xdmp:unquote($payload)
+
+let $event := <event>
+        {for $i in $attributes-as-xml/* return attribute {$i/name()} {$i/text()} }
+        {attribute {"datetime"} {fn:current-dateTime()}}
+        {for $i in $flags-as-xml//basic:item return attribute {$i/text()} {"true"}}
+        {$payload}
+      </event>
+
+let $history := xdmp:document-get-properties($uri, xs:QName("history"))
+
+return if (fn:exists($history)) then
+   xdmp:node-insert-child($history, $event)
+else
+   xdmp:document-set-property($uri, <history>{$event}</history>)
diff --git a/src/caselawclient/xquery_type_dicts.py b/src/caselawclient/xquery_type_dicts.py
index 10547480..a60b14a9 100644
--- a/src/caselawclient/xquery_type_dicts.py
+++ b/src/caselawclient/xquery_type_dicts.py
@@ -17,6 +17,14 @@ class MarkLogicAPIDict(TypedDict):
     pass
 
 
+# append_history.xqy
+class AppendHistoryDict(MarkLogicAPIDict):
+    attributes: dict[Any, Any]
+    flags: list[Any]
+    payload: str
+    uri: MarkLogicDocumentURIString
+
+
 # break_judgment_checkout.xqy
 class BreakJudgmentCheckoutDict(MarkLogicAPIDict):
     uri: MarkLogicDocumentURIString

From 68721f1c8f051c11832479c5780960e38230bc89 Mon Sep 17 00:00:00 2001
From: David McKee <david.mckee@dxw.com>
Date: Tue, 12 Dec 2023 12:39:05 +0000
Subject: [PATCH 2/5] A stab at a getter

---
 smoketest/smoketest.py                   |  3 ++-
 src/caselawclient/Client.py              | 13 +++++++++++++
 src/caselawclient/models/history.py      |  5 +++++
 src/caselawclient/xquery/get_history.xqy |  6 ++++++
 src/caselawclient/xquery_type_dicts.py   |  5 +++++
 5 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100644 src/caselawclient/xquery/get_history.xqy

diff --git a/smoketest/smoketest.py b/smoketest/smoketest.py
index 712baf1e..31a83c93 100644
--- a/smoketest/smoketest.py
+++ b/smoketest/smoketest.py
@@ -51,4 +51,5 @@ def test_append_history():
     api_client.append_history(
         URI, HistoryEvent({"id": "1"}, ["flag"], "<kittens>1<cat/></kittens>")
     )
-    # assert api_client.get_history() ...
+    event = api_client.get_history(URI)[-1]
+    assert event.attributes["id"] == "1"
diff --git a/src/caselawclient/Client.py b/src/caselawclient/Client.py
index e48796ed..a8be30f1 100644
--- a/src/caselawclient/Client.py
+++ b/src/caselawclient/Client.py
@@ -11,6 +11,7 @@
 from xml.etree.ElementTree import Element
 
 import environ
+import lxml.etree
 import requests
 from requests.auth import HTTPBasicAuth
 from requests.structures import CaseInsensitiveDict
@@ -743,6 +744,7 @@ def original_judgment_transformation(
         )
 
     def get_property(self, judgment_uri: DocumentURIString, name: str) -> str:
+        """This only gets the text of a property"""
         uri = self._format_uri_for_marklogic(judgment_uri)
         vars: query_dicts.GetPropertyDict = {
             "uri": uri,
@@ -963,3 +965,14 @@ def append_history(self, uri: DocumentURIString, history: HistoryEvent) -> None:
             "payload": history.payload,
         }
         self._send_to_eval(vars, "append_history.xqy")
+
+    def get_history(self, uri: DocumentURIString) -> list[HistoryEvent]:
+        formatted_uri = self._format_uri_for_marklogic(uri)
+        vars: query_dicts.GetHistoryDict = {"uri": formatted_uri}
+        response = self._send_to_eval(vars, "get_history.xqy")
+        bytes = get_single_bytestring_from_marklogic_response(response)
+        breakpoint()
+        events = []
+        for event in lxml.etree.fromstring(bytes).xpath("/history/event"):
+            events.append(HistoryEvent.from_xml(event))
+        return events
diff --git a/src/caselawclient/models/history.py b/src/caselawclient/models/history.py
index 18f99614..2a449f04 100644
--- a/src/caselawclient/models/history.py
+++ b/src/caselawclient/models/history.py
@@ -16,3 +16,8 @@ def __init__(
             self.payload = lxml.etree.tostring(payload)
         else:
             self.payload = payload
+
+    @classmethod
+    def from_xml(cls, element: lxml.etree._Element) -> "HistoryEvent":
+        flags = [a for a in element.attrib.keys() if element.attrib[a] == "true"]
+        return cls(element.attrib, flags, element)
diff --git a/src/caselawclient/xquery/get_history.xqy b/src/caselawclient/xquery/get_history.xqy
new file mode 100644
index 00000000..6e3bd054
--- /dev/null
+++ b/src/caselawclient/xquery/get_history.xqy
@@ -0,0 +1,6 @@
+xquery version "1.0-ml";
+
+import module namespace json="http://marklogic.com/xdmp/json" at "/MarkLogic/json/json.xqy";
+declare variable $uri as xs:string external;
+
+xdmp:document-get-properties($uri, xs:QName("history"))
diff --git a/src/caselawclient/xquery_type_dicts.py b/src/caselawclient/xquery_type_dicts.py
index a60b14a9..7a7a5bc5 100644
--- a/src/caselawclient/xquery_type_dicts.py
+++ b/src/caselawclient/xquery_type_dicts.py
@@ -63,6 +63,11 @@ class DocumentExistsDict(MarkLogicAPIDict):
     uri: MarkLogicDocumentURIString
 
 
+# get_history.xqy
+class GetHistoryDict(MarkLogicAPIDict):
+    uri: MarkLogicDocumentURIString
+
+
 # get_judgment.xqy
 class GetJudgmentDict(MarkLogicAPIDict):
     show_unpublished: Optional[bool]

From 42ff0cc8fe9f9eebf99dcea593436fcd433be9b7 Mon Sep 17 00:00:00 2001
From: David McKee <david.mckee@dxw.com>
Date: Wed, 13 Dec 2023 11:50:16 +0000
Subject: [PATCH 3/5] WIP flags

---
 src/caselawclient/xquery/append_history.xqy | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/caselawclient/xquery/append_history.xqy b/src/caselawclient/xquery/append_history.xqy
index 3c59b28f..57dc1f39 100644
--- a/src/caselawclient/xquery/append_history.xqy
+++ b/src/caselawclient/xquery/append_history.xqy
@@ -2,6 +2,7 @@ xquery version "1.0-ml";
 
 import module namespace json="http://marklogic.com/xdmp/json" at "/MarkLogic/json/json.xqy";
 declare namespace basic="http://marklogic.com/xdmp/json/basic";
+declare namespace flag="http://caselaw.nationalarchives.gov.uk/history/flags";
 
 (: let $attributes := json:transform-from-json(xdmp:unquote('{"id": "3", "type": "telemetry", "service": "ingester"}'))
 let $flags := json:transform-from-json(xdmp:unquote('["failed", "automated"]'))
@@ -18,7 +19,7 @@ let $payload := xdmp:unquote($payload)
 let $event := <event>
         {for $i in $attributes-as-xml/* return attribute {$i/name()} {$i/text()} }
         {attribute {"datetime"} {fn:current-dateTime()}}
-        {for $i in $flags-as-xml//basic:item return attribute {$i/text()} {"true"}}
+        {for $i in $flags-as-xml//basic:item return attribute {fn:QName("http://caselaw.nationalarchives.gov.uk/history/flags", $i/text())} {"true"}}
         {$payload}
       </event>
 
@@ -27,4 +28,4 @@ let $history := xdmp:document-get-properties($uri, xs:QName("history"))
 return if (fn:exists($history)) then
    xdmp:node-insert-child($history, $event)
 else
-   xdmp:document-set-property($uri, <history>{$event}</history>)
+   xdmp:document-set-property($uri, <history xmlns:flag="http://caselaw.nationalarchives.gov.uk/history/flags">{$event}</history>)

From f10e8235cfb9cd9f6e1a046f7d440cb4c1fadffb Mon Sep 17 00:00:00 2001
From: David McKee <david.mckee@dxw.com>
Date: Thu, 14 Dec 2023 12:14:02 +0000
Subject: [PATCH 4/5] wip more flags

---
 src/caselawclient/models/history.py         | 3 +++
 src/caselawclient/xquery/append_history.xqy | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/caselawclient/models/history.py b/src/caselawclient/models/history.py
index 2a449f04..be245dc8 100644
--- a/src/caselawclient/models/history.py
+++ b/src/caselawclient/models/history.py
@@ -2,6 +2,8 @@
 
 import lxml.etree
 
+namespaces = {"flag": "http://caselaw.nationalarchives.gov.uk/history/flags"}
+
 
 class HistoryEvent:
     def __init__(
@@ -20,4 +22,5 @@ def __init__(
     @classmethod
     def from_xml(cls, element: lxml.etree._Element) -> "HistoryEvent":
         flags = [a for a in element.attrib.keys() if element.attrib[a] == "true"]
+        flags = element.attrib.keys()
         return cls(element.attrib, flags, element)
diff --git a/src/caselawclient/xquery/append_history.xqy b/src/caselawclient/xquery/append_history.xqy
index 57dc1f39..4973c99d 100644
--- a/src/caselawclient/xquery/append_history.xqy
+++ b/src/caselawclient/xquery/append_history.xqy
@@ -16,7 +16,7 @@ let $attributes-as-xml := json:transform-from-json($attributes)
 let $flags-as-xml := json:transform-from-json($flags)
 let $payload := xdmp:unquote($payload)
 
-let $event := <event>
+let $event := <event xmlns:flag="http://caselaw.nationalarchives.gov.uk/history/flags">
         {for $i in $attributes-as-xml/* return attribute {$i/name()} {$i/text()} }
         {attribute {"datetime"} {fn:current-dateTime()}}
         {for $i in $flags-as-xml//basic:item return attribute {fn:QName("http://caselaw.nationalarchives.gov.uk/history/flags", $i/text())} {"true"}}
@@ -28,4 +28,4 @@ let $history := xdmp:document-get-properties($uri, xs:QName("history"))
 return if (fn:exists($history)) then
    xdmp:node-insert-child($history, $event)
 else
-   xdmp:document-set-property($uri, <history xmlns:flag="http://caselaw.nationalarchives.gov.uk/history/flags">{$event}</history>)
+   xdmp:document-set-property($uri, <history>{$event}</history>)

From 453fa5701393dad9899f3c68250b8776b563462a Mon Sep 17 00:00:00 2001
From: David McKee <david.mckee@dxw.com>
Date: Fri, 15 Dec 2023 17:43:57 +0000
Subject: [PATCH 5/5] Make payload naturally a string, payload proper

---
 smoketest/smoketest.py              | 12 +++++++-
 src/caselawclient/Client.py         |  7 +++--
 src/caselawclient/models/history.py | 43 ++++++++++++++++++++++++-----
 3 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/smoketest/smoketest.py b/smoketest/smoketest.py
index 31a83c93..8f1f77aa 100644
--- a/smoketest/smoketest.py
+++ b/smoketest/smoketest.py
@@ -49,7 +49,17 @@ def test_get_version_annotation():
 @pytest.mark.write
 def test_append_history():
     api_client.append_history(
-        URI, HistoryEvent({"id": "1"}, ["flag"], "<kittens>1<cat/></kittens>")
+        URI,
+        HistoryEvent(
+            {"id": "1"}, ["flag"], "<payload><kittens>1<cat/></kittens></payload>"
+        ),
     )
     event = api_client.get_history(URI)[-1]
     assert event.attributes["id"] == "1"
+    assert "datetime" in event.attributes
+    assert "flag" not in event.attributes
+    assert event.flags == ["flag"]
+    assert (
+        event.payload
+        == '<payload xmlns:flag="http://caselaw.nationalarchives.gov.uk/history/flags" xmlns:prop="http://marklogic.com/xdmp/property"><kittens>1<cat/></kittens></payload>'
+    )
diff --git a/src/caselawclient/Client.py b/src/caselawclient/Client.py
index a8be30f1..ad44021c 100644
--- a/src/caselawclient/Client.py
+++ b/src/caselawclient/Client.py
@@ -971,8 +971,9 @@ def get_history(self, uri: DocumentURIString) -> list[HistoryEvent]:
         vars: query_dicts.GetHistoryDict = {"uri": formatted_uri}
         response = self._send_to_eval(vars, "get_history.xqy")
         bytes = get_single_bytestring_from_marklogic_response(response)
+        events = [
+            HistoryEvent.from_xml(event)
+            for event in lxml.etree.fromstring(bytes).xpath("/history/event")
+        ]
         breakpoint()
-        events = []
-        for event in lxml.etree.fromstring(bytes).xpath("/history/event"):
-            events.append(HistoryEvent.from_xml(event))
         return events
diff --git a/src/caselawclient/models/history.py b/src/caselawclient/models/history.py
index be245dc8..ab574bba 100644
--- a/src/caselawclient/models/history.py
+++ b/src/caselawclient/models/history.py
@@ -2,25 +2,54 @@
 
 import lxml.etree
 
-namespaces = {"flag": "http://caselaw.nationalarchives.gov.uk/history/flags"}
+flag_ns_bare = "http://caselaw.nationalarchives.gov.uk/history/flags"
+flag_ns_wrapped = f"{{{flag_ns_bare}}}"
+namespaces = {"flag": flag_ns_bare}
+
+
+class EventPayloadInvalid(RuntimeError):
+    """The payload did not start with a <payload> tag."""
+
+    pass
 
 
 class HistoryEvent:
+    def __repr__(self) -> str:
+        return f"HistoryEvent({repr(self.attributes)}, {repr(self.flags)}, payload?)"
+
     def __init__(
         self,
         attributes: dict[str, str],
         flags: list[str],
-        payload: Union[bytes, str, lxml.etree._Element],
+        payload: Union[str, lxml.etree._Element],
     ):
         self.attributes = attributes
         self.flags = flags
         if isinstance(payload, lxml.etree._Element):
-            self.payload = lxml.etree.tostring(payload)
-        else:
+            self.payload = lxml.etree.tostring(payload).decode("utf-8")
+        elif isinstance(payload, bytes):
+            self.payload = payload.decode("utf-8")
+        elif isinstance(payload, str):
             self.payload = payload
 
+        print(self.payload)
+
+        if not self.payload.startswith("<payload"):
+            raise EventPayloadInvalid("Event payloads must start with a <payload> tag")
+
     @classmethod
     def from_xml(cls, element: lxml.etree._Element) -> "HistoryEvent":
-        flags = [a for a in element.attrib.keys() if element.attrib[a] == "true"]
-        flags = element.attrib.keys()
-        return cls(element.attrib, flags, element)
+        raw_attribs = element.attrib
+        attributes = {
+            k: v for k, v in raw_attribs.items() if not k.startswith(flag_ns_wrapped)
+        }
+        flags = [
+            k.partition(flag_ns_wrapped)[2]
+            for k in raw_attribs.keys()
+            if k.startswith(flag_ns_wrapped)
+        ]
+        try:
+            (payload,) = element.xpath("./payload")
+        except ValueError:  # there might not be exactly 1 payload
+            payload = "<payload/>"
+        return cls(attributes, flags, payload)