Merge pull request #78 from nationalarchives/user-agent

⚠️ Set user agent and build own API client object

.gitignore

@@ -8,3 +8,6 @@ dist
 .DS_Store
 .aws-sam
 */__pycache__
+
+tmp
+volume

ds-caselaw-ingester/lambda_function.py

@@ -13,14 +13,27 @@
 from boto3.session import Session
 from botocore.exceptions import NoCredentialsError
 from caselawclient.Client import (
+    DEFAULT_USER_AGENT,
+    MarklogicApiClient,
     MarklogicCommunicationError,
     MarklogicResourceNotFoundError,
-    api_client,
 )
+from dotenv import load_dotenv
 from notifications_python_client.notifications import NotificationsAPIClient
 
+load_dotenv()
+
+
 rollbar.init(os.getenv("ROLLBAR_TOKEN"), environment=os.getenv("ROLLBAR_ENV"))
 
+api_client = MarklogicApiClient(
+    host=os.getenv("MARKLOGIC_HOST", default=None),
+    username=os.getenv("MARKLOGIC_USER", default=None),
+    password=os.getenv("MARKLOGIC_PASSWORD", default=None),
+    use_https=os.getenv("MARKLOGIC_USE_HTTPS", default=False),
+    user_agent=f"ds-caselaw-ingester/unknown {DEFAULT_USER_AGENT}",
+)
+
 
 class Message(object):
     @classmethod

ds-caselaw-ingester/tests.py

@@ -13,7 +13,6 @@
 from caselawclient.Client import (
     MarklogicCommunicationError,
     MarklogicResourceNotFoundError,
-    api_client,
 )
 from notifications_python_client.notifications import NotificationsAPIClient
 
@@ -239,7 +238,8 @@ def test_extract_docx_filename_failure(self):
         with pytest.raises(lambda_function.DocxFilenameNotFoundException):
             lambda_function.extract_docx_filename(metadata, "anything")
 
-    def test_store_metadata(self):
+    @patch("lambda_function.api_client")
+    def test_store_metadata(self, api_client):
         metadata = {
             "parameters": {
                 "TDR": {
@@ -269,23 +269,23 @@ def test_store_file_success(self, mock_print):
         session.upload_fileobj = MagicMock()
         lambda_function.store_file(None, "folder", "filename.ext", session)
         mock_print.assert_called_with("Upload Successful folder/filename.ext")
-        session.upload_fileobj.assert_called_with(None, None, "folder/filename.ext")
+        session.upload_fileobj.assert_called_with(None, ANY, "folder/filename.ext")
 
     @patch("builtins.print")
     def test_store_file_file_not_found(self, mock_print):
         session = boto3.Session
         session.upload_fileobj = MagicMock(side_effect=FileNotFoundError)
         lambda_function.store_file(None, "folder", "filename.ext", session)
         mock_print.assert_called_with("The file folder/filename.ext was not found")
-        session.upload_fileobj.assert_called_with(None, None, "folder/filename.ext")
+        session.upload_fileobj.assert_called_with(None, ANY, "folder/filename.ext")
 
     @patch("builtins.print")
     def test_store_file_file_no_credentials(self, mock_print):
         session = boto3.Session
         session.upload_fileobj = MagicMock(side_effect=NoCredentialsError)
         lambda_function.store_file(None, "folder", "filename.ext", session)
         mock_print.assert_called_with("Credentials not available")
-        session.upload_fileobj.assert_called_with(None, None, "folder/filename.ext")
+        session.upload_fileobj.assert_called_with(None, ANY, "folder/filename.ext")
 
     @patch.dict(
         os.environ,
@@ -580,14 +580,16 @@ def test_malformed_message(self):
         with pytest.raises(lambda_function.InvalidMessageException):
             lambda_function.get_consignment_reference(message)
 
-    def test_update_judgment_xml_success(self):
+    @patch("lambda_function.api_client")
+    def test_update_judgment_xml_success(self, api_client):
         xml = ET.XML("<xml>Here's some xml</xml>")
         api_client.get_judgment_xml = MagicMock(return_value=True)
         api_client.save_judgment_xml = MagicMock(return_value=True)
         result = lambda_function.update_judgment_xml("a/fake/uri", xml)
         assert result is True
 
-    def test_update_judgment_xml_judgment_does_not_exist(self):
+    @patch("lambda_function.api_client")
+    def test_update_judgment_xml_judgment_does_not_exist(self, api_client):
         xml = ET.XML("<xml>Here's some xml</xml>")
         api_client.get_judgment_xml = MagicMock(
             side_effect=MarklogicResourceNotFoundError("error")
@@ -596,7 +598,8 @@ def test_update_judgment_xml_judgment_does_not_exist(self):
         result = lambda_function.update_judgment_xml("a/fake/uri", xml)
         assert result is False
 
-    def test_update_judgment_xml_judgment_does_not_save(self):
+    @patch("lambda_function.api_client")
+    def test_update_judgment_xml_judgment_does_not_save(self, api_client):
         xml = ET.XML("<xml>Here's some xml</xml>")
         api_client.get_judgment_xml = MagicMock(return_value=True)
         api_client.save_judgment_xml = MagicMock(
@@ -605,13 +608,15 @@ def test_update_judgment_xml_judgment_does_not_save(self):
         result = lambda_function.update_judgment_xml("a/fake/uri", xml)
         assert result is False
 
-    def test_insert_document_xml_success(self):
+    @patch("lambda_function.api_client")
+    def test_insert_document_xml_success(self, api_client):
         xml = ET.XML("<xml>Here's some xml</xml>")
         api_client.insert_document_xml = MagicMock(return_value=True)
         result = lambda_function.insert_document_xml("a/fake/uri", xml)
         assert result is True
 
-    def test_insert_document_xml_failure(self):
+    @patch("lambda_function.api_client")
+    def test_insert_document_xml_failure(self, api_client):
         xml = ET.XML("<xml>Here's some xml</xml>")
         api_client.insert_document_xml = MagicMock(
             side_effect=MarklogicCommunicationError("error")
@@ -695,8 +700,12 @@ def test_get_best_xml_with_no_xml_file(self):
             assert result.__class__ == ET.Element
             assert result.tag == "error"
 
-    def test_unpublish_updated_judgment(self):
+    @patch("lambda_function.api_client")
+    def test_unpublish_updated_judgment(self, api_client):
         uri = "a/fake/uri"
         api_client.set_published = MagicMock()
         lambda_function.unpublish_updated_judgment(uri)
         api_client.set_published.assert_called_with(uri, False)
+
+    def test_user_agent(self):
+        assert "ingester" in lambda_function.api_client.session.headers["User-Agent"]

requirements/base.txt

@@ -1,5 +1,5 @@
 django-environ~=0.10
-ds-caselaw-marklogic-api-client==14.0.2
+ds-caselaw-marklogic-api-client==14.1.0
 requests-toolbelt~=1.0
 urllib3~=1.26
 boto3
@@ -8,3 +8,4 @@ notifications-python-client~=8.0
 
 mypy-boto3-s3
 mypy-boto3-sns
+python-dotenv

scripts/test

@@ -1 +1 @@
-MARKLOGIC_HOST= MARKLOGIC_USER= MARKLOGIC_PASSWORD= python -m pytest ds-caselaw-ingester/tests.py $*
+python -m pytest ds-caselaw-ingester/tests.py $*