move http response to own column

digitalmethodsinitiative · Sep 12, 2023 · 2b9b74a · 2b9b74a
1 parent 6c64ad1
commit 2b9b74a
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/datasources/web_archive_scraper/search_web_archive.py b/datasources/web_archive_scraper/search_web_archive.py
@@ -139,6 +139,7 @@ def get_items(self, query):
                 "subject": None,
                 "body": None,
                 "html": None,
+                "http_html": None,
                 "detected_404": None,
                 "timestamp": None,
                 "error": '',
@@ -256,9 +257,9 @@ def get_items(self, query):
                     try:
                         http_response = self.request_get_w_error_handling(scraped_page.get('final_url'), timeout=120)
                         self.dataset.log('Collected HTTP response: %s' % scraped_page.get('final_url'))
-                        result['html'] = 'SELENIUM RESPONSE:\n' + str(result['html']) + '\nHTTP RESPONSE:\n' + http_response.text
+                        result['http_html'] = http_response.text
                     except Exception as e:
-                        result['html'] = 'SELENIUM RESPONSE:\n' + str(result['html']) + '\nHTTP RESPONSE:\nNone'
+                        result['http_html'] = None
                         http_error = '\nHTTP ERROR:\n' + str(e)
                         result['error'] = 'SELENIUM ERROR:\n' + str(result['error']) + http_error