Omitting authentication tokens from the request text while logging to…

… the trace database files (#912)

* Omitting authentication tokens from the request text while logging to the trace databases when no_tokens_in_logs setting is set to true

* Fixing the code to address unit test failures realted to the changes

* Have to pass the attribute no_tokens_in_logs, without which the setting will not be honored

---------

Co-authored-by: Swamy Nallamalli <[email protected]>

restler/unit_tests/test_basic_functionality_end_to_end.py

@@ -1490,17 +1490,19 @@ def test_trace_database_ndjson_replay_blocks(self):
         The main purpose of this test is to confirm that an ndjson containing only replay blocks (without 
         the request text) can be used for replay.
         """
-        def run_test(omit_request_text=True):
+        def run_test(omit_request_text=True, remove_tokens_from_logs=True):
 
             # create a settings file with the trace database enabled
             new_settings_file_path = os.path.join(Test_File_Directory, f"tmp_trace_db_settings.json")
             if os.path.exists(new_settings_file_path):
                 os.remove(new_settings_file_path)
             settings = {}
             settings["use_trace_database"] = True
+            settings["no_tokens_in_logs"] = remove_tokens_from_logs
             settings["include_unique_sequence_id"] = False
             settings["trace_database"] = {}
             settings["trace_database"]["omit_request_text"] = omit_request_text
+            settings["trace_database"]["remove_tokens_from_logs"] = remove_tokens_from_logs
 
             with open(new_settings_file_path, "w") as outfile:
                 outfile.write(json.dumps(settings, indent=4))
@@ -1525,7 +1527,7 @@ def run_test(omit_request_text=True):
             baseline_trace_db_path = replay_file_path
             trace_db_path = os.path.join(self.get_experiments_dir(), "trace_data.ndjson")
 
-            print(f"Comparing trace DB to baseline: {baseline_trace_db_path}")
+            print(f"Comparing trace DB {trace_db_path} to baseline: {baseline_trace_db_path}")
 
             baseline_deserializer = trace_db.JsonTraceLogReader(log_file_paths=[baseline_trace_db_path])
             actual_deserializer = trace_db.JsonTraceLogReader(log_file_paths=[trace_db_path])
@@ -1559,5 +1561,7 @@ def run_test(omit_request_text=True):
                     message = f"different baseline log \n{json.dumps(x.to_dict(), indent=4)} \nto actual log \n{json.dumps(y.to_dict(), indent=4)}"
                     self.fail(f"Trace DBs do not match: {message}")
 
-        run_test(omit_request_text=True)
-        run_test(omit_request_text=False)
+        run_test(omit_request_text=True, remove_tokens_from_logs=True)
+        run_test(omit_request_text=False, remove_tokens_from_logs=True)
+        run_test(omit_request_text=True, remove_tokens_from_logs=False)
+        run_test(omit_request_text=False, remove_tokens_from_logs=False)

restler/utils/logging/ndjson_serializer.py

@@ -13,7 +13,7 @@
 
 from restler_settings import Settings
 from utils.logging.serializer_base import *
-
+import utils.logger as logger
 
 class CustomRotatingFileHandler(RotatingFileHandler):
     def rotation_filename(self, default_name):
@@ -106,7 +106,7 @@ def response(self):
     def response(self, value):
         self._response = value
 
-    def to_dict(self, omit_request_text=None):
+    def to_dict(self, omit_request_text=None, remove_tokens_from_logs=True):
         tags = {}
         if self.request_id is not None:
             tags["request_id"] = self.request_id
@@ -120,6 +120,7 @@ def to_dict(self, omit_request_text=None):
         tags.update(self.tags)
         tags.update(self.sequence_tags)
         request_text = None if omit_request_text == True else self.request
+        request_text = logger.remove_tokens_from_logs(request_text) if remove_tokens_from_logs else request_text
 
         return {
             'sent_timestamp': self.sent_timestamp,

restler/utils/logging/trace_db.py

@@ -198,7 +198,7 @@ def log_request_response(self, request=None, response=None,  tags={}, timestamp=
 
             if 'origin' not in trace_log.tags and 'origin' not in trace_log.sequence_tags and trace_log.origin is None:
                 raise Exception(f"Missing origin: request: {trace_log.request_id}, sequence: {trace_log.sequence_id}")
-            record = trace_log.to_dict(omit_request_text=Settings().trace_db_omit_request_text)
+            record = trace_log.to_dict(omit_request_text=Settings().trace_db_omit_request_text, remove_tokens_from_logs=Settings().no_tokens_in_logs)
             self.log(record)
         except Exception as error:
             # print the callstack