Merge pull request #27 from joawan/logging

Added logging library to allow ignore of debug log

cloudformation.json

@@ -63,6 +63,11 @@
       "Type" : "CommaDelimitedList",
       "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lamda functions will be deployed into. Only required if VPC is enabled."
     },
+    "HumioLambdaLogLevel" : {
+      "Type" : "String",
+      "AllowedValues" : ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+      "Default" : "INFO"
+    },
     "Version" : {
       "Type" : "String",
       "Description" : "The version of the integration you want installed.",
@@ -182,7 +187,8 @@
           "Variables" : {
             "humio_protocol" : { "Ref" : "HumioProtocol" },
             "humio_host" : { "Ref" : "HumioHost" },
-            "humio_ingest_token" : { "Ref" : "HumioIngestToken" }
+            "humio_ingest_token" : { "Ref" : "HumioIngestToken" },
+            "log_level" : { "Ref" : "HumioLambdaLogLevel" }
           }
         },
         "VpcConfig" : {
@@ -243,7 +249,8 @@
             "humio_log_ingester_arn" : {
               "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ]
             },
-            "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }
+            "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" },
+            "log_level" : { "Ref" : "HumioLambdaLogLevel" }
           }
         },
         "Description" : "CloudWatch Logs to Humio log group subscriber.",
@@ -312,7 +319,8 @@
             "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" },
             "humio_protocol" : { "Ref" : "HumioProtocol" },
             "humio_host" : { "Ref" : "HumioHost" },
-            "humio_ingest_token" : { "Ref" : "HumioIngestToken" }
+            "humio_ingest_token" : { "Ref" : "HumioIngestToken" },
+            "log_level" : { "Ref" : "HumioLambdaLogLevel" }
           }
         },
         "Description" : "CloudWatch Logs to Humio logs backfiller.",
@@ -464,7 +472,8 @@
           "Variables" : {
             "humio_protocol" : { "Ref" : "HumioProtocol" },
             "humio_host" : { "Ref" : "HumioHost" },
-            "humio_ingest_token" : { "Ref" : "HumioIngestToken" }
+            "humio_ingest_token" : { "Ref" : "HumioIngestToken" },
+            "log_level" : { "Ref" : "HumioLambdaLogLevel" }
           }
         },
         "VpcConfig" : {
@@ -524,7 +533,8 @@
           "Variables" : {
             "humio_protocol" : { "Ref" : "HumioProtocol" },
             "humio_host" : { "Ref" : "HumioHost" },
-            "humio_ingest_token" : { "Ref" : "HumioIngestToken" }
+            "humio_ingest_token" : { "Ref" : "HumioIngestToken" },
+            "log_level" : { "Ref" : "HumioLambdaLogLevel" }
           }
         },
         "VpcConfig" : {

src/helpers.py

@@ -4,7 +4,12 @@
 import os
 import json
 import requests
+import logging
 
+level = os.getenv("log_level", "INFO")
+logging.basicConfig(level=level)
+logger = logging.getLogger()
+logger.setLevel(level)
 
 def setup():
     """
@@ -45,7 +50,7 @@ def ingest_events(humio_events, host_type):
     # Prepare events to be sent to Humio.
     wrapped_data = [{"tags": {"host": host_type}, "events": humio_events}]
 
-    print("Data being sent to Humio: %s" % wrapped_data)
+    logger.debug("Data being sent to Humio: %s" % wrapped_data)
 
     # Make request. 
     request = http_session.post(
@@ -92,12 +97,12 @@ def create_subscription(log_client, log_group_name, humio_log_ingester_arn, cont
     """   
     # We cannot subscribe to the log group that our stdout/err goes to.
     if context.log_group_name == log_group_name:
-        print("Skipping our own log group name...")
+        logger.debug("Skipping our own log group name...")
     # And we do not want to subscribe to other Humio log ingesters - if there are any. 
     if "HumioCloudWatchLogsIngester" in log_group_name:
-        print("Skipping cloudwatch2humio ingesters...") 
+        logger.debug("Skipping cloudwatch2humio ingesters...")
     else:
-        print("Creating subscription for %s" % log_group_name)
+        logger.info("Creating subscription for %s" % log_group_name)
         try:
             log_client.put_subscription_filter(
                 logGroupName=log_group_name,
@@ -106,9 +111,9 @@ def create_subscription(log_client, log_group_name, humio_log_ingester_arn, cont
                 destinationArn=humio_log_ingester_arn,
                 distribution="ByLogStream"
             )
-            print("Successfully subscribed to %s!" % log_group_name)
+            logger.debug("Successfully subscribed to %s!" % log_group_name)
         except Exception as exception:
-            print("Error creating subscription to %s. Exception: %s" % (log_group_name, exception))
+            logger.error("Error creating subscription to %s. Exception: %s" % (log_group_name, exception))
 
 
 def delete_subscription(log_client, log_group_name, filter_name):
@@ -126,7 +131,7 @@ def delete_subscription(log_client, log_group_name, filter_name):
 
     :return: None
     """    
-    print("Deleting subscription for %s" % log_group_name)
+    logger.info("Deleting subscription for %s" % log_group_name)
     log_client.delete_subscription_filter(
         logGroupName=log_group_name,
         filterName=filter_name

src/logs_backfiller.py

@@ -3,7 +3,12 @@
 import os
 import helpers
 import requests
+import logging
 
+level = os.getenv("log_level", "INFO")
+logging.basicConfig(level=level)
+logger = logging.getLogger()
+logger.setLevel(level)
 
 def lambda_handler(event, context):
     """
@@ -82,7 +87,7 @@ def lambda_handler(event, context):
                 )
             # We are now subscribed.
             else:
-                print("We are already subscribed to %s" % log_group["logGroupName"])
+                logger.debug("We are already subscribed to %s" % log_group["logGroupName"])
         # When there are no subscription filters, let us subscribe!
         else:
             helpers.create_subscription(
@@ -107,4 +112,4 @@ def send_custom_resource_response(event, context):
                 data=json.dumps(response_content)
             )
             # Used for debugging. 
-            print("Response status from Custom Resource: %s " % response.status_code)
+            logger.debug("Response status from Custom Resource: %s " % response.status_code)

src/logs_ingester.py

@@ -1,11 +1,17 @@
 import re
 import json
 import helpers
+import os
+import logging
+
+level = os.getenv("log_level", "INFO")
+logging.basicConfig(level=level)
+logger = logging.getLogger()
+logger.setLevel(level)
 
 # False when setup has not been performed.
 _is_setup = False
 
-
 def lambda_handler(event, context):
     """
     Extract log data from CloudWatch Logs events and
@@ -27,7 +33,7 @@ def lambda_handler(event, context):
     decoded_event = helpers.decode_event(event)
 
     # Debug output.
-    print("Event from CloudWatch Logs: %s" % (json.dumps(decoded_event)))
+    logger.debug("Event from CloudWatch Logs: %s" % (json.dumps(decoded_event)))
 
     # Extract the general attributes from the event batch.
     batch_attrs = {
@@ -73,4 +79,4 @@ def lambda_handler(event, context):
     response = request.text
 
     # Debug output.
-    print("Got response %s from Humio." % response)
+    logger.debug("Got response %s from Humio." % response)

src/metric_ingester.py

@@ -2,6 +2,13 @@
 import json
 import helpers
 from datetime import datetime, timedelta, timezone
+import os
+import logging
+
+level = os.getenv("log_level", "INFO")
+logging.basicConfig(level=level)
+logger = logging.getLogger()
+logger.setLevel(level)
 
 _is_setup = False
 
@@ -70,7 +77,7 @@ def lambda_handler(event, context):
 
     # Debug the response.
     response = request.text
-    print("Got response %s from Humio." % response)
+    logger.debug("Got response %s from Humio." % response)
 
 
 def get_metric_data(configurations):

src/metric_statistics_ingester.py

@@ -2,6 +2,13 @@
 import json
 import helpers
 from datetime import datetime, timedelta, timezone
+import os
+import logging
+
+level = os.getenv("log_level", "INFO")
+logging.basicConfig(level=level)
+logger = logging.getLogger()
+logger.setLevel(level)
 
 _is_setup = False
 
@@ -29,7 +36,7 @@ def lambda_handler(event, context):
     metric_statistics, api_parameters = get_metric_statistics(configurations)
 
     # Used for debugging.
-    print("Statistics from CloudWatch Metrics: %s" % metric_statistics)
+    logger.debug("Statistics from CloudWatch Metrics: %s" % metric_statistics)
 
     # Format metric data to Humio event data.
     humio_events = create_humio_events(metric_statistics, api_parameters)
@@ -39,7 +46,7 @@ def lambda_handler(event, context):
 
     # Debug the response.
     response = request.text
-    print("Got response %s from Humio." % response)
+    logger.debug("Got response %s from Humio." % response)
 
 
 def get_metric_statistics(configurations):
@@ -69,7 +76,7 @@ def get_metric_statistics(configurations):
             api_parameters["EndTime"] = api_parameters["EndTime"].replace(tzinfo=timezone.utc).isoformat()
 
         # Used for debugging.
-        print("Start time: %s, End time: %s" % (api_parameters["StartTime"], api_parameters["EndTime"]))
+        logger.debug("Start time: %s, End time: %s" % (api_parameters["StartTime"], api_parameters["EndTime"]))
 
         # Make GetMetricStatistics API request.
         metric_statistics = metric_client.get_metric_statistics(
@@ -95,7 +102,7 @@ def create_humio_events(metrics, api_parameters):
     humio_events = []
 
     # Used for debuggin.
-    print("Datapoints: %s" % metrics["Datapoints"])
+    logger.debug("Datapoints: %s" % metrics["Datapoints"])
 
     # Create one Humio event per datapoint/timestamp.
     for datapoint in metrics["Datapoints"]: