Include log_level as key in template (#762)

Add possibility for the user to override the the log_level either as a CLI option to gordo-components or as a key under globals runtime in the config file.

gordo_components/cli/cli.py

@@ -4,7 +4,6 @@
 CLI interfaces
 """
 
-import os
 import logging
 import sys
 import traceback
@@ -46,19 +45,29 @@
 
 @click.group("gordo-components")
 @click.version_option(version=__version__, message=__version__)
-def gordo():
+@click.option(
+    "--log-level",
+    type=str,
+    default="INFO",
+    help="Run workflow with custom log-level.",
+    envvar="GORDO_LOG_LEVEL",
+)
+@click.pass_context
+def gordo(gordo_ctx: click.Context, **ctx):
     """
     The main entry point for the CLI interface
     """
-
     # Set log level, defaulting to INFO
-    log_level = os.getenv("LOG_LEVEL", "INFO").upper()
-
     logging.basicConfig(
-        level=getattr(logging, log_level),
+        level=getattr(logging, str(gordo_ctx.params.get("log_level")).upper()),
         format="[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s",
     )
 
+    # Set matplotlib log level to INFO to avoid noise
+    logging.getLogger("matplotlib").setLevel(logging.INFO)
+
+    gordo_ctx.obj = gordo_ctx.params
+
 
 @click.command()
 @click.argument("machine-config", envvar="MACHINE", type=yaml.safe_load)

gordo_components/cli/workflow_generator.py

@@ -3,6 +3,7 @@
 import pkg_resources
 import json
 import sys
+import os
 
 from typing import Dict, Any
 
@@ -21,7 +22,8 @@
 
 
 @click.group("workflow")
-def workflow_cli():
+@click.pass_context
+def workflow_cli(gordo_ctx):
     pass
 
 
@@ -111,15 +113,23 @@ def workflow_cli():
     help="The docker registry to use for pulling component images from",
     envvar=f"{PREFIX}_DOCKER_REGISTRY",
 )
-def workflow_generator_cli(**ctx: dict):
+@click.pass_context
+def workflow_generator_cli(gordo_ctx, **ctx):
     """
     Machine Configuration to Argo Workflow
     """
 
     context: Dict[Any, Any] = ctx.copy()
-
     yaml_content = wg.get_dict_from_yaml(context["machine_config"])
 
+    try:
+        log_level = yaml_content["globals"]["runtime"]["log_level"]
+    except KeyError:
+        log_level = os.getenv("GORDO_LOG_LEVEL", gordo_ctx.obj["log_level"])
+
+    logging.getLogger("gordo_components").setLevel(log_level.upper())
+    context["log_level"] = log_level.upper()
+
     # Create normalized config
     config = NormalizedConfig(yaml_content, project_name=context["project_name"])
 

gordo_components/workflow/workflow_generator/resources/argo-workflow.yml.template

@@ -79,6 +79,8 @@ spec:
         value: "{{project_name}}"
       - name: GORDO_PROJECT_REVISION
         value: "{{project_revision}}"
+      - name: GORDO_LOG_LEVEL
+        value: "{{log_level}}"
 
   - name: apply-with-retries
     retryStrategy:
@@ -654,6 +656,8 @@ spec:
           secretKeyRef:
             name: dlserviceauth
             key: tenant_id_secret
+      - name: GORDO_LOG_LEVEL
+        value: "{{log_level}}"
       volumeMounts:
       - name: azurefile
         mountPath: /gordo
@@ -876,6 +880,8 @@ spec:
                              env:
                                - name: MODEL_COLLECTION_DIR
                                  value: /gordo/models/{{project_name}}/{{project_revision}}
+                               - name: GORDO_LOG_LEVEL
+                                 value: "{{log_level}}"
 
                              resources:
                                requests:
@@ -1034,6 +1040,8 @@ spec:
           secretKeyRef:
             name: dlserviceauth
             key: tenant_id_secret
+      - name: GORDO_LOG_LEVEL
+        value: "{{log_level}}"
 
   - name: influx-cleanup
     activeDeadlineSeconds: 300

tests/gordo_components/cli/test_cli.py

@@ -1,6 +1,7 @@
 import os
 import pytest
 import logging
+import re
 
 from click.testing import CliRunner
 from unittest import mock
@@ -441,3 +442,31 @@ def test_server_to_sql_cli():
     args = ["workflow", "server-to-sql", "--help"]
     result = runner.invoke(cli.gordo, args)
     assert result.exit_code == 0
+
+
+def test_log_level_cli():
+    """
+    Test that the --log-level option in the CLI sets the correct log-level in the genreated config file.
+    """
+
+    runner = CliRunner()
+    args = [
+        "--log-level",
+        "test_log_level",
+        "workflow",
+        "generate",
+        "--machine-config",
+        "examples/config_crd.yaml",
+        "--project-name",
+        "test",
+    ]
+
+    result = runner.invoke(cli.gordo, args)
+
+    # Find the value on the next line after the key GORDO_LOG_LEVEL
+    gordo_log_levels = re.findall(
+        r"(?<=GORDO_LOG_LEVEL\r|GORDO_LOG_LEVEL\n)[^\r\n]+", result.stdout
+    )
+
+    # Assert all the values to the GORDO_LOG_LEVEL key contains the correct log-level
+    assert all(["TEST_LOG_LEVEL" in value for value in gordo_log_levels])

tests/gordo_components/workflow/test_workflow_generator/data/config-test-with-log-key.yml

@@ -0,0 +1,28 @@
+machines:
+
+  - name: ct-23-0001 #1st machine
+    dataset:
+      tags: #list of tags for 1st machine
+        - GRA-TE  -23-0733.PV
+        - GRA-TT  -23-0719.PV
+      train_start_date: 2016-11-07T09:11:30+01:00
+      train_end_date: 2018-09-15T03:01:00+01:00
+
+  - name: ct-23-0002 #2nd machine
+    dataset:
+      tags: #list of tags for 2nd machine
+        - CT/1
+        - CT/2
+        - CT/3
+      train_start_date: 2016-11-07T09:11:30+01:00
+      train_end_date: 2018-09-15T03:01:00+01:00
+
+globals:
+  model:
+    sklearn.pipeline.Pipeline:
+      steps:
+        - sklearn.preprocessing.data.MinMaxScaler
+        - gordo_components.machine.model.models.KerasAutoEncoder:
+            kind: feedforward_hourglass
+  runtime:
+    log_level: debug

tests/gordo_components/workflow/test_workflow_generator/test_workflow_generator.py

@@ -2,6 +2,7 @@
 
 import logging
 import os
+import re
 
 import docker
 import pytest
@@ -491,3 +492,26 @@ def test_valid_owner_ref(owner_ref_str: str, valid: bool):
     else:
         with pytest.raises(TypeError):
             wg._valid_owner_ref(owner_ref_str)
+
+
+@pytest.mark.parametrize(
+    "test_file, log_level",
+    (
+        ("config-test-with-log-key.yml", "DEBUG"),
+        ("config-test-with-models.yml", "INFO"),
+    ),
+)
+def test_log_level_key(test_file: str, log_level: str, path_to_config_files: str):
+    """
+    Test that GORDO_LOG_LEVEL is set to the correct value if specified in the config file, or default to INFO if not
+    specified.
+    """
+    workflow_str = _generate_test_workflow_str(path_to_config_files, test_file)
+
+    # Find the value on the next line after the key GORDO_LOG_LEVEL
+    gordo_log_levels = re.findall(
+        r"(?<=GORDO_LOG_LEVEL\r|GORDO_LOG_LEVEL\n)[^\r\n]+", workflow_str
+    )
+
+    # Assert all the values to the GORDO_LOG_LEVEL key contains the correct log-level
+    assert all([log_level in value for value in gordo_log_levels])