Convert fluent-bit.cm.yaml to helm-managed template

Signed-off-by: Jeremy Ho <[email protected]>

.github/actions/deploy-to-environment/action.yaml

@@ -45,7 +45,7 @@ runs:
       run: >-
         helm upgrade --install --atomic ${{ inputs.job_name }} ${{ inputs.app_name }}
         --namespace ${{ inputs.namespace_prefix }}-${{ inputs.namespace_environment }}
-        --repo https://bcgov.github.io/common-object-management-service
+        --repo https://bcgov.github.io/common-document-generation-service
         --values ./.github/environments/values.${{ inputs.environment }}.yaml
         --set image.repository=ghcr.io/${{ github.repository_owner }}
         --set image.tag=sha-$(git rev-parse --short HEAD)

charts/cdogs/README.md

@@ -35,6 +35,12 @@ Kubernetes: `>= 1.13.0`
 | config.enabled | bool | `false` |  |
 | config.releaseScoped | bool | `false` | This should be set to true if and only if you require configmaps and secrets to be release scoped. In the event you want all instances in the same namespace to share a similar configuration, this should be set to false |
 | failurePolicy | string | `"Retry"` |  |
+| fluentBit.config.aws.defaultRegion | string | `"ca-central-1"` |  |
+| fluentBit.config.aws.kinesisStream | string | `"nress-prod-iit-logs"` |  |
+| fluentBit.config.aws.roleArn | string | `nil` |  |
+| fluentBit.config.logHostname | string | `"fluentd-csst.apps.silver.devops.gov.bc.ca"` |  |
+| fluentBit.config.namespace | string | `nil` |  |
+| fluentBit.config.product | string | `"cdogs"` |  |
 | fluentBit.enabled | bool | `false` |  |
 | fluentBit.image.name | string | `"fluent-bit"` |  |
 | fluentBit.image.repository | string | `"docker.io/fluent"` |  |

charts/cdogs/templates/configmap.yaml

@@ -12,3 +12,223 @@ metadata:
   name: {{ include "cdogs.configname" . }}-config
 data: {{ toYaml .Values.config.configMap | nindent 2 }}
 {{- end }}
+{{- if .Values.fluentBit.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  {{- if not .Values.config.releaseScoped }}
+  annotations:
+    "helm.sh/resource-policy": keep
+  {{- else }}
+  labels: {{ include "cdogs.labels" . | nindent 4 }}
+  {{- end }}
+  name: {{ include "cdogs.configname" . }}-fluent-bit
+data:
+  fluent-bit.conf: |
+    [SERVICE]
+      Flush         5
+      Daemon        Off
+      # define the log format (see additional config map key/value)
+      Parsers_File  parsers.conf
+      Log_Level     info
+      HTTP_Server   On
+      HTTP_Listen   0.0.0.0
+      HTTP_Port     2020
+
+    [INPUT]
+      # get logs from file written by node app (eg: CDOGS)
+      Name        tail
+      Path        /var/log/*
+      Tag         app
+      Offset_Key  logFileOffset
+      Path_Key    logFilePath
+
+    [FILTER]
+      # exclude kube probe logs from app logs
+      name     grep
+      match    app
+      Exclude  agent kube*
+
+    [FILTER]
+      name          parser
+      match         app
+      Key_Name      log
+      Parser        json
+      Reserve_Data  On
+      Preserve_Key  On
+
+    [FILTER]
+      # modify log entry to include more key/value pairs
+      name    record_modifier
+      match   app
+      # add pod name
+      Record  hostname ${HOSTNAME}
+      # add productname (eg: 'cdogs')
+      Record  product {{ .Values.fluentBit.config.product }}
+      # add namespace
+      Record  namespace {{ .Values.fluentBit.config.namespace }}
+
+    [FILTER]
+      Name          rewrite_tag
+      Match         app
+      Rule          $level ([a-zA-Z]*)$ $TAG.$level true
+      Emitter_Name  re_emitted
+
+    # for now just send out http level ('access') logs to AWS
+    [FILTER]
+      Name           lua
+      Match          app.*
+      script         script.lua
+      time_as_table  True
+      call           ecsMap
+
+    # Note: only currently sending 'access' (level: http) logs to AWS
+    # TODO: format 'metrics' logs to match a 'fingerprint' in Lambda
+    [OUTPUT]
+      Name      kinesis_streams
+      Match     app.http
+      region    {{ .Values.fluentBit.config.aws.defaultRegion }}
+      stream    {{ .Values.fluentBit.config.aws.kinesisStream }}
+      role_arn  {{ .Values.fluentBit.config.aws.roleArn }}
+      time_key  @timestamp
+
+    [OUTPUT]
+      #### send logs to fluentd:
+      Name           http
+      Match          app
+      Host           {{ .Values.fluentBit.config.logHostname }}
+      Port           80
+      Format         json
+      # the URI becomes the Tag available in fluentd
+      URI            /app
+      # we can also send tag as a header
+      #header_tag    app
+      json_date_key  timestamp
+
+      ### security:
+      #tls           On
+      #tls.debug     4
+      #tls.verify    On
+      #tls.ca_file   /fluent-bit/ssl/ca.crt.pem
+      #tls.crt_file  /fluent-bit/ssl/client.crt.pem
+      #tls.key_file  /fluent-bit/ssl/client.key.pem
+
+    [OUTPUT]
+      Name    stdout
+      Match   *
+      Format  json_lines
+
+  parsers.conf: |
+    [PARSER]
+      Name             json
+      Format           json
+      Time_Key         timestamp
+      Decode_Field_as  escaped_utf8 log do_next
+      Decode_Field_as  json log
+
+  script.lua: |
+    -- add extra ECS fields
+    function ecsMap(tag, timestamp, record)
+      -- map existing fields to a new variable
+      new_record = {}
+
+      -- derive full environment (stage) name from namespace
+      -- see: https://www.lua.org/pil/20.3.html
+      _, _, part1, environmentAbbreviation = string.find(record["namespace"], "([a-zA-Z0-9_+-]+)-([a-zA-Z0-9_+-]+)")
+
+      environmentsArray = {
+        ["localhost"] = "development",
+        ["dev"] = "development",
+        ["test"] = "test",
+        ["prod"] = "production"
+      }
+
+      -- get event.type from log.level
+      eventTypesArray = {
+        ["http"] = "access",
+        ["info"] = "info",
+        ["verbose"] = "metric"
+      }
+
+      ---- for all logs
+
+      new_record["ecs"] = {
+        ["version"] = "1.12"
+      }
+
+      new_record["log"] = {
+        ["file"] = {
+          ["path"] = record["logFilePath"]
+        },
+        ["level"] = record["level"]
+      }
+
+      new_record["service"] = {
+        ["environment"] = environmentsArray[environmentAbbreviation],
+        ["name"] = record["product"],
+        ["type"] = "node"
+      }
+
+      new_record["event"] = {
+        ["kind"] = "event",
+        ["category"] = "web",
+        ["original"] = record["message"],
+        ["duration"] = record["responseTime"],
+        ["sequence"] = record["logFileOffset"],
+        ["created"] = (os.date("!%Y-%m-%dT%H:%M:%S", timestamp["sec"]) .. '.' .. math.floor(timestamp["nsec"] / 1000000) .. 'Z')
+      }
+
+      new_record["agent"] = {
+        ["type"] = "fluentbit",
+        ["version"] = "1.8"
+      }
+
+      new_record["labels"] = {
+        ["project"] = record["product"]
+      }
+
+      new_record["host"] = {
+        ["hostname"] = record["hostname"],
+        ["ip"] = record["ip"],
+        ["name"] = record["namespace"]
+      }
+
+      new_record["user_agent"] = {
+        ["original"] = record["userAgent"]
+      }
+
+      new_record["source.user.id"] = record["azp"]
+
+      ---- access logs
+
+      if record["level"] == "http" then
+        new_record["event"]["type"] = eventTypesArray[record["level"]]
+        new_record["event"]["dataset"] = "express." .. eventTypesArray[record["level"]]
+        new_record["http"] = {
+          ["request"] = {
+            ["body"] = {
+              ["bytes"] = record["contentLength"]
+            },
+            ["method"] = record["method"],
+            ["referrer"] = record["path"]
+          },
+          ["response"] = {
+            ["status_code"] = record["statusCode"]
+          },
+          ["version"] = record["httpVersion"]
+        }
+      end
+
+      ---- metrics logs
+
+      -- if log contains a 'metrics' field
+      if record["metrics"] ~= nill then
+        new_record["metrics"] = record["metrics"]
+        new_record["event"]["type"] = eventTypesArray[record["level"]]
+      end
+
+      -- return the transformed new record
+      return 2, timestamp, new_record
+    end
+{{- end }}

charts/cdogs/templates/deploymentconfig.yaml

@@ -152,7 +152,7 @@ spec:
           emptyDir: {}
         - name: fluent-bit-config
           configMap:
-            name: fluent-bit-config
+            name: {{ include "cdogs.configname" . }}-fluent-bit
         {{- end }}
   test: false
   triggers:

charts/cdogs/values.yaml

@@ -156,6 +156,15 @@ keycloakSecretOverride:
 fluentBit:
   enabled: false
 
+  config:
+    aws:
+      defaultRegion: ca-central-1
+      kinesisStream: nress-prod-iit-logs
+      roleArn: ~
+    logHostname: fluentd-csst.apps.silver.devops.gov.bc.ca
+    namespace: ~
+    product: cdogs
+
   image:
     name: fluent-bit
     repository: docker.io/fluent