diff --git a/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-configmap.yaml b/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-configmap.yaml index f179441..0fee092 100644 --- a/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-configmap.yaml +++ b/charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-configmap.yaml @@ -14,16 +14,26 @@ data: {{- end }} parsers.conf: | {{- .Values.containerLogs.fluentBit.config.customParsers | nindent 4 }} -{{- if contains "us-iso-" .Values.region }} +{{- if hasPrefix "us-iso-" .Values.region }} {{- range $key, $val := .Values.containerLogs.fluentBit.config.adcIsoExtraFiles }} {{ $key }}: | {{- (tpl $val $) | nindent 4 }} {{- end -}} -{{- else if contains "us-isob-" .Values.region }} +{{- else if hasPrefix "us-isob-" .Values.region }} {{- range $key, $val := .Values.containerLogs.fluentBit.config.adcIsobExtraFiles }} {{ $key }}: | {{- (tpl $val $) | nindent 4 }} {{- end -}} +{{- else if hasPrefix "us-isof-" .Values.region }} + {{- range $key, $val := .Values.containerLogs.fluentBit.config.adcIsofExtraFiles }} + {{ $key }}: | + {{- (tpl $val $) | nindent 4 }} + {{- end -}} +{{- else if hasPrefix "eu-isoe-" .Values.region }} + {{- range $key, $val := .Values.containerLogs.fluentBit.config.euAdcIsoeExtraFiles }} + {{ $key }}: | + {{- (tpl $val $) | nindent 4 }} + {{- end -}} {{- else }} {{- range $key, $val := .Values.containerLogs.fluentBit.config.extraFiles }} {{ $key }}: | diff --git a/charts/amazon-cloudwatch-observability/values.yaml b/charts/amazon-cloudwatch-observability/values.yaml index dd85f80..1922e4e 100644 --- a/charts/amazon-cloudwatch-observability/values.yaml +++ b/charts/amazon-cloudwatch-observability/values.yaml @@ -554,6 +554,318 @@ containerLogs: auto_create_group true endpoint logs.${AWS_REGION}.sc2s.sgov.gov extra_user_agent container-insights + adcIsofExtraFiles: + application-log.conf: | + [INPUT] + Name tail + Tag application.* + Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy* + Path /var/log/containers/*.log + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_container.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + Read_from_Head ${READ_FROM_HEAD} + + [INPUT] + Name tail + Tag application.* + Path /var/log/containers/fluent-bit* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_log.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [INPUT] + Name tail + Tag application.* + Path /var/log/containers/cloudwatch-agent* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_cwagent.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [FILTER] + Name kubernetes + Match application.* + Kube_URL https://kubernetes.default.svc:443 + Kube_Tag_Prefix application.var.log.containers. + Merge_Log On + Merge_Log_Key log_processed + K8S-Logging.Parser On + K8S-Logging.Exclude Off + Labels Off + Annotations Off + Use_Kubelet On + Kubelet_Port 10250 + Buffer_Size 0 + + [OUTPUT] + Name cloudwatch_logs + Match application.* + region ${AWS_REGION} + log_group_name /aws/containerinsights/${CLUSTER_NAME}/application + log_stream_prefix ${HOST_NAME}- + auto_create_group true + endpoint logs.${AWS_REGION}.csp.hci.ic.gov + extra_user_agent container-insights + dataplane-log.conf: | + [INPUT] + Name systemd + Tag dataplane.systemd.* + Systemd_Filter _SYSTEMD_UNIT=docker.service + Systemd_Filter _SYSTEMD_UNIT=containerd.service + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + DB /var/fluent-bit/state/systemd.db + Path /var/log/journal + Read_From_Tail ${READ_FROM_TAIL} + + [INPUT] + Name tail + Tag dataplane.tail.* + Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_dataplane_tail.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + Read_from_Head ${READ_FROM_HEAD} + + [FILTER] + Name modify + Match dataplane.systemd.* + Rename _HOSTNAME hostname + Rename _SYSTEMD_UNIT systemd_unit + Rename MESSAGE message + Remove_regex ^((?!hostname|systemd_unit|message).)*$ + + [FILTER] + Name aws + Match dataplane.* + imds_version v2 + + [OUTPUT] + Name cloudwatch_logs + Match dataplane.* + region ${AWS_REGION} + log_group_name /aws/containerinsights/${CLUSTER_NAME}/dataplane + log_stream_prefix ${HOST_NAME}- + auto_create_group true + endpoint logs.${AWS_REGION}.csp.hci.ic.gov + extra_user_agent container-insights + host-log.conf: | + [INPUT] + Name tail + Tag host.dmesg + Path /var/log/dmesg + Key message + DB /var/fluent-bit/state/flb_dmesg.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [INPUT] + Name tail + Tag host.messages + Path /var/log/messages + Parser syslog + DB /var/fluent-bit/state/flb_messages.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [INPUT] + Name tail + Tag host.secure + Path /var/log/secure + Parser syslog + DB /var/fluent-bit/state/flb_secure.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [FILTER] + Name aws + Match host.* + imds_version v2 + + [OUTPUT] + Name cloudwatch_logs + Match host.* + region ${AWS_REGION} + log_group_name /aws/containerinsights/${CLUSTER_NAME}/host + log_stream_prefix ${HOST_NAME}. + auto_create_group true + endpoint logs.${AWS_REGION}.csp.hci.ic.gov + extra_user_agent container-insights + euAdcIsoeExtraFiles: + application-log.conf: | + [INPUT] + Name tail + Tag application.* + Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy* + Path /var/log/containers/*.log + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_container.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + Read_from_Head ${READ_FROM_HEAD} + + [INPUT] + Name tail + Tag application.* + Path /var/log/containers/fluent-bit* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_log.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [INPUT] + Name tail + Tag application.* + Path /var/log/containers/cloudwatch-agent* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_cwagent.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [FILTER] + Name kubernetes + Match application.* + Kube_URL https://kubernetes.default.svc:443 + Kube_Tag_Prefix application.var.log.containers. + Merge_Log On + Merge_Log_Key log_processed + K8S-Logging.Parser On + K8S-Logging.Exclude Off + Labels Off + Annotations Off + Use_Kubelet On + Kubelet_Port 10250 + Buffer_Size 0 + + [OUTPUT] + Name cloudwatch_logs + Match application.* + region ${AWS_REGION} + log_group_name /aws/containerinsights/${CLUSTER_NAME}/application + log_stream_prefix ${HOST_NAME}- + auto_create_group true + endpoint logs.${AWS_REGION}.cloud.adc-e.uk + extra_user_agent container-insights + dataplane-log.conf: | + [INPUT] + Name systemd + Tag dataplane.systemd.* + Systemd_Filter _SYSTEMD_UNIT=docker.service + Systemd_Filter _SYSTEMD_UNIT=containerd.service + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + DB /var/fluent-bit/state/systemd.db + Path /var/log/journal + Read_From_Tail ${READ_FROM_TAIL} + + [INPUT] + Name tail + Tag dataplane.tail.* + Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy* + multiline.parser docker, cri + DB /var/fluent-bit/state/flb_dataplane_tail.db + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type filesystem + Read_from_Head ${READ_FROM_HEAD} + + [FILTER] + Name modify + Match dataplane.systemd.* + Rename _HOSTNAME hostname + Rename _SYSTEMD_UNIT systemd_unit + Rename MESSAGE message + Remove_regex ^((?!hostname|systemd_unit|message).)*$ + + [FILTER] + Name aws + Match dataplane.* + imds_version v2 + + [OUTPUT] + Name cloudwatch_logs + Match dataplane.* + region ${AWS_REGION} + log_group_name /aws/containerinsights/${CLUSTER_NAME}/dataplane + log_stream_prefix ${HOST_NAME}- + auto_create_group true + endpoint logs.${AWS_REGION}.cloud.adc-e.uk + extra_user_agent container-insights + host-log.conf: | + [INPUT] + Name tail + Tag host.dmesg + Path /var/log/dmesg + Key message + DB /var/fluent-bit/state/flb_dmesg.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [INPUT] + Name tail + Tag host.messages + Path /var/log/messages + Parser syslog + DB /var/fluent-bit/state/flb_messages.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [INPUT] + Name tail + Tag host.secure + Path /var/log/secure + Parser syslog + DB /var/fluent-bit/state/flb_secure.db + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 10 + Read_from_Head ${READ_FROM_HEAD} + + [FILTER] + Name aws + Match host.* + imds_version v2 + + [OUTPUT] + Name cloudwatch_logs + Match host.* + region ${AWS_REGION} + log_group_name /aws/containerinsights/${CLUSTER_NAME}/host + log_stream_prefix ${HOST_NAME}. + auto_create_group true + endpoint logs.${AWS_REGION}.cloud.adc-e.uk + extra_user_agent container-insights configWindows: service: | [ SERVICE ]