-
Notifications
You must be signed in to change notification settings - Fork 31
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
1. Added terraform templates for fluentbit and CW agent on windows. 2. Fix fluentbit tests for Windows
- Loading branch information
1 parent
63de48b
commit 47b88f8
Showing
14 changed files
with
1,251 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
62 changes: 62 additions & 0 deletions
62
terraform/eks/daemon/default_resources/cwagent-windows.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| # create amazon-cloudwatch namespace | ||
| apiVersion: apps/v1 | ||
| kind: DaemonSet | ||
| metadata: | ||
| name: cloudwatch-agent-windows | ||
| namespace: amazon-cloudwatch | ||
| spec: | ||
| selector: | ||
| matchLabels: | ||
| app: cloudwatch-agent | ||
| template: | ||
| metadata: | ||
| labels: | ||
| app: cloudwatch-agent | ||
| spec: | ||
| securityContext: | ||
| windowsOptions: | ||
| hostProcess: true | ||
| runAsUserName: "NT AUTHORITY\\SYSTEM" | ||
| hostNetwork: true | ||
| containers: | ||
| - name: cloudwatch-agent | ||
| image: CW_TEST_IMAGE | ||
| volumeMounts: | ||
| - name: cwagentconfig | ||
| mountPath: C:\Program Files\Amazon\AmazonCloudWatchAgent\cwagentconfig | ||
| resources: | ||
| limits: | ||
| cpu: 400m | ||
| memory: 400Mi | ||
| requests: | ||
| cpu: 400m | ||
| memory: 400Mi | ||
| imagePullPolicy: Always | ||
| env: | ||
| - name: HOST_IP | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: "status.hostIP" | ||
| - name: HOST_NAME | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: spec.nodeName | ||
| - name: K8S_NAMESPACE | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: metadata.namespace | ||
| - name: CI_VERSION | ||
| value: "k8s/1.3.20" | ||
| - name: CWAGENT_LOG_LEVEL | ||
| value: DEBUG | ||
| - name: RUN_IN_CONTAINER | ||
| value: "True" | ||
| - name: RUN_AS_HOST_PROCESS_CONTAINER | ||
| value: "True" | ||
| nodeSelector: | ||
| kubernetes.io/os: windows | ||
| serviceAccountName: cloudwatch-agent | ||
| volumes: | ||
| - name: cwagentconfig | ||
| configMap: | ||
| name: cwagentconfig |
147 changes: 147 additions & 0 deletions
147
terraform/eks/daemon/default_resources/fluenbit-windows-configmap.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| name: fluent-bit-windows-config | ||
| namespace: amazon-cloudwatch | ||
| labels: | ||
| k8s-app: fluent-bit | ||
| data: | ||
| fluent-bit.conf: | | ||
| [SERVICE] | ||
| Flush 5 | ||
| Log_Level info | ||
| Daemon off | ||
| net.dns.resolver LEGACY | ||
| Parsers_File parsers.conf | ||
| @INCLUDE application-log.conf | ||
| @INCLUDE dataplane-log.conf | ||
| @INCLUDE host-log.conf | ||
| application-log.conf: | | ||
| [INPUT] | ||
| Name tail | ||
| Tag application.* | ||
| Exclude_Path C:\\var\\log\\containers\\fluent-bit*, C:\\var\\log\\containers\\cloudwatch-agent* | ||
| Path C:\\var\\log\\containers\\*.log | ||
| Parser docker | ||
| DB C:\\var\\fluent-bit\\state\\flb_container.db | ||
| Mem_Buf_Limit 5MB | ||
| Skip_Long_Lines On | ||
| Rotate_Wait 30 | ||
| Read_from_Head ${READ_FROM_HEAD} | ||
| [INPUT] | ||
| Name tail | ||
| Tag application.* | ||
| Path C:\\var\\log\\containers\\fluent-bit* | ||
| Parser docker | ||
| DB C:\\var\\fluent-bit\\state\\flb_log.db | ||
| Mem_Buf_Limit 5MB | ||
| Skip_Long_Lines On | ||
| Rotate_Wait 30 | ||
| Refresh_Interval 10 | ||
| Read_from_Head ${READ_FROM_HEAD} | ||
| [INPUT] | ||
| Name tail | ||
| Tag application.* | ||
| Path C:\\var\\log\\containers\\cloudwatch-agent* | ||
| Parser docker | ||
| DB C:\\var\\fluent-bit\\state\\flb_cwagent.db | ||
| Mem_Buf_Limit 5MB | ||
| Skip_Long_Lines On | ||
| Rotate_Wait 30 | ||
| Refresh_Interval 10 | ||
| Read_from_Head ${READ_FROM_HEAD} | ||
| [OUTPUT] | ||
| Name cloudwatch_logs | ||
| Match application.* | ||
| region ${AWS_REGION} | ||
| log_group_name /aws/containerinsights/${CLUSTER_NAME}/application | ||
| log_stream_prefix ${HOST_NAME}- | ||
| auto_create_group true | ||
| extra_user_agent container-insights | ||
| dataplane-log.conf: | | ||
| [INPUT] | ||
| Name tail | ||
| Tag dataplane.tail.* | ||
| Path C:\\ProgramData\\containerd\\root\\*.log, C:\\ProgramData\\Amazon\\EKS\\logs\\*.log | ||
| Parser dataplane_firstline | ||
| DB C:\\var\\fluent-bit\\state\\flb_dataplane_tail.db | ||
| Mem_Buf_Limit 5MB | ||
| Skip_Long_Lines On | ||
| Rotate_Wait 30 | ||
| Refresh_Interval 10 | ||
| Read_from_Head ${READ_FROM_HEAD} | ||
| [INPUT] | ||
| Name tail | ||
| Tag dataplane.tail.C.ProgramData.Amazon.EKS.logs.vpc-bridge | ||
| Path C:\\ProgramData\\Amazon\\EKS\\logs\\*.log.* | ||
| Path_Key file_name | ||
| Parser dataplane_firstline | ||
| DB C:\\var\\fluent-bit\\state\\flb_dataplane_cni_tail.db | ||
| Mem_Buf_Limit 5MB | ||
| Skip_Long_Lines On | ||
| Rotate_Wait 30 | ||
| Refresh_Interval 10 | ||
| Read_from_Head ${READ_FROM_HEAD} | ||
| [FILTER] | ||
| Name aws | ||
| Match dataplane.* | ||
| imds_version v2 | ||
| [OUTPUT] | ||
| Name cloudwatch_logs | ||
| Match dataplane.* | ||
| region ${AWS_REGION} | ||
| log_group_name /aws/containerinsights/${CLUSTER_NAME}/dataplane | ||
| log_stream_prefix ${HOST_NAME}- | ||
| auto_create_group true | ||
| extra_user_agent container-insights | ||
| host-log.conf: | | ||
| [INPUT] | ||
| Name winlog | ||
| Channels EKS, System | ||
| DB C:\\var\\fluent-bit\\state\\flb_system_winlog.db | ||
| Interval_Sec 60 | ||
| [FILTER] | ||
| Name aws | ||
| Match winlog.* | ||
| imds_version v2 | ||
| [OUTPUT] | ||
| Name cloudwatch_logs | ||
| Match winlog.* | ||
| region ${AWS_REGION} | ||
| log_group_name /aws/containerinsights/${CLUSTER_NAME}/host | ||
| log_stream_prefix ${HOST_NAME}. | ||
| auto_create_group true | ||
| extra_user_agent container-insights | ||
| parsers.conf: | | ||
| [PARSER] | ||
| Name docker | ||
| Format json | ||
| Time_Key time | ||
| Time_Format %b %d %H:%M:%S | ||
| [PARSER] | ||
| Name container_firstline | ||
| Format regex | ||
| Regex (?<log>(?<="log":")\S(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=}) | ||
| Time_Key time | ||
| Time_Format %Y-%m-%dT%H:%M:%S.%LZ | ||
| [PARSER] | ||
| Name dataplane_firstline | ||
| Format regex | ||
| Regex (?<log>(?<="log":")\S(?!\.).*?)(?<!\\)".*(?<stream>(?<="stream":").*?)".*(?<time>\d{4}-\d{1,2}-\d{1,2}T\d{2}:\d{2}:\d{2}\.\w*).*(?=}) | ||
| Time_Key time | ||
| Time_Format %Y-%m-%dT%H:%M:%S.%LZ |
76 changes: 76 additions & 0 deletions
76
terraform/eks/daemon/default_resources/fluenbit-windows-daemonset.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| apiVersion: apps/v1 | ||
| kind: DaemonSet | ||
| metadata: | ||
| name: fluent-bit-windows | ||
| namespace: amazon-cloudwatch | ||
| labels: | ||
| k8s-app: fluent-bit | ||
| version: v1 | ||
| kubernetes.io/cluster-service: "true" | ||
| spec: | ||
| selector: | ||
| matchLabels: | ||
| k8s-app: fluent-bit | ||
| template: | ||
| metadata: | ||
| labels: | ||
| k8s-app: fluent-bit | ||
| version: v1 | ||
| kubernetes.io/cluster-service: "true" | ||
| spec: | ||
| securityContext: | ||
| windowsOptions: | ||
| hostProcess: true | ||
| runAsUserName: "NT AUTHORITY\\System" | ||
| hostNetwork: true | ||
| nodeSelector: | ||
| kubernetes.io/os: windows | ||
| containers: | ||
| - name: fluent-bit | ||
| image: public.ecr.aws/aws-observability/aws-for-fluent-bit:windowsservercore-stable | ||
| imagePullPolicy: Always | ||
| command: ["powershell.exe", "-Command", "New-Item -ItemType Directory -Path C:\\var\\fluent-bit\\state -Force;", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/bin/fluent-bit.exe", "-e", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/kinesis.dll", "-e", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/firehose.dll", "-e", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/cloudwatch.dll", "-c", "%CONTAINER_SANDBOX_MOUNT_POINT%/fluent-bit/configuration/fluent-bit.conf"] | ||
| env: | ||
| - name: AWS_REGION | ||
| valueFrom: | ||
| configMapKeyRef: | ||
| name: "fluent-bit-cluster-info" | ||
| key: "logs.region" | ||
| - name: CLUSTER_NAME | ||
| valueFrom: | ||
| configMapKeyRef: | ||
| name: "fluent-bit-cluster-info" | ||
| key: "cluster.name" | ||
| - name: READ_FROM_HEAD | ||
| valueFrom: | ||
| configMapKeyRef: | ||
| name: "fluent-bit-cluster-info" | ||
| key: "read.head" | ||
| - name: HOST_NAME | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: spec.nodeName | ||
| - name: HOSTNAME | ||
| valueFrom: | ||
| fieldRef: | ||
| apiVersion: v1 | ||
| fieldPath: metadata.name | ||
| - name: CI_VERSION | ||
| value: "k8s/1.3.17" | ||
| resources: | ||
| limits: | ||
| cpu: 500m | ||
| memory: 600Mi | ||
| requests: | ||
| cpu: 500m | ||
| memory: 600Mi | ||
| volumeMounts: | ||
| - name: fluent-bit-config | ||
| mountPath: fluent-bit\configuration\ | ||
| volumes: | ||
| - name: fluent-bit-config | ||
| configMap: | ||
| name: fluent-bit-windows-config | ||
| terminationGracePeriodSeconds: 10 | ||
| dnsPolicy: ClusterFirstWithHostNet | ||
| serviceAccountName: cloudwatch-agent |
28 changes: 28 additions & 0 deletions
28
terraform/eks/daemon/default_resources/test-sample-windows.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| apiVersion: apps/v1 | ||
| kind: Deployment | ||
| metadata: | ||
| name: windows-test-deployment | ||
| spec: | ||
| selector: | ||
| matchLabels: | ||
| app: windows-test-deployment | ||
| tier: backend | ||
| track: stable | ||
| replicas: 1 | ||
| template: | ||
| metadata: | ||
| labels: | ||
| app: windows-test-deployment | ||
| tier: backend | ||
| track: stable | ||
| spec: | ||
| containers: | ||
| - name: windows-test-deployment | ||
| image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2022 | ||
| command: | ||
| - powershell.exe | ||
| - -command | ||
| - " ping -t google.com " | ||
| nodeSelector: | ||
| kubernetes.io/os: windows | ||
|
|
Oops, something went wrong.