ilogtail-daemonset-file-to-sls.yaml

# Copyright 2022 iLogtail Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# To collect file logs in container and write to SLS.
# 
# You may change ContainerInfo parameters to select containers for collection.
# For a full list of parameters, visit https://ilogtail.gitbook.io/ilogtail-docs/data-pipeline/input/file-log
# You may change <sls_endpoint>, <sls_project> and <sls_logstore> according to you SLS setup.
# For a full list of parameters, visit https://ilogtail.gitbook.io/ilogtail-docs/data-pipeline/flusher/sls

---
apiVersion: v1
kind: Namespace
metadata:
  name: ilogtail
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: ilogtail-user-cm
  namespace: ilogtail
data:
  json_log.yaml: |
    enable: true
    inputs:
      - Type: file_log
        LogPath: /root/log
        FilePattern: "json.log"
        ContainerFile: true
        ContainerInfo:
          K8sContainerRegex: json-log
          IncludeK8sLabel:
            app: json-log
          ExcludeEnv:
            COLLECT_FLAG: false
    processors:
      - Type: processor_json
        SourceKey: content
        KeepSource: false
        ExpandDepth: 1
        ExpandConnector: ""
    flushers:
      - Type: flusher_sls
        Endpoint: <sls_endpoint>
        ProjectName: <sls_project>
        LogstoreName: <sls_logstore>
---
apiVersion: v1
kind: Secret
metadata:
  name: ilogtail-secret
  namespace: ilogtail
type: Opaque
data:
  access_key: <base64_access_key_secret>
  access_key_id: <base64_access_key_id>
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: ilogtail-ds
  namespace: ilogtail
  labels:
    k8s-app: logtail-ds
spec:
  selector:
    matchLabels:
      k8s-app: logtail-ds
  template:
    metadata:
      labels:
        k8s-app: logtail-ds
    spec:
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      containers:
        - name: logtail
          env:
            - name: ALIYUN_LOG_ENV_TAGS # add log tags from env
              value: _node_name_|_node_ip_
            - name: _node_name_
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
            - name: _node_ip_
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.hostIP
            - name: default_access_key_id
              valueFrom:
                secretKeyRef:
                  name: ilogtail-secret
                  key: access_key_id
                  optional: true
            - name: default_access_key
              valueFrom:
                secretKeyRef:
                  name: ilogtail-secret
                  key: access_key
                  optional: true
            - name: cpu_usage_limit
              value: "1"
            - name: mem_usage_limit
              value: "512"
          image: >-
            sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/ilogtail:latest
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 1000m
              memory: 1Gi
            requests:
              cpu: 400m
              memory: 384Mi
          volumeMounts:
            - mountPath: /var/run
              name: run
            - mountPath: /logtail_host
              mountPropagation: HostToContainer
              name: root
              readOnly: true
            - mountPath: /usr/local/ilogtail/checkpoint
              name: checkpoint
            - mountPath: /usr/local/ilogtail/user_yaml_config.d
              name: user-config
              readOnly: true
      dnsPolicy: ClusterFirstWithHostNet
      hostNetwork: true
      volumes:
        - hostPath:
            path: /var/run
            type: Directory
          name: run
        - hostPath:
            path: /
            type: Directory
          name: root
        - hostPath:
            path: /var/lib/ilogtail-ilogtail-ds/checkpoint
            type: DirectoryOrCreate
          name: checkpoint
        - configMap:
            defaultMode: 420
            name: ilogtail-user-cm
          name: user-config