diff --git a/.cruft.json b/.cruft.json index 80c2285..59b9794 100644 --- a/.cruft.json +++ b/.cruft.json @@ -7,11 +7,11 @@ "name": "fluentbit", "slug": "fluentbit", "parameter_key": "fluentbit", - "test_cases": "defaults", + "test_cases": "defaults config", "add_lib": "n", "add_pp": "y", "add_golden": "y", - "add_matrix": "n", + "add_matrix": "y", "add_go_unit": "n", "automerge_patch": "y", "automerge_patch_v0": "n", diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 483910b..9507a39 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -29,6 +29,11 @@ jobs: args: 'check' test: runs-on: ubuntu-latest + strategy: + matrix: + instance: + - defaults + - config defaults: run: working-directory: ${{ env.COMPONENT_NAME }} @@ -37,9 +42,14 @@ jobs: with: path: ${{ env.COMPONENT_NAME }} - name: Compile component - run: make test + run: make test -e instance=${{ matrix.instance }} golden: runs-on: ubuntu-latest + strategy: + matrix: + instance: + - defaults + - config defaults: run: working-directory: ${{ env.COMPONENT_NAME }} @@ -48,4 +58,4 @@ jobs: with: path: ${{ env.COMPONENT_NAME }} - name: Golden diff - run: make golden-diff + run: make golden-diff -e instance=${{ matrix.instance }} diff --git a/Makefile b/Makefile index 8b9ce19..0646f90 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,22 @@ golden-diff: commodore_args += -f tests/$(instance).yml golden-diff: clean .compile ## Diff compile output against the reference version. Review output and run `make gen-golden golden-diff` if this target fails. @git diff --exit-code --minimal --no-index -- tests/golden/$(instance) compiled/ +.PHONY: golden-diff-all +golden-diff-all: recursive_target=golden-diff +golden-diff-all: $(test_instances) ## Run golden-diff for all instances. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: gen-golden-all +gen-golden-all: recursive_target=gen-golden +gen-golden-all: $(test_instances) ## Run gen-golden for all instances. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: lint_kubent_all +lint_kubent_all: recursive_target=lint_kubent +lint_kubent_all: $(test_instances) ## Lint deprecated Kubernetes API versions for all golden test instances. Will exit on first error. Note: this doesn't work when running make with multiple parallel jobs (-j != 1). + +.PHONY: $(test_instances) +$(test_instances): + $(MAKE) $(recursive_target) -e instance=$(basename $(@F)) + .PHONY: clean clean: ## Clean the project rm -rf .cache compiled dependencies vendor helmcharts jsonnetfile*.json || true diff --git a/Makefile.vars.mk b/Makefile.vars.mk index b460c73..fb473e3 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -57,3 +57,4 @@ KUBENT_IMAGE ?= ghcr.io/doitintl/kube-no-trouble:latest KUBENT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE) instance ?= defaults +test_instances = tests/defaults.yml tests/config.yml diff --git a/class/defaults.yml b/class/defaults.yml index 1186932..109cc1c 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -67,7 +67,8 @@ parameters: K8S-Logging.Exclude: "On" # Add additional secrets - secrets: {} + secretName: ${_instance} + secret: {} # Add volumes to the pod spec extraVolumes: [] diff --git a/component/main.jsonnet b/component/main.jsonnet index bb15156..eb2bed7 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -162,24 +162,24 @@ local configmap = kube.ConfigMap(params.configMapName) { }, }; -local secret = kube.Secret(instanceName) { +local secret = kube.Secret(params.secretName) { metadata+: { labels+: { - 'app.kubernetes.io/name': 'fluent-bit', + 'app.kubernetes.io/name': params.secretName, 'app.kubernetes.io/instance': instanceName, 'app.kubernetes.io/component': 'fluent-bit', 'app.kubernetes.io/managed-by': 'commodore', }, }, stringData: { - [s]: params.secrets[s] - for s in std.objectFields(params.secrets) + [s]: params.secret[s] + for s in std.objectFields(params.secret) }, }; { [if params.createNamespace then '00_namespace']: kube.Namespace(params.namespace), - [if std.length(params.secrets) > 0 then '10_custom_secret']: secret, + [if std.length(params.secret) > 0 then '10_custom_secret']: secret, '10_custom_config': configmap, [if params.monitoring.enabled then '20_service_monitor']: kube._Object('monitoring.coreos.com/v1', 'ServiceMonitor', 'fluent-bit') { diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 78c96ef..a2167fe 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -180,6 +180,22 @@ capitalization of keys and values). If the dict for a section doesn't have a key `Name`, the key for the section will be used as the plugin name for the section. This allows avoiding repetition, when it's unnecessary, while still supporting having multiple outputs using the same plugin. +== `secretName` + +[horizontal] +type:: string +default:: `${_instance}` + +The name of the generated secret. + +== `secret` + +[horizontal] +type:: dict +default:: {} + +Create a custom secret which containes the key-value pairs defined in this dict. + == `annotations` [horizontal] diff --git a/renovate.json b/renovate.json index 00c6a19..f2e1d67 100644 --- a/renovate.json +++ b/renovate.json @@ -13,7 +13,7 @@ "separateMinorPatch": true, "postUpgradeTasks": { "commands": [ - "make gen-golden" + "make gen-golden-all" ], "fileFilters": [ "tests/golden/**" diff --git a/tests/config.yml b/tests/config.yml new file mode 100644 index 0000000..30b8b75 --- /dev/null +++ b/tests/config.yml @@ -0,0 +1,10 @@ +parameters: + fluentbit: + secret: + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + + helm_values: + envFrom: + - secretRef: + name: ${fluentbit:secretName} diff --git a/tests/defaults.yml b/tests/defaults.yml index 9334ee2..a4da5b7 100644 --- a/tests/defaults.yml +++ b/tests/defaults.yml @@ -1,3 +1,3 @@ ---- -parameters: - fluentbit: {} +# Overwrite parameters here + +# parameters: {...} diff --git a/tests/golden/config/fluentbit/apps/fluentbit.yaml b/tests/golden/config/fluentbit/apps/fluentbit.yaml new file mode 100644 index 0000000..1d84d89 --- /dev/null +++ b/tests/golden/config/fluentbit/apps/fluentbit.yaml @@ -0,0 +1,3 @@ +spec: + source: + path: manifests/fluentbit/fluentbit diff --git a/tests/golden/config/fluentbit/fluentbit/fluentbit/00_namespace.yaml b/tests/golden/config/fluentbit/fluentbit/fluentbit/00_namespace.yaml new file mode 100644 index 0000000..a51ef51 --- /dev/null +++ b/tests/golden/config/fluentbit/fluentbit/fluentbit/00_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: {} + labels: + name: syn-fluentbit + name: syn-fluentbit diff --git a/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/clusterrole.yaml b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/clusterrole.yaml new file mode 100644 index 0000000..9ffe5ed --- /dev/null +++ b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/clusterrole.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/version: 3.1.4 + helm.sh/chart: fluent-bit-0.47.5 + name: fluentbit +rules: + - apiGroups: + - '' + resources: + - namespaces + - pods + verbs: + - get + - list + - watch diff --git a/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/clusterrolebinding.yaml b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..30554ae --- /dev/null +++ b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/version: 3.1.4 + helm.sh/chart: fluent-bit-0.47.5 + name: fluentbit +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: fluentbit +subjects: + - kind: ServiceAccount + name: fluentbit + namespace: syn-fluentbit diff --git a/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/daemonset.yaml b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/daemonset.yaml new file mode 100644 index 0000000..d719107 --- /dev/null +++ b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/daemonset.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/version: 3.1.4 + helm.sh/chart: fluent-bit-0.47.5 + name: fluentbit + namespace: syn-fluentbit +spec: + selector: + matchLabels: + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/name: fluent-bit + template: + metadata: + annotations: + checksum/config: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + checksum/syn-config: 06e588437d65a5b9ef0171f6ac0a59e7 + fluentbit.io/exclude: 'true' + labels: + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/name: fluent-bit + spec: + containers: + - args: + - --workdir=/fluent-bit/etc + - --config=/fluent-bit/etc/conf/fluent-bit.conf + command: + - /fluent-bit/bin/fluent-bit + envFrom: + - secretRef: + name: fluentbit + image: docker.io/fluent/fluent-bit:3.1.4 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: / + port: http + name: fluent-bit + ports: + - containerPort: 2020 + name: http + protocol: TCP + readinessProbe: + httpGet: + path: /api/v1/health + port: http + volumeMounts: + - mountPath: /fluent-bit/etc/conf + name: config + - mountPath: /var/log + name: varlog + - mountPath: /var/lib/docker/containers + name: varlibdockercontainers + readOnly: true + - mountPath: /etc/machine-id + name: etcmachineid + readOnly: true + dnsPolicy: ClusterFirst + hostNetwork: false + serviceAccountName: fluentbit + volumes: + - configMap: + name: fluentbit + name: config + - hostPath: + path: /var/log + name: varlog + - hostPath: + path: /var/lib/docker/containers + name: varlibdockercontainers + - hostPath: + path: /etc/machine-id + type: File + name: etcmachineid diff --git a/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/service.yaml b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/service.yaml new file mode 100644 index 0000000..f27d202 --- /dev/null +++ b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/version: 3.1.4 + helm.sh/chart: fluent-bit-0.47.5 + name: fluentbit + namespace: syn-fluentbit +spec: + ports: + - name: http + port: 2020 + protocol: TCP + targetPort: http + selector: + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/name: fluent-bit + type: ClusterIP diff --git a/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/serviceaccount.yaml b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/serviceaccount.yaml new file mode 100644 index 0000000..b6ff6de --- /dev/null +++ b/tests/golden/config/fluentbit/fluentbit/fluentbit/01_fluentbit_helmchart/fluent-bit/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/version: 3.1.4 + helm.sh/chart: fluent-bit-0.47.5 + name: fluentbit + namespace: syn-fluentbit diff --git a/tests/golden/config/fluentbit/fluentbit/fluentbit/10_custom_config.yaml b/tests/golden/config/fluentbit/fluentbit/fluentbit/10_custom_config.yaml new file mode 100644 index 0000000..b41c050 --- /dev/null +++ b/tests/golden/config/fluentbit/fluentbit/fluentbit/10_custom_config.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +data: + custom_parsers.conf: '' + fluent-bit.conf: | + [SERVICE] + Daemon Off + Flush 1 + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + HTTP_Server On + Log_Level info + Parsers_File parsers.conf + Plugins_File plugins.conf + [FILTER] + Name kubernetes + K8S-Logging.Exclude On + K8S-Logging.Parser On + Keep_Log On + Match kube.* + Merge_Log On + [INPUT] + Name tail + Mem_Buf_Limit 5MB + Parser docker + Path /var/log/containers/*.log + Skip_Long_lines On + Tag kube.* + [INPUT] + Name systemd + Read_From_Tail On + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Tag host.* +kind: ConfigMap +metadata: + annotations: {} + labels: + app.kubernetes.io/component: fluent-bit + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: fluent-bit + app.kubernetes.io/version: 3.1.4 + name: fluentbit + name: fluentbit + namespace: syn-fluentbit diff --git a/tests/golden/config/fluentbit/fluentbit/fluentbit/10_custom_secret.yaml b/tests/golden/config/fluentbit/fluentbit/fluentbit/10_custom_secret.yaml new file mode 100644 index 0000000..f2b9336 --- /dev/null +++ b/tests/golden/config/fluentbit/fluentbit/fluentbit/10_custom_secret.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +data: {} +kind: Secret +metadata: + annotations: {} + labels: + app.kubernetes.io/component: fluent-bit + app.kubernetes.io/instance: fluentbit + app.kubernetes.io/managed-by: commodore + app.kubernetes.io/name: fluentbit + name: fluentbit + name: fluentbit +stringData: + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: +type: Opaque