diff --git a/README.md b/README.md index d46517a..a829e61 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,12 @@ Terraform module that deploys cloud-platform ingress controllers among another resources (like certificates) +This module is also responsilbe for our WAF. It is provided by [modsec](https://github.com/SpiderLabs/ModSecurity). Although we have a cluster wide set of fluent-bit containers which collect and ship our logs to open search/ elastic search. We can't rely on that to collect modsec audit logs which are written to file. We need to write these logs to file because when we push them directly to stdout we lose logs. This is due to the scale of traffic in our live cluster. We increase log reliability by writing to file. + +This means we need to ship modsec audit logs separtely as the cluster level fluent-bit cannot access internal container files. So we introduced a fluent-bit side car which has the filesystem mounted and accessible. We have one further sidecar mounted to handle log rotation using [logrotate](https://linux.die.net/man/8/logrotate), this prevents our logs filling up our master node file space and causing node issues. + +![modsec audit logs diagram]("./images/modsec-audit-logs-diagram.png/" "modsec pod architecture") + ## Usage See [example](example/) dir @@ -40,12 +46,9 @@ No modules. | [kubectl_manifest.nginx_ingress_default_certificate](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [kubernetes_config_map.fluent-bit-config](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [kubernetes_config_map.fluent_bit_lua_script](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | +| [kubernetes_config_map.logrotate_config](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [kubernetes_config_map.modsecurity_nginx_config](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | -| [kubernetes_cron_job_v1.restart_modsec_containers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cron_job_v1) | resource | | [kubernetes_namespace.ingress_controllers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | -| [kubernetes_role_binding_v1.restart_modsec_containers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding_v1) | resource | -| [kubernetes_role_v1.restart_modsec_containers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_v1) | resource | -| [kubernetes_service_account_v1.restart_modsec_containers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account_v1) | resource | | [template_file.nginx_ingress_default_certificate](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | ## Inputs diff --git a/configmap.tf b/configmap.tf index ce6ad55..59b5581 100644 --- a/configmap.tf +++ b/configmap.tf @@ -136,8 +136,8 @@ resource "kubernetes_config_map" "fluent_bit_lua_script" { data = { "cb_extract_tag_value.lua" = <<-EOT function cb_extract_tag_value(tag, timestamp, record) - local github_team = string.gmatch(record["log"], '%[tag "github_team=([%a+|%-]*)"%]') - local github_team_from_json = string.gmatch(record["log"], '"tags":%[.*"github_team=([%a+|%-]*)".*%]') + local github_team = string.gmatch(record["log"], '%[tag "github_team=([%w+|%-]*)"%]') + local github_team_from_json = string.gmatch(record["log"], '"tags":%[.*"github_team=([%w+|%-]*)".*%]') local new_record = record local team_matches = {} @@ -199,3 +199,44 @@ resource "kubernetes_config_map" "modsecurity_nginx_config" { } } + +resource "kubernetes_config_map" "logrotate_config" { + count = var.enable_modsec ? 1 : 0 + + metadata { + name = "logrotate-config" + namespace = "ingress-controllers" + labels = { + "k8s-app" = var.controller_name + } + } + data = { + "logrotate.conf" = <<-EOT + /var/log/audit/**/**/* { + hourly + rotate 0 + missingok + maxage 1 + } + + /var/log/audit/*.log { + su root 82 + hourly + rotate 2 + missingok + compress + delaycompress + copytruncate + maxage 1 + } + EOT + } + + depends_on = [ + kubernetes_namespace.ingress_controllers, + ] + + lifecycle { + ignore_changes = [metadata[0].annotations] + } +} diff --git a/cron.tf b/cron.tf deleted file mode 100644 index ff9da7e..0000000 --- a/cron.tf +++ /dev/null @@ -1,69 +0,0 @@ -resource "kubernetes_service_account_v1" "restart_modsec_containers" { - metadata { - name = "restart-modsec-containers" - namespace = "ingress-controllers" - } -} - -resource "kubernetes_role_v1" "restart_modsec_containers" { - metadata { - name = "restart-modsec-containers" - namespace = "ingress-controllers" - } - - rule { - api_groups = ["apps", "applications"] - resources = ["deployments"] - verbs = ["get", "list", "patch"] - } -} - -resource "kubernetes_role_binding_v1" "restart_modsec_containers" { - metadata { - name = "restart-modsec-containers" - namespace = "ingress-controllers" - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "Role" - name = "restart-modsec-containers" - } - subject { - kind = "ServiceAccount" - name = "restart-modsec-containers" - namespace = "ingress-controllers" - } -} - -resource "kubernetes_cron_job_v1" "restart_modsec_containers" { - metadata { - name = "restart-modsec-containers-nightly" - namespace = "ingress-controllers" - } - spec { - concurrency_policy = "Forbid" - failed_jobs_history_limit = 2 - schedule = "00 23 * * *" - starting_deadline_seconds = 10 - successful_jobs_history_limit = 0 - job_template { - metadata {} - spec { - backoff_limit = 2 - active_deadline_seconds = 600 - ttl_seconds_after_finished = 10 - template { - metadata {} - spec { - service_account_name = "restart-modsec-containers" - container { - name = "kubectl" - image = "bitnami/kubectl" - command = ["kubectl", "rollout", "restart", "deployment/nginx-ingress-modsec-controller"] - } - } - } - } - } - } -} diff --git a/images/modsec-audit-logs-diagram.png b/images/modsec-audit-logs-diagram.png new file mode 100644 index 0000000..39ebe34 Binary files /dev/null and b/images/modsec-audit-logs-diagram.png differ diff --git a/templates/values.yaml.tpl b/templates/values.yaml.tpl index fa3b276..07c7a7e 100644 --- a/templates/values.yaml.tpl +++ b/templates/values.yaml.tpl @@ -18,7 +18,9 @@ controller: - name: fluent-bit-luascripts configMap: name: fluent-bit-luascripts - + - name: logrotate-config + configMap: + name: logrotate-config extraVolumeMounts: ## Additional volumeMounts to the controller main container. @@ -56,6 +58,26 @@ controller: mountPath: /fluent-bit/scripts/ - name: logs-volume mountPath: /var/log/audit/ + - name: logrotate + securityContext: + runAsGroup: 82 + image: debian:bookworm-slim + command: + - sh + - -c + - | + apt update + apt install logrotate -y + groupadd -g 82 82 + cp /home/logrotate.conf /etc/logrotate.conf + ln -s /etc/cron.daily/logrotate /etc/cron.hourly/logrotate + service cron start + sleep infinity + volumeMounts: + - name: logrotate-config + mountPath: /home + - name: logs-volume + mountPath: /var/log/audit/ %{ endif ~} # -- Process Ingress objects without ingressClass annotation/ingressClassName field