diff --git a/README.md b/README.md index 9bb4159..d46517a 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,14 @@ No modules. |------|------| | [helm_release.nginx_ingress](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.nginx_ingress_default_certificate](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubernetes_config_map.fluent-bit-config](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | +| [kubernetes_config_map.fluent_bit_lua_script](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [kubernetes_config_map.modsecurity_nginx_config](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | +| [kubernetes_cron_job_v1.restart_modsec_containers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cron_job_v1) | resource | | [kubernetes_namespace.ingress_controllers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_role_binding_v1.restart_modsec_containers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding_v1) | resource | +| [kubernetes_role_v1.restart_modsec_containers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_v1) | resource | +| [kubernetes_service_account_v1.restart_modsec_containers](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account_v1) | resource | | [template_file.nginx_ingress_default_certificate](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) | data source | ## Inputs @@ -48,6 +54,7 @@ No modules. |------|-------------|------|---------|:--------:| | [backend\_repo](#input\_backend\_repo) | repository for the default backend app | `string` | `"ministryofjustice/cloud-platform-custom-error-pages"` | no | | [backend\_tag](#input\_backend\_tag) | tag of the default backend app | `string` | `"0.6"` | no | +| [cluster](#input\_cluster) | cluster name used for opensearch indicies | `string` | `""` | no | | [cluster\_domain\_name](#input\_cluster\_domain\_name) | The cluster domain used for externalDNS annotations and certmanager | `any` | n/a | yes | | [controller\_name](#input\_controller\_name) | Will be used as the ingress controller name and the class annotation | `string` | n/a | yes | | [default\_cert](#input\_default\_cert) | Useful if you want to use a default certificate for your ingress controller. Format: namespace/secretName | `string` | `"ingress-controllers/default-certificate"` | no | @@ -56,9 +63,11 @@ No modules. | [enable\_latest\_tls](#input\_enable\_latest\_tls) | Provide support to tlsv1.3 along with tlsv1.2 | `bool` | `false` | no | | [enable\_modsec](#input\_enable\_modsec) | Enable https://github.com/SpiderLabs/ModSecurity-nginx | `bool` | `false` | no | | [enable\_owasp](#input\_enable\_owasp) | Use default ruleset from https://github.com/SpiderLabs/owasp-modsecurity-crs/ | `bool` | `false` | no | +| [fluent\_bit\_version](#input\_fluent\_bit\_version) | fluent bit container version used to exrtact modsec audit logs | `string` | `"2.1.8-amd64"` | no | | [is\_live\_cluster](#input\_is\_live\_cluster) | For live clusters externalDNS annotation will have var.live\_domain (default *.cloud-platform.service.justice.gov.uk) | `bool` | `false` | no | | [live1\_cert\_dns\_name](#input\_live1\_cert\_dns\_name) | This is to add the live-1 dns name for eks-live cluster default certificate | `string` | `""` | no | | [live\_domain](#input\_live\_domain) | The live domain used for externalDNS annotation | `string` | `"cloud-platform.service.justice.gov.uk"` | no | +| [opensearch\_modsec\_audit\_host](#input\_opensearch\_modsec\_audit\_host) | domain endpoint for the opensearch cluster | `string` | `""` | no | | [replica\_count](#input\_replica\_count) | Number of replicas set in deployment | `string` | n/a | yes | ## Outputs diff --git a/configmap.tf b/configmap.tf new file mode 100644 index 0000000..ce6ad55 --- /dev/null +++ b/configmap.tf @@ -0,0 +1,201 @@ +resource "kubernetes_config_map" "fluent-bit-config" { + count = var.enable_modsec ? 1 : 0 + + metadata { + name = "fluent-bit-config" + namespace = "ingress-controllers" + labels = { + "k8s-app" = var.controller_name + } + } + data = { + "fluent-bit.conf" = <<-EOT + [SERVICE] + Flush 1 + Log_Level info + Daemon Off + Grace 30 + Parsers_File parsers.conf + Parsers_File custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + Storage.path /var/log/flb-storage/ + Storage.max_chunks_up 64 + Storage.backlog.mem_limit 5MB + + [INPUT] + Name tail + Alias modsec_nginx_ingress_audit_index + Tag cp-ingress-modsec-index-audit.* + Path /var/log/audit/*.log + Parser modsec-audit-log-index + Refresh_Interval 5 + Buffer_Max_Size 5MB + Buffer_Chunk_Size 1M + Offset_Key pause_position_modsec-audit-index + DB cp-ingress-modsec-audit-index.db + DB.locking true + Storage.type filesystem + Storage.pause_on_chunks_overlimit True + + [INPUT] + Name tail + Alias modsec_nginx_ingress_audit + Tag cp-ingress-modsec-audit.* + Path /var/log/audit/**/**/* + Parser docker + Refresh_Interval 5 + Buffer_Max_Size 5MB + Buffer_Chunk_Size 1M + Offset_Key pause_position_modsec-audit + DB cp-ingress-modsec-audit.db + DB.locking true + Storage.type filesystem + Storage.pause_on_chunks_overlimit True + + [FILTER] + Name lua + Match cp-ingress-modsec-audit.* + script /fluent-bit/scripts/cb_extract_tag_value.lua + call cb_extract_tag_value + + [FILTER] + Name parser + Parser generic-json + Match cp-ingress-modsec-audit.* + Key_Name log + Reserve_Data On + Preserve_Key On + + [OUTPUT] + Name opensearch + Alias modsec_nginx_ingress_audit + Match * + Host ${var.opensearch_modsec_audit_host} + Port 443 + Type _doc + Time_Key @timestamp + Logstash_Prefix ${var.cluster}_k8s_modsec_ingress + tls On + Logstash_Format On + Replace_Dots On + Generate_ID On + Retry_Limit False + AWS_AUTH On + AWS_REGION eu-west-2 + Suppress_Type_Name On + Buffer_Size False + EOT + + "custom_parsers.conf" = <<-EOT + [PARSER] + Name modsec-audit-log-index + Format regex + Regex ^(?[^ ]+) (?[^ ]+) (?.*)$ + Time_Key time + Time_Format %d/%m/%Y:T%H:%M:%S.%z + [PARSER] + Name initial-json + Format json + Time_Key time + Time_Keep On + + [PARSER] + Name generic-json + Format json + Time_Key time + Time_Format %Y-%b-%dT%H:%M:%S + Time_Keep On + # Command | Decoder | Field | Optional Action + # =============|==================|================= + Decode_Field_As escaped_utf8 log do_next + Decode_Field_As json log + EOT + } + + depends_on = [ + kubernetes_namespace.ingress_controllers, + ] + + lifecycle { + ignore_changes = [metadata[0].annotations] + } +} + +resource "kubernetes_config_map" "fluent_bit_lua_script" { + count = var.enable_modsec ? 1 : 0 + + metadata { + name = "fluent-bit-luascripts" + namespace = "ingress-controllers" + labels = { + "k8s-app" = var.controller_name + } + } + data = { + "cb_extract_tag_value.lua" = <<-EOT + function cb_extract_tag_value(tag, timestamp, record) + local github_team = string.gmatch(record["log"], '%[tag "github_team=([%a+|%-]*)"%]') + local github_team_from_json = string.gmatch(record["log"], '"tags":%[.*"github_team=([%a+|%-]*)".*%]') + + local new_record = record + local team_matches = {} + local json_matches = {} + + for team in github_team do + table.insert(team_matches, team) + end + + for team in github_team_from_json do + table.insert(json_matches, team) + end + + if #team_matches > 0 then + new_record["github_teams"] = team_matches + return 1, timestamp, new_record + + elseif #json_matches > 0 then + new_record["github_teams"] = json_matches + + return 1, timestamp, new_record + + else + return 0, timestamp, record + end + end + EOT + } + + depends_on = [ + kubernetes_namespace.ingress_controllers, + ] + + lifecycle { + ignore_changes = [metadata[0].annotations] + } +} + +resource "kubernetes_config_map" "modsecurity_nginx_config" { + count = var.enable_modsec ? 1 : 0 + + metadata { + name = "modsecurity-nginx-config" + namespace = "ingress-controllers" + labels = { + "k8s-app" = var.controller_name + } + } + data = { + "modsecurity.conf" = file("${path.module}/templates/modsecurity.conf"), + } + + depends_on = [ + kubernetes_namespace.ingress_controllers, + ] + + lifecycle { + ignore_changes = [metadata[0].annotations] + } +} + diff --git a/cron.tf b/cron.tf new file mode 100644 index 0000000..ff9da7e --- /dev/null +++ b/cron.tf @@ -0,0 +1,69 @@ +resource "kubernetes_service_account_v1" "restart_modsec_containers" { + metadata { + name = "restart-modsec-containers" + namespace = "ingress-controllers" + } +} + +resource "kubernetes_role_v1" "restart_modsec_containers" { + metadata { + name = "restart-modsec-containers" + namespace = "ingress-controllers" + } + + rule { + api_groups = ["apps", "applications"] + resources = ["deployments"] + verbs = ["get", "list", "patch"] + } +} + +resource "kubernetes_role_binding_v1" "restart_modsec_containers" { + metadata { + name = "restart-modsec-containers" + namespace = "ingress-controllers" + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = "restart-modsec-containers" + } + subject { + kind = "ServiceAccount" + name = "restart-modsec-containers" + namespace = "ingress-controllers" + } +} + +resource "kubernetes_cron_job_v1" "restart_modsec_containers" { + metadata { + name = "restart-modsec-containers-nightly" + namespace = "ingress-controllers" + } + spec { + concurrency_policy = "Forbid" + failed_jobs_history_limit = 2 + schedule = "00 23 * * *" + starting_deadline_seconds = 10 + successful_jobs_history_limit = 0 + job_template { + metadata {} + spec { + backoff_limit = 2 + active_deadline_seconds = 600 + ttl_seconds_after_finished = 10 + template { + metadata {} + spec { + service_account_name = "restart-modsec-containers" + container { + name = "kubectl" + image = "bitnami/kubectl" + command = ["kubectl", "rollout", "restart", "deployment/nginx-ingress-modsec-controller"] + } + } + } + } + } + } +} diff --git a/main.tf b/main.tf index 47f8e6c..f6390b5 100644 --- a/main.tf +++ b/main.tf @@ -59,6 +59,7 @@ resource "helm_release" "nginx_ingress" { enable_external_dns_annotation = var.enable_external_dns_annotation backend_repo = var.backend_repo backend_tag = var.backend_tag + fluent_bit_version = var.fluent_bit_version })] depends_on = [ @@ -98,26 +99,3 @@ resource "kubectl_manifest" "nginx_ingress_default_certificate" { var.dependence_certmanager ] } - -resource "kubernetes_config_map" "modsecurity_nginx_config" { - count = var.enable_modsec ? 1 : 0 - - metadata { - name = "modsecurity-nginx-config" - namespace = "ingress-controllers" - labels = { - "k8s-app" = var.controller_name - } - } - data = { - "modsecurity.conf" = file("${path.module}/templates/modsecurity.conf"), - } - - depends_on = [ - kubernetes_namespace.ingress_controllers, - ] - - lifecycle { - ignore_changes = [metadata[0].annotations] - } -} diff --git a/templates/modsecurity.conf b/templates/modsecurity.conf index 45e3084..95eb15b 100644 --- a/templates/modsecurity.conf +++ b/templates/modsecurity.conf @@ -233,8 +233,15 @@ SecAuditLogParts AEFHKZ # Use a single file for logging. This is much easier to look at, but # assumes that you will use the audit log only ocassionally. # -SecAuditLogType Serial -SecAuditLog /dev/stdout +SecAuditLogType Concurrent + +SecAuditLogDirMode 0777 +SecAuditLogFileMode 0777 + +SecAuditLog /var/log/audit/index.log + +SecAuditLog2 /var/log/audit/index2.log +SecAuditLogStorageDir /var/log/audit/ SecAuditLogFormat JSON SecRuleRemoveById 920350 @@ -268,7 +275,5 @@ SecUnicodeMapFile unicode.mapping 20127 # The following information will be shared: ModSecurity version, # Web Server version, APR version, PCRE version, Lua version, Libxml2 # version, Anonymous unique id for host. -SecStatusEngine On - -SecAuditLogStorageDir /var/log/audit/ +SecStatusEngine Off diff --git a/templates/values.yaml.tpl b/templates/values.yaml.tpl index eda0392..fa3b276 100644 --- a/templates/values.yaml.tpl +++ b/templates/values.yaml.tpl @@ -5,18 +5,29 @@ controller: replicaCount: ${replica_count} %{ if enable_modsec ~} + extraVolumes: + ## Additional volumes to the controller pod. + - name: logs-volume + emptyDir: {} + - name: modsecurity-nginx-config + configMap: + name: modsecurity-nginx-config + - name: fluent-bit-config + configMap: + name: fluent-bit-config + - name: fluent-bit-luascripts + configMap: + name: fluent-bit-luascripts + + extraVolumeMounts: ## Additional volumeMounts to the controller main container. + - name: logs-volume + mountPath: /var/log/audit/ - name: modsecurity-nginx-config mountPath: /etc/nginx/modsecurity/modsecurity.conf subPath: modsecurity.conf readOnly: true - - extraVolumes: - ## Additional volumes to the controller pod. - - name: modsecurity-nginx-config - configMap: - name: modsecurity-nginx-config %{ endif ~} updateStrategy: @@ -26,6 +37,27 @@ controller: minReadySeconds: 12 +%{ if enable_modsec ~} + extraInitContainers: + - name: init-file-permissions + image: busybox + command: ["sh", "-c", "chmod -R 777 /var/log/audit"] + volumeMounts: + - name: logs-volume + mountPath: /var/log/audit + + extraContainers: + - name: flb-modsec-audit-logs + image: fluent/fluent-bit:${fluent_bit_version} + volumeMounts: + - name: fluent-bit-config + mountPath: /fluent-bit/etc/ + - name: fluent-bit-luascripts + mountPath: /fluent-bit/scripts/ + - name: logs-volume + mountPath: /var/log/audit/ +%{ endif ~} + # -- Process Ingress objects without ingressClass annotation/ingressClassName field # Overrides value for --watch-ingress-without-class flag of the controller binary # Defaults to false diff --git a/variables.tf b/variables.tf index 30940d2..0028b83 100644 --- a/variables.tf +++ b/variables.tf @@ -70,4 +70,22 @@ variable "enable_external_dns_annotation" { variable "dependence_certmanager" { description = "cert-manager module dependences in order to be executed." -} \ No newline at end of file +} + +variable "cluster" { + description = " cluster name used for opensearch indicies" + type = string + default = "" +} + +variable "opensearch_modsec_audit_host" { + description = "domain endpoint for the opensearch cluster" + type = string + default = "" +} + +variable "fluent_bit_version" { + description = "fluent bit container version used to exrtact modsec audit logs" + type = string + default = "2.1.8-amd64" +}