From e6cd59617dcb7a676cd12f21e376537a6023c223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Lindesk=C3=A4r?= Date: Thu, 29 Apr 2021 13:33:20 +0200 Subject: [PATCH] add module files --- dataflow-sink-filter-catchall.tpl | 1 + dataflow-sink-filter.tpl | 1 + dataflow-udf-js.tpl | 8 ++ main.tf | 119 ++++++++++++++++++++++++++++++ outputs.tf | 0 variables.tf | 58 +++++++++++++++ 6 files changed, 187 insertions(+) create mode 100644 dataflow-sink-filter-catchall.tpl create mode 100644 dataflow-sink-filter.tpl create mode 100644 dataflow-udf-js.tpl create mode 100644 main.tf create mode 100644 outputs.tf create mode 100644 variables.tf diff --git a/dataflow-sink-filter-catchall.tpl b/dataflow-sink-filter-catchall.tpl new file mode 100644 index 0000000..4a110f8 --- /dev/null +++ b/dataflow-sink-filter-catchall.tpl @@ -0,0 +1 @@ +logName:"organizations/" OR logName:"folders/" OR logName:"projects/" NOT (resource.type = "dataflow_step" AND resource.labels.job_name = "${dataflow_name}") diff --git a/dataflow-sink-filter.tpl b/dataflow-sink-filter.tpl new file mode 100644 index 0000000..a79304d --- /dev/null +++ b/dataflow-sink-filter.tpl @@ -0,0 +1 @@ +logName:"organizations/${org_id}/logs/" OR logName:"projects/${log_project}/logs/" %{ for e_proj in extra_projects ~} OR logName:"projects/${e_proj}/logs/" %{ endfor ~} NOT (resource.type = "dataflow_step" AND resource.labels.job_name = "${dataflow_name}") diff --git a/dataflow-udf-js.tpl b/dataflow-udf-js.tpl new file mode 100644 index 0000000..9ed819a --- /dev/null +++ b/dataflow-udf-js.tpl @@ -0,0 +1,8 @@ +function transform(inJson) { + var obj = { "data" : JSON.parse(inJson) }; + obj._metadata = { + source: obj.data.logName || "default", + sourcetype: "google:gcp:pubsub:message" + }; + return JSON.stringify(obj); +} diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..fb55676 --- /dev/null +++ b/main.tf @@ -0,0 +1,119 @@ + +# Pub/Sub topics + +resource "google_pubsub_topic" "log-topic" { + name = "${var.dataflow_base_name}-log-topic" + project = var.gcp_log_project +} + +resource "google_pubsub_topic" "log-dl-topic" { + name = "${var.dataflow_base_name}-log-dl-topic" + project = var.gcp_log_project +} + + +# Pub/Sub subscriptions + +resource "google_pubsub_subscription" "log-sub" { + name = "${var.dataflow_base_name}-log-sub" + project = var.gcp_log_project + topic = google_pubsub_topic.log-topic.name + + message_retention_duration = "1200s" +} + +resource "google_pubsub_subscription" "log-dl-sub" { + name = "${var.dataflow_base_name}-log-dl-sub" + project = var.gcp_log_project + topic = google_pubsub_topic.log-dl-topic.name + + message_retention_duration = "1200s" +} + + +# Log sinks + +resource "google_logging_organization_sink" "log-org-sink" { + name = "${var.dataflow_base_name}-log-org-sink" + count = var.log_sink_org_id == "" ? 0 : 1 + + org_id = var.log_sink_org_id + include_children = true + filter = templatefile("${path.module}/dataflow-sink-filter-catchall.tpl", { + org_id = var.log_sink_org_id, + log_project = var.gcp_log_project, + extra_projects = var.log_sink_org_filter_projects, + dataflow_name = var.dataflow_enable == 1 ? google_dataflow_job.splunk-job[0].name : "" + }) + + destination = "pubsub.googleapis.com/${google_pubsub_topic.log-topic.id}" +} + +## Add project log sink ? + + +# Pub/Sub topic IAM policy + +resource "google_pubsub_topic_iam_member" "log-topic-sink-member" { + project = google_pubsub_topic.log-topic.project + topic = google_pubsub_topic.log-topic.name + role = "roles/pubsub.publisher" + member = google_logging_organization_sink.log-org-sink.0.writer_identity +} + + +# Bucket for temp storage + +resource "google_storage_bucket" "log-bucket" { + name = "${var.dataflow_base_name}-log-bucket" + project = var.gcp_log_project + location = var.gcp_region + force_destroy = true + uniform_bucket_level_access = true +} + +resource "google_storage_bucket_object" "temp" { + name = "${var.dataflow_base_name}/temp/.ignore" + content = "IGNORE" + bucket = google_storage_bucket.log-bucket.name + depends_on = [google_storage_bucket.log-bucket] +} + +resource "google_storage_bucket_object" "splunk-udf" { + name = "${var.dataflow_base_name}/js/splunk-udf.js" + content = templatefile("${path.module}/dataflow-udf-js.tpl", { + input_sub = google_pubsub_subscription.log-sub.name + }) + bucket = google_storage_bucket.log-bucket.name + depends_on = [google_storage_bucket.log-bucket] +} + + +# Dataflow +resource "google_dataflow_job" "splunk-job" { + name = "${var.dataflow_base_name}-splunk-job" + count = var.dataflow_enable == 1 ? 1 : 0 + project = var.gcp_log_project + region = var.gcp_region + zone = var.gcp_zone + machine_type = var.dataflow_worker_machine_type + max_workers = var.dataflow_max_workers + + template_gcs_path = "gs://dataflow-templates-${var.gcp_region}/latest/Cloud_PubSub_to_Splunk" + + temp_gcs_location = "${google_storage_bucket.log-bucket.url}/${dirname(google_storage_bucket_object.temp.name)}" + + parameters = { + inputSubscription = google_pubsub_subscription.log-sub.id + outputDeadletterTopic = google_pubsub_topic.log-dl-topic.id + url = var.splunk_hec_url + token = var.splunk_hec_token + javascriptTextTransformGcsPath = "${google_storage_bucket.log-bucket.url}/${google_storage_bucket_object.splunk-udf.name}" + javascriptTextTransformFunctionName = "transform" + disableCertificateValidation = var.splunk_hec_disable_cert_validation + batchCount = var.dataflow_batchCount, + parallelism = var.dataflow_parallelism + } + + on_delete = "cancel" +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..803feb2 --- /dev/null +++ b/variables.tf @@ -0,0 +1,58 @@ +variable "gcp_region" { + description = "" +} +variable "gcp_zone" { + description = "" +} +variable "gcp_log_project" { + description = "" +} +variable "log_sink_org_id" { + description = "" + type = string + default = "" +} +variable "log_sink_org_filter_projects" { + description = "" + type = list(string) + default = [] +} +variable "dataflow_base_name" { + description = "" +} +variable "dataflow_max_workers" { + description = "" + type = number + default = null +} +variable "dataflow_enable" { + description = "" + type = number + default = 1 +} +variable "dataflow_worker_machine_type" { + description = "" + type = string + default = null +} +variable "splunk_hec_url" { + description = "" +} +variable "splunk_hec_token" { + description = "" +} +variable "splunk_hec_disable_cert_validation" { + description = "" + type = string + default = "false" +} +variable "dataflow_batchCount" { + description = "" + type = number + default = 10 +} +variable "dataflow_parallelism" { + description = "" + type = number + default = 4 +}