From 2cedf442b514fa935b134e75830cec3369c1e570 Mon Sep 17 00:00:00 2001 From: elasticmachine Date: Thu, 19 Dec 2024 13:49:37 +0000 Subject: [PATCH] [automation] Publish kubernetes templates for elastic-agent --- .../templates.d/apache.yml | 100 +- .../templates.d/cef.yml | 42 +- .../templates.d/fireeye.yml | 58 +- .../templates.d/haproxy.yml | 56 +- .../templates.d/iis.yml | 54 +- .../templates.d/logstash.yml | 120 +- .../templates.d/mimecast.yml | 2082 ++++++++--------- .../templates.d/mongodb.yml | 4 +- .../templates.d/mysql.yml | 78 +- .../templates.d/nats.yml | 54 +- .../templates.d/panw.yml | 74 +- .../templates.d/pfsense.yml | 40 +- .../templates.d/postgresql.yml | 62 +- .../templates.d/prometheus.yml | 4 +- .../templates.d/redis.yml | 58 +- .../templates.d/sentinel_one.yml | 44 +- .../templates.d/synthetics.yml | 102 +- .../templates.d/udp.yml | 26 +- 18 files changed, 1529 insertions(+), 1529 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml index 3520dca77fc..9375bfd7b83 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/apache.yml @@ -1,4 +1,54 @@ inputs: + - name: filestream-apache + id: filestream-apache-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.apache.access.enabled} == true or ${kubernetes.hints.apache.enabled} == true + data_stream: + dataset: apache.access + type: logs + exclude_files: + - .gz$ + file_identity: + fingerprint: null + parsers: + - container: + format: auto + stream: ${kubernetes.hints.apache.access.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + fingerprint: + enabled: true + symlinks: true + tags: + - apache-access + - condition: ${kubernetes.hints.apache.error.enabled} == true or ${kubernetes.hints.apache.enabled} == true + data_stream: + dataset: apache.error + type: logs + exclude_files: + - .gz$ + file_identity: + fingerprint: null + parsers: + - container: + format: auto + stream: ${kubernetes.hints.apache.error.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale: null + prospector: + scanner: + fingerprint: + enabled: true + symlinks: true + tags: + - apache-error + data_stream.namespace: default - name: httpjson-apache id: httpjson-apache-${kubernetes.hints.container_id} type: httpjson @@ -97,53 +147,3 @@ inputs: period: ${kubernetes.hints.apache.status.period|kubernetes.hints.apache.period|'30s'} server_status_path: /server-status data_stream.namespace: default - - name: filestream-apache - id: filestream-apache-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.apache.access.enabled} == true or ${kubernetes.hints.apache.enabled} == true - data_stream: - dataset: apache.access - type: logs - exclude_files: - - .gz$ - file_identity: - fingerprint: null - parsers: - - container: - format: auto - stream: ${kubernetes.hints.apache.access.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - fingerprint: - enabled: true - symlinks: true - tags: - - apache-access - - condition: ${kubernetes.hints.apache.error.enabled} == true or ${kubernetes.hints.apache.enabled} == true - data_stream: - dataset: apache.error - type: logs - exclude_files: - - .gz$ - file_identity: - fingerprint: null - parsers: - - container: - format: auto - stream: ${kubernetes.hints.apache.error.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale: null - prospector: - scanner: - fingerprint: - enabled: true - symlinks: true - tags: - - apache-error - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml index e4c87ed361e..dffd870c717 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/cef.yml @@ -1,25 +1,4 @@ inputs: - - name: tcp-cef - id: tcp-cef-${kubernetes.hints.container_id} - type: tcp - use_output: default - streams: - - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true - data_stream: - dataset: cef.log - type: logs - host: localhost:9004 - processors: - - rename: - fields: - - from: message - to: event.original - - decode_cef: - field: event.original - tags: - - cef - - forwarded - data_stream.namespace: default - name: filestream-cef id: filestream-cef-${kubernetes.hints.container_id} type: filestream @@ -76,3 +55,24 @@ inputs: - cef - forwarded data_stream.namespace: default + - name: tcp-cef + id: tcp-cef-${kubernetes.hints.container_id} + type: tcp + use_output: default + streams: + - condition: ${kubernetes.hints.cef.log.enabled} == true or ${kubernetes.hints.cef.enabled} == true + data_stream: + dataset: cef.log + type: logs + host: localhost:9004 + processors: + - rename: + fields: + - from: message + to: event.original + - decode_cef: + field: event.original + tags: + - cef + - forwarded + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/fireeye.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/fireeye.yml index 5ce213a2ff1..0fe5a42c2a3 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/fireeye.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/fireeye.yml @@ -1,33 +1,4 @@ inputs: - - name: filestream-fireeye - id: filestream-fireeye-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.fireeye.nx.enabled} == true or ${kubernetes.hints.fireeye.enabled} == true - data_stream: - dataset: fireeye.nx - type: logs - exclude_files: - - .gz$ - file_identity: - fingerprint: null - parsers: - - container: - format: auto - stream: ${kubernetes.hints.fireeye.nx.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale: null - prospector: - scanner: - fingerprint: - enabled: true - symlinks: true - tags: - - fireeye-nx - data_stream.namespace: default - name: udp-fireeye id: udp-fireeye-${kubernetes.hints.container_id} type: udp @@ -64,3 +35,32 @@ inputs: - forwarded tcp: null data_stream.namespace: default + - name: filestream-fireeye + id: filestream-fireeye-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.fireeye.nx.enabled} == true or ${kubernetes.hints.fireeye.enabled} == true + data_stream: + dataset: fireeye.nx + type: logs + exclude_files: + - .gz$ + file_identity: + fingerprint: null + parsers: + - container: + format: auto + stream: ${kubernetes.hints.fireeye.nx.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale: null + prospector: + scanner: + fingerprint: + enabled: true + symlinks: true + tags: + - fireeye-nx + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/haproxy.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/haproxy.yml index be3b22b57ac..3db3bea5b96 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/haproxy.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/haproxy.yml @@ -1,4 +1,32 @@ inputs: + - name: haproxy/metrics-haproxy + id: haproxy/metrics-haproxy-${kubernetes.hints.container_id} + type: haproxy/metrics + use_output: default + streams: + - condition: ${kubernetes.hints.haproxy.info.enabled} == true or ${kubernetes.hints.haproxy.enabled} == true + data_stream: + dataset: haproxy.info + type: metrics + hosts: + - ${kubernetes.hints.haproxy.info.host|kubernetes.hints.haproxy.host|'tcp://127.0.0.1:14567'} + metricsets: + - info + password: ${kubernetes.hints.haproxy.info.password|kubernetes.hints.haproxy.password|'admin'} + period: ${kubernetes.hints.haproxy.info.period|kubernetes.hints.haproxy.period|'10s'} + username: ${kubernetes.hints.haproxy.info.username|kubernetes.hints.haproxy.username|'admin'} + - condition: ${kubernetes.hints.haproxy.stat.enabled} == true or ${kubernetes.hints.haproxy.enabled} == true + data_stream: + dataset: haproxy.stat + type: metrics + hosts: + - ${kubernetes.hints.haproxy.stat.host|kubernetes.hints.haproxy.host|'tcp://127.0.0.1:14567'} + metricsets: + - stat + password: ${kubernetes.hints.haproxy.stat.password|kubernetes.hints.haproxy.password|'admin'} + period: ${kubernetes.hints.haproxy.stat.period|kubernetes.hints.haproxy.period|'10s'} + username: ${kubernetes.hints.haproxy.stat.username|kubernetes.hints.haproxy.username|'admin'} + data_stream.namespace: default - name: filestream-haproxy id: filestream-haproxy-${kubernetes.hints.container_id} type: filestream @@ -45,31 +73,3 @@ inputs: - forwarded - haproxy-log data_stream.namespace: default - - name: haproxy/metrics-haproxy - id: haproxy/metrics-haproxy-${kubernetes.hints.container_id} - type: haproxy/metrics - use_output: default - streams: - - condition: ${kubernetes.hints.haproxy.info.enabled} == true or ${kubernetes.hints.haproxy.enabled} == true - data_stream: - dataset: haproxy.info - type: metrics - hosts: - - ${kubernetes.hints.haproxy.info.host|kubernetes.hints.haproxy.host|'tcp://127.0.0.1:14567'} - metricsets: - - info - password: ${kubernetes.hints.haproxy.info.password|kubernetes.hints.haproxy.password|'admin'} - period: ${kubernetes.hints.haproxy.info.period|kubernetes.hints.haproxy.period|'10s'} - username: ${kubernetes.hints.haproxy.info.username|kubernetes.hints.haproxy.username|'admin'} - - condition: ${kubernetes.hints.haproxy.stat.enabled} == true or ${kubernetes.hints.haproxy.enabled} == true - data_stream: - dataset: haproxy.stat - type: metrics - hosts: - - ${kubernetes.hints.haproxy.stat.host|kubernetes.hints.haproxy.host|'tcp://127.0.0.1:14567'} - metricsets: - - stat - password: ${kubernetes.hints.haproxy.stat.password|kubernetes.hints.haproxy.password|'admin'} - period: ${kubernetes.hints.haproxy.stat.period|kubernetes.hints.haproxy.period|'10s'} - username: ${kubernetes.hints.haproxy.stat.username|kubernetes.hints.haproxy.username|'admin'} - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml index 7708e49d3e5..24f7dee6c94 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/iis.yml @@ -1,31 +1,4 @@ inputs: - - name: iis/metrics-iis - id: iis/metrics-iis-${kubernetes.hints.container_id} - type: iis/metrics - use_output: default - streams: - - condition: ${kubernetes.hints.iis.application_pool.enabled} == true or ${kubernetes.hints.iis.enabled} == true - data_stream: - dataset: iis.application_pool - type: metrics - metricsets: - - application_pool - period: ${kubernetes.hints.iis.application_pool.period|kubernetes.hints.iis.period|'10s'} - - condition: ${kubernetes.hints.iis.webserver.enabled} == true or ${kubernetes.hints.iis.enabled} == true - data_stream: - dataset: iis.webserver - type: metrics - metricsets: - - webserver - period: ${kubernetes.hints.iis.webserver.period|kubernetes.hints.iis.period|'10s'} - - condition: ${kubernetes.hints.iis.website.enabled} == true or ${kubernetes.hints.iis.enabled} == true - data_stream: - dataset: iis.website - type: metrics - metricsets: - - website - period: ${kubernetes.hints.iis.website.period|kubernetes.hints.iis.period|'10s'} - data_stream.namespace: default - name: filestream-iis id: filestream-iis-${kubernetes.hints.container_id} type: filestream @@ -80,3 +53,30 @@ inputs: tags: - iis-error data_stream.namespace: default + - name: iis/metrics-iis + id: iis/metrics-iis-${kubernetes.hints.container_id} + type: iis/metrics + use_output: default + streams: + - condition: ${kubernetes.hints.iis.application_pool.enabled} == true or ${kubernetes.hints.iis.enabled} == true + data_stream: + dataset: iis.application_pool + type: metrics + metricsets: + - application_pool + period: ${kubernetes.hints.iis.application_pool.period|kubernetes.hints.iis.period|'10s'} + - condition: ${kubernetes.hints.iis.webserver.enabled} == true or ${kubernetes.hints.iis.enabled} == true + data_stream: + dataset: iis.webserver + type: metrics + metricsets: + - webserver + period: ${kubernetes.hints.iis.webserver.period|kubernetes.hints.iis.period|'10s'} + - condition: ${kubernetes.hints.iis.website.enabled} == true or ${kubernetes.hints.iis.enabled} == true + data_stream: + dataset: iis.website + type: metrics + metricsets: + - website + period: ${kubernetes.hints.iis.website.period|kubernetes.hints.iis.period|'10s'} + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml index 7b889c42cf4..2b74d004657 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/logstash.yml @@ -1,64 +1,4 @@ inputs: - - name: filestream-logstash - id: filestream-logstash-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.logstash.log.enabled} == true or ${kubernetes.hints.logstash.enabled} == true - data_stream: - dataset: logstash.log - type: logs - exclude_files: - - .gz$ - file_identity: - fingerprint: null - multiline: - match: after - negate: true - pattern: ^((\[[0-9]{4}-[0-9]{2}-[0-9]{2}[^\]]+\])|({.+})) - parsers: - - container: - format: auto - stream: ${kubernetes.hints.logstash.log.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale.when.not.regexp.message: ^{ - - add_fields: - fields: - ecs.version: 1.10.0 - target: "" - prospector: - scanner: - fingerprint: - enabled: true - symlinks: true - - condition: ${kubernetes.hints.logstash.slowlog.enabled} == true or ${kubernetes.hints.logstash.enabled} == true - data_stream: - dataset: logstash.slowlog - type: logs - exclude_files: - - .gz$ - file_identity: - fingerprint: null - parsers: - - container: - format: auto - stream: ${kubernetes.hints.logstash.slowlog.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - processors: - - add_locale.when.not.regexp.message: ^{ - - add_fields: - fields: - ecs.version: 1.10.0 - target: "" - prospector: - scanner: - fingerprint: - enabled: true - symlinks: true - data_stream.namespace: default - name: logstash/metrics-logstash id: logstash/metrics-logstash-${kubernetes.hints.container_id} type: logstash/metrics @@ -381,3 +321,63 @@ inputs: fields: null resource.url: http://localhost:9600/_node data_stream.namespace: default + - name: filestream-logstash + id: filestream-logstash-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.logstash.log.enabled} == true or ${kubernetes.hints.logstash.enabled} == true + data_stream: + dataset: logstash.log + type: logs + exclude_files: + - .gz$ + file_identity: + fingerprint: null + multiline: + match: after + negate: true + pattern: ^((\[[0-9]{4}-[0-9]{2}-[0-9]{2}[^\]]+\])|({.+})) + parsers: + - container: + format: auto + stream: ${kubernetes.hints.logstash.log.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale.when.not.regexp.message: ^{ + - add_fields: + fields: + ecs.version: 1.10.0 + target: "" + prospector: + scanner: + fingerprint: + enabled: true + symlinks: true + - condition: ${kubernetes.hints.logstash.slowlog.enabled} == true or ${kubernetes.hints.logstash.enabled} == true + data_stream: + dataset: logstash.slowlog + type: logs + exclude_files: + - .gz$ + file_identity: + fingerprint: null + parsers: + - container: + format: auto + stream: ${kubernetes.hints.logstash.slowlog.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + processors: + - add_locale.when.not.regexp.message: ^{ + - add_fields: + fields: + ecs.version: 1.10.0 + target: "" + prospector: + scanner: + fingerprint: + enabled: true + symlinks: true + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mimecast.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mimecast.yml index 1e029fbba34..2283e22ec2b 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/mimecast.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/mimecast.yml @@ -1,617 +1,429 @@ inputs: - - name: cel-mimecast - id: cel-mimecast-${kubernetes.hints.container_id} - type: cel + - name: httpjson-mimecast + id: httpjson-mimecast-${kubernetes.hints.container_id} + type: httpjson use_output: default streams: - - condition: ${kubernetes.hints.mimecast.archive_search_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 + - condition: ${kubernetes.hints.mimecast.archive_search_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_date: + value: '[[.first_event.createTime]]' data_stream: dataset: mimecast.archive_search_logs type: logs - fields_under_root: true interval: 5m - keep_null: true - program: | - // This program is shared amongst archive_search_logs, dlp_logs, - // message_release_logs, ttp_ap_logs, ttp_ip_logs, and ttp_url_logs. - // If it is changed here changes should be reflected in the other - // data streams. Do not differentiate the logic between these data - // streams lightly; use the state variable for this unless absolutely - // required. - state.with( - ( - (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? - // The token we have is still valid. - state.token - : - // Get a new token. - post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", - { - "client_id": [state.client_id], - "client_secret": [state.client_secret], - "grant_type": ["client_credentials"], - }.format_query() - ).do_request().as(auth, auth.StatusCode == 200 ? - bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ - // Include 60s grace period to avoid attempting to make - // a request with a stale authentication token. - "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), - })) - : - { - "events": { - "error": { - "code": string(auth.StatusCode), - "id": string(auth.Status), - "message": "POST /oauth/token: "+( - size(auth.Body) != 0 ? - string(auth.Body) - : - string(auth.Status) + ' (' + string(auth.StatusCode) + ')' - ), - }, - }, - "want_more": false, - } - ) - ).as(token, !has(token.access_token) ? token : - { - "data": state.?last_page.data.orValue([{ - state.start_field: state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339)), - state.end_field: now.format(time_layout.RFC3339), - }]), - }.as(req, - post_request(state.url.trim_right("/") + state.path, "application/json", - { - "meta": { - "pagination": { - "pageSize": state.page_size, - ?"pageToken": state.?last_page.next, - } - }, - "data": req.data, - }.encode_json() - ).with({ - "Header": { - "Authorization": ["Bearer " + token.access_token], - "Accept": ["application/json"], - "Content-Type": ["application/json"], - } - }).do_request().as(resp, resp.StatusCode == 200 ? - bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ? - { - "events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(), - "cursor": { - "last": ([now] + body.data.map(e, - e[state.data_path].map(l, - l[state.time_field].parse_time(["2006-01-02T15:04:05-0700", time_layout.RFC3339]) - ) - ).flatten()).max().format(time_layout.RFC3339), - }, - ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ? - optional.of({ - ?"next": body.?meta.pagination.next, - "data": req.data, - }) - : - optional.none(), - "token": { - "access_token": token.access_token, - "expires": token.expires, - }, - "want_more": has(body.?meta.pagination.next) && size(body.data) != 0, - } - : - // Mimecast can return failure states with a 200. This - // is detected by a non-empty fail array at the root - // of the response body. Don't attempt to parse this - // out, just dump the whole body into the error message. - { - "events": { - "error": { - "code": string(resp.StatusCode), - "id": string(resp.Status), - "message": "POST " + state.path + ":" + string(resp.Body), // We know this is not empty. - }, - }, - "want_more": false, - } - ) - : - { - "events": { - "error": { - "code": string(resp.StatusCode), - "id": string(resp.Status), - "message": "POST " + state.path + ": " + ( - size(resp.Body) != 0 ? - string(resp.Body) - : - string(resp.Status) + ' (' + string(resp.StatusCode) + ')' - ), - }, - }, - "want_more": false, - } - ) - ) - ) - ) - redact: - fields: - - client_id - - client_secret - - token.access_token - resource.url: https://api.services.mimecast.com - state: - client_id: null - client_secret: null - data_path: logs - end_field: end - look_back: 24h - page_size: 100 - path: /api/archive/get-archive-search-logs - start_field: start - time_field: createTime + request.method: POST + request.transforms: + - set: + target: body.meta.pagination.pageSize + value: 100 + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/archive/get-archive-search-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + request.url: https://eu-api.mimecast.com/api/archive/get-archive-search-logs + response.decode_as: application/json + response.pagination: + - delete: + target: body.data + - set: + fail_on_template_error: true + target: body.meta.pagination.pageToken + value: |- + [[- if index .last_response.body.meta.pagination "next" -]] + [[- .last_response.body.meta.pagination.next -]] + [[- end -]] + response.split: + ignore_empty_value: true + split: + ignore_empty_value: true + keep_parent: false + target: body.logs + target: body.data tags: - forwarded - mimecast-archive-search-logs - - condition: ${kubernetes.hints.mimecast.audit_events.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 + - condition: ${kubernetes.hints.mimecast.audit_events.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_date: + value: '[[.first_event.eventTime]]' data_stream: dataset: mimecast.audit_events type: logs - fields_under_root: true - interval: 5m - keep_null: true - program: "state.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token:\"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n {\n \"data\": state.?last_page.data.orValue([{\n state.start_field: state.?cursor.last.orValue(now - duration(state.look_back)).format(time_layout.RFC3339),\n state.end_field: now.format(time_layout.RFC3339),\n }]),\n }.as(req,\n post_request(state.url.trim_right(\"/\") + state.path, \"application/json\", \n {\n \"meta\": {\n \"pagination\": {\n \"pageSize\": state.page_size,\n ?\"pageToken\": state.?last_page.next,\n }\n },\n \"data\": req.data,\n }.encode_json()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?\n {\n \"events\": body.data.map(e, {\"message\": e.encode_json()}),\n \"cursor\": {\n \"last\": ([now] + body.data.map(e, e[state.time_field].parse_time([\"2006-01-02T15:04:05-0700\", time_layout.RFC3339]))).max(),\n },\n ?\"last_page\": has(body.?meta.pagination.next) && size(body.data) != 0 ?\n optional.of({\n ?\"next\": body.?meta.pagination.next,\n \"data\": req.data,\n })\n :\n optional.none(),\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": has(body.?meta.pagination.next) && size(body.data) != 0,\n }\n :\n // Mimecast can return failure states with a 200. This\n // is detected by a non-empty fail array at the root\n // of the response body. Don't attempt to parse this\n // out, just dump the whole body into the error message.\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \":\" + string(resp.Body), // We know this is not empty.\n },\n },\n \"want_more\": false,\n }\n )\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \":\" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" - redact: - fields: - - client_id - - client_secret - - token.access_token - resource.url: https://api.services.mimecast.com - state: - client_id: null - client_secret: null - end_field: endDateTime - look_back: 24h - page_size: 100 - path: /api/audit/get-audit-events - start_field: startDateTime - time_field: eventTime - tags: - - forwarded - - mimecast-audit-events - - condition: ${kubernetes.hints.mimecast.cloud_integrated_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 - data_stream: - dataset: mimecast.cloud_integrated_logs - type: logs - fields_under_root: true interval: 5m - keep_null: true - program: "// This program is shared between cloud_integrated_logs and siem_logs\n// If it is changed here changes should be reflected in the other data\n// streams. Do not differentiate the logic between these data streams\n// lightly; use the state variable for this unless absolutely required.\nstate.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token: \"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n state.?cursor.work_list.orValue(state.types.map(t, {\"type\": t})).as(work_list, size(work_list) == 0 ?\n state.types.map(t, {\"type\": t})\n :\n work_list\n ).as(work_list,\n get_request(\n state.url.trim_right(\"/\") + state.path + \"?\" + {\n \"type\": [work_list[0].type],\n ?\"nextPage\": work_list[0].?next.optMap(next, [next]),\n ?\"dateRangeStartsAt\": state.?start.optMap(start, [start.format(\"2006-01-02\")]),\n ?\"dateRangeEndsAt\": state.?end.optMap(end, [end.format(\"2006-01-02\")]),\n ?\"pageSize\": state.?page_size.optMap(size, [string(int(size))]),\n }.format_query()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body,\n {\n \"events\": body.value.map(b, has(b.url),\n get(b.url).as(batch, batch.StatusCode == 200 ?\n bytes(batch.Body).mime(\"application/gzip\").mime(\"application/x-ndjson\").map(e,\n {\n \"message\": dyn(e.encode_json()),\n }\n )\n :\n [{\n \"error\": {\n \"code\": string(batch.StatusCode),\n \"id\": string(batch.Status),\n \"message\": \"GET \" + b.url + \": \" + (\n size(batch.Body) != 0 ?\n string(batch.Body)\n :\n string(batch.Status) + ' (' + string(batch.StatusCode) + ')'\n ),\n },\n }]\n )\n ).flatten(),\n \"cursor\": {\n \"work_list\": (\n \"@nextPage\" in body && size(body.value) != 0 ?\n [work_list[0].with({\"next\": body[\"@nextPage\"]})]\n :\n []\n ) + tail(work_list),\n },\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": \"@nextPage\" in body && size(body.value) != 0,\n }.as(to_publish, to_publish.with({\n \"want_more\": to_publish.want_more || size(to_publish.cursor.work_list) != 0,\n }))\n ).as(state, \n // Check whether we still need to get more, but have\n // no event for this type. If we do, populate events\n // with a place-holder to be discarded by the ingest\n // pipeline.\n state.want_more && size(state.events) == 0 ?\n state.with({\"events\": [{\"message\": \"want_more\"}]})\n :\n state\n )\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"GET \" + state.path + \": \" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" - redact: - fields: - - client_id - - client_secret - - token.access_token - resource.url: https://api.services.mimecast.com - state: - client_id: null - client_secret: null - end_field: dateRangeEndsAt - look_back: 24h - page_size: null - path: /siem/v1/batch/events/ci - start_field: dateRangeStartsAt - types: - - entities - - mailflow - - urlclick - tags: - - forwarded - - mimecast-cloud-integrated-logs - - condition: ${kubernetes.hints.mimecast.dlp_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 - data_stream: - dataset: mimecast.dlp_logs + request.method: POST + request.transforms: + - set: + target: body.meta.pagination.pageSize + value: 100 + - set: + default: '[{"endDateTime": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "startDateTime":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' + target: body.data + value: '[{"endDateTime": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "startDateTime":"[[.cursor.next_date]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/audit/get-audit-events:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + request.url: https://eu-api.mimecast.com/api/audit/get-audit-events + response.decode_as: application/json + response.pagination: + - delete: + target: body.data + - set: + fail_on_template_error: true + target: body.meta.pagination.pageToken + value: |- + [[- if index .last_response.body.meta.pagination "next" -]] + [[- .last_response.body.meta.pagination.next -]] + [[- end -]] + response.split: + ignore_empty_value: true + target: body.data + tags: + - forwarded + - mimecast-audit-events + - condition: ${kubernetes.hints.mimecast.dlp_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_date: + value: '[[.first_event.eventTime]]' + data_stream: + dataset: mimecast.dlp_logs type: logs - fields_under_root: true interval: 5m - keep_null: true - program: | - // This program is shared amongst archive_search_logs, dlp_logs, - // message_release_logs, ttp_ap_logs, ttp_ip_logs, and ttp_url_logs. - // If it is changed here changes should be reflected in the other - // data streams. Do not differentiate the logic between these data - // streams lightly; use the state variable for this unless absolutely - // required. - state.with( - ( - (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? - // The token we have is still valid. - state.token - : - // Get a new token. - post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", - { - "client_id": [state.client_id], - "client_secret": [state.client_secret], - "grant_type": ["client_credentials"], - }.format_query() - ).do_request().as(auth, auth.StatusCode == 200 ? - bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ - // Include 60s grace period to avoid attempting to make - // a request with a stale authentication token. - "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), - })) - : - { - "events": { - "error": { - "code": string(auth.StatusCode), - "id": string(auth.Status), - "message": "POST /oauth/token: "+( - size(auth.Body) != 0 ? - string(auth.Body) - : - string(auth.Status) + ' (' + string(auth.StatusCode) + ')' - ), - }, - }, - "want_more": false, - } - ) - ).as(token, !has(token.access_token) ? token : - { - "data": state.?last_page.data.orValue([{ - state.start_field: state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339)), - state.end_field: now.format(time_layout.RFC3339), - }]), - }.as(req, - post_request(state.url.trim_right("/") + state.path, "application/json", - { - "meta": { - "pagination": { - "pageSize": state.page_size, - ?"pageToken": state.?last_page.next, - } - }, - "data": req.data, - }.encode_json() - ).with({ - "Header": { - "Authorization": ["Bearer " + token.access_token], - "Accept": ["application/json"], - "Content-Type": ["application/json"], - } - }).do_request().as(resp, resp.StatusCode == 200 ? - bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ? - { - "events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(), - "cursor": { - "last": ([now] + body.data.map(e, - e[state.data_path].map(l, - l[state.time_field].parse_time(["2006-01-02T15:04:05-0700", time_layout.RFC3339]) - ) - ).flatten()).max().format(time_layout.RFC3339), - }, - ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ? - optional.of({ - ?"next": body.?meta.pagination.next, - "data": req.data, - }) - : - optional.none(), - "token": { - "access_token": token.access_token, - "expires": token.expires, - }, - "want_more": has(body.?meta.pagination.next) && size(body.data) != 0, - } - : - // Mimecast can return failure states with a 200. This - // is detected by a non-empty fail array at the root - // of the response body. Don't attempt to parse this - // out, just dump the whole body into the error message. - { - "events": { - "error": { - "code": string(resp.StatusCode), - "id": string(resp.Status), - "message": "POST " + state.path + ":" + string(resp.Body), // We know this is not empty. - }, - }, - "want_more": false, - } - ) - : - { - "events": { - "error": { - "code": string(resp.StatusCode), - "id": string(resp.Status), - "message": "POST " + state.path + ": " + ( - size(resp.Body) != 0 ? - string(resp.Body) - : - string(resp.Status) + ' (' + string(resp.StatusCode) + ')' - ), - }, - }, - "want_more": false, - } - ) - ) - ) - ) - redact: - fields: - - client_id - - client_secret - - token.access_token - resource.url: https://api.services.mimecast.com - state: - client_id: null - client_secret: null - data_path: dlpLogs - end_field: to - look_back: 24h - page_size: 100 - path: /api/dlp/get-logs - start_field: from - time_field: eventTime + request.method: POST + request.transforms: + - set: + default: '[{"to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' + target: body.data + value: '[{"to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[.cursor.next_date]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/dlp/get-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + request.url: https://eu-api.mimecast.com/api/dlp/get-logs + response.decode_as: application/json + response.pagination: + - delete: + target: body.data + - set: + fail_on_template_error: true + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + response.split: + ignore_empty_value: true + split: + target: body.dlpLogs + target: body.data tags: - forwarded - mimecast-dlp-logs - - condition: ${kubernetes.hints.mimecast.message_release_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 + - condition: ${kubernetes.hints.mimecast.siem_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_token: + value: '[[.last_response.header.Get "mc-siem-token"]]' data_stream: - dataset: mimecast.message_release_logs + dataset: mimecast.siem_logs type: logs - fields_under_root: true interval: 5m - keep_null: true - program: | - // This program is shared amongst archive_search_logs, dlp_logs, - // message_release_logs, ttp_ap_logs, ttp_ip_logs, and ttp_url_logs. - // If it is changed here changes should be reflected in the other - // data streams. Do not differentiate the logic between these data - // streams lightly; use the state variable for this unless absolutely - // required. - state.with( - ( - (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? - // The token we have is still valid. - state.token - : - // Get a new token. - post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", - { - "client_id": [state.client_id], - "client_secret": [state.client_secret], - "grant_type": ["client_credentials"], - }.format_query() - ).do_request().as(auth, auth.StatusCode == 200 ? - bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ - // Include 60s grace period to avoid attempting to make - // a request with a stale authentication token. - "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), - })) - : - { - "events": { - "error": { - "code": string(auth.StatusCode), - "id": string(auth.Status), - "message": "POST /oauth/token: "+( - size(auth.Body) != 0 ? - string(auth.Body) - : - string(auth.Status) + ' (' + string(auth.StatusCode) + ')' - ), - }, - }, - "want_more": false, - } - ) - ).as(token, !has(token.access_token) ? token : - { - "data": state.?last_page.data.orValue([{ - state.start_field: state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339)), - state.end_field: now.format(time_layout.RFC3339), - }]), - }.as(req, - post_request(state.url.trim_right("/") + state.path, "application/json", - { - "meta": { - "pagination": { - "pageSize": state.page_size, - ?"pageToken": state.?last_page.next, - } - }, - "data": req.data, - }.encode_json() - ).with({ - "Header": { - "Authorization": ["Bearer " + token.access_token], - "Accept": ["application/json"], - "Content-Type": ["application/json"], - } - }).do_request().as(resp, resp.StatusCode == 200 ? - bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ? - { - "events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(), - "cursor": { - "last": ([now] + body.data.map(e, - e[state.data_path].map(l, - l[state.time_field].parse_time(["2006-01-02T15:04:05-0700", time_layout.RFC3339]) - ) - ).flatten()).max().format(time_layout.RFC3339), - }, - ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ? - optional.of({ - ?"next": body.?meta.pagination.next, - "data": req.data, - }) - : - optional.none(), - "token": { - "access_token": token.access_token, - "expires": token.expires, - }, - "want_more": has(body.?meta.pagination.next) && size(body.data) != 0, - } - : - // Mimecast can return failure states with a 200. This - // is detected by a non-empty fail array at the root - // of the response body. Don't attempt to parse this - // out, just dump the whole body into the error message. - { - "events": { - "error": { - "code": string(resp.StatusCode), - "id": string(resp.Status), - "message": "POST " + state.path + ":" + string(resp.Body), // We know this is not empty. - }, - }, - "want_more": false, - } - ) - : - { - "events": { - "error": { - "code": string(resp.StatusCode), - "id": string(resp.Status), - "message": "POST " + state.path + ": " + ( - size(resp.Body) != 0 ? - string(resp.Body) - : - string(resp.Status) + ' (' + string(resp.StatusCode) + ')' - ), - }, - }, - "want_more": false, - } - ) - ) - ) - ) - redact: - fields: - - client_id - - client_secret - - token.access_token - resource.url: https://api.services.mimecast.com - state: - client_id: null - client_secret: null - data_path: heldReleaseLogs - end_field: end - look_back: 24h - page_size: 100 - path: /api/gateway/get-held-release-logs - start_field: start - time_field: released - tags: - - forwarded - - mimecast-message-release-logs - - condition: ${kubernetes.hints.mimecast.siem_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 - data_stream: - dataset: mimecast.siem_logs - type: logs - fields_under_root: true - interval: 5m - keep_null: true - program: "// This program is shared between cloud_integrated_logs and siem_logs\n// If it is changed here changes should be reflected in the other data\n// streams. Do not differentiate the logic between these data streams\n// lightly; use the state variable for this unless absolutely required.\nstate.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token: \"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n state.?cursor.work_list.orValue(state.types.map(t, {\"type\": t})).as(work_list, size(work_list) == 0 ?\n state.types.map(t, {\"type\": t})\n :\n work_list\n ).as(work_list,\n get_request(\n state.url.trim_right(\"/\") + state.path + \"?\" + {\n \"type\": [work_list[0].type],\n ?\"nextPage\": work_list[0].?next.optMap(next, [next]),\n ?\"dateRangeStartsAt\": state.?start.optMap(start, [start.format(\"2006-01-02\")]),\n ?\"dateRangeEndsAt\": state.?end.optMap(end, [end.format(\"2006-01-02\")]),\n ?\"pageSize\": state.?page_size.optMap(size, [string(int(size))]),\n }.format_query()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body,\n {\n \"events\": body.value.map(b, has(b.url),\n get(b.url).as(batch, batch.StatusCode == 200 ?\n bytes(batch.Body).mime(\"application/gzip\").mime(\"application/x-ndjson\").map(e,\n {\n \"message\": dyn(e.encode_json()),\n }\n )\n :\n [{\n \"error\": {\n \"code\": string(batch.StatusCode),\n \"id\": string(batch.Status),\n \"message\": \"GET \" + b.url + \": \" + (\n size(batch.Body) != 0 ?\n string(batch.Body)\n :\n string(batch.Status) + ' (' + string(batch.StatusCode) + ')'\n ),\n },\n }]\n )\n ).flatten(),\n \"cursor\": {\n \"work_list\": (\n \"@nextPage\" in body && size(body.value) != 0 ?\n [work_list[0].with({\"next\": body[\"@nextPage\"]})]\n :\n []\n ) + tail(work_list),\n },\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": \"@nextPage\" in body && size(body.value) != 0,\n }.as(to_publish, to_publish.with({\n \"want_more\": to_publish.want_more || size(to_publish.cursor.work_list) != 0,\n }))\n ).as(state, \n // Check whether we still need to get more, but have\n // no event for this type. If we do, populate events\n // with a place-holder to be discarded by the ingest\n // pipeline.\n state.want_more && size(state.events) == 0 ?\n state.with({\"events\": [{\"message\": \"want_more\"}]})\n :\n state\n )\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"GET \" + state.path + \": \" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" - redact: - fields: - - client_id - - client_secret - - token.access_token - resource.url: https://api.services.mimecast.com - state: - client_id: null - client_secret: null - end_field: dateRangeEndsAt - look_back: 24h - page_size: null - path: /siem/v1/batch/events/cg - start_field: dateRangeStartsAt - types: - - av - - delivery - - internal email protect - - impersonation protect - - journal - - process - - receipt - - attachment protect - - spam - - url protect + request.method: POST + request.transforms: + - set: + default: '[{"type":"MTA","fileFormat":"json", "compress":true}]' + target: body.data + value: '[{"type":"MTA","fileFormat":"json", "compress":true, "token": "[[.cursor.next_token]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/audit/get-siem-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + - set: + target: header.Accept + value: '*/*' + request.url: https://eu-api.mimecast.com/api/audit/get-siem-logs + response.decode_as: application/zip + response.split: + ignore_empty_value: true + target: body.data + transforms: + - set: + target: body.Content-Disposition + value: '[[.last_response.header.Get "Content-Disposition"]]' tags: - forwarded - mimecast-siem-logs - - condition: ${kubernetes.hints.mimecast.threat_intel_malware_customer.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 + - condition: ${kubernetes.hints.mimecast.threat_intel_malware_customer.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_date: + value: '[[.first_event.created]]' data_stream: dataset: mimecast.threat_intel_malware_customer type: logs - fields_under_root: true interval: 5m - keep_null: true - program: "// This program is shared between threat_intel_malware_customer and\n// threat_intel_malware_grid. If it is changed here changes should be\n// reflected in the other data streams. Do not differentiate the logic\n// between these data streams lightly; use the state variable for this\n// unless absolutely required.\nstate.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token: \"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n {\n \"data\": state.?last_page.data.orValue([{\n ?\"start\": has(state.?cursor.token) ? optional.none() :\n optional.of(state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339))),\n ?\"end\": has(state.?cursor.token) ? optional.none() :\n optional.of(now.format(time_layout.RFC3339)),\n \"feedType\": state.feed_type,\n ?\"token\": state.?cursor.token,\n \"fileType\": \"stix\",\n }]),\n }.as(req,\n post_request(state.url.trim_right(\"/\") + state.path, \"application/json\", \n req.encode_json()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?\n {\n \"events\": body.objects.map(e, e.type == \"indicator\", {\"message\": e.encode_json()}),\n \"cursor\": {\n // The last timestamp may step past the last timestamp\n // seen for an indicator. We assume here that if another\n // type has a later timestamp, then the time at the API\n // has progressed past the last indicator and we do not\n // need to reach back that far.\n \"last\": ([now] + body.objects.map(e, timestamp(e.modified))).max().format(time_layout.RFC3339),\n ?\"token\": resp.?Header[\"X-Mc-Threat-Feed-Next-Token\"][?0],\n },\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": resp.?Header[\"X-Mc-Threat-Feed-Next-Token\"].hasValue(),\n }\n :\n // Mimecast can return failure states with a 200. This\n // is detected by a non-empty fail array at the root\n // of the response body. Don't attempt to parse this\n // out, just dump the whole body into the error message.\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \": \" + string(resp.Body), // We know this is not empty.\n },\n },\n \"want_more\": false,\n }\n )\n : resp.StatusCode == 429 ?\n // For reasons, Mimecast does not set X-RateLimit-* headers\n // until the rate limit has been exceeded, so treat 429 codes\n // as a sentinel to back off. We don't want to log errors and\n // we do not want to update the cursor, so return an empty\n // events array.\n {\n \"events\": [],\n // Log the rate limit excession at DEBUG level.\n \"rate_limited\": debug(\"rate_limit_exceeded\", bytes(resp.Body).decode_json().?fail[0].message.orValue(\"missing message\")),\n \"want_more\": false,\n }\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \": \" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" - redact: - fields: - - client_id - - client_secret - - token.access_token - resource.url: https://api.services.mimecast.com - state: - client_id: null - client_secret: null - feed_type: malware_customer - look_back: 24h - page_size: null - path: /api/ttp/threat-intel/get-feed + request.method: POST + request.transforms: + - set: + default: '[{"feedType": "malware_customer","fileType": "stix","compress": false,"end": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "start":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' + target: body.data + value: '[{"feedType": "malware_customer","fileType": "stix","compress": false,"token": "[[.last_response.header.Get "x-mc-threat-feed-next-token"]]", "end": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "start":"[[formatDate (.cursor.next_date) "2006-01-02T15:04:05-0700"]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/threat-intel/get-feed:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + request.url: https://eu-api.mimecast.com/api/ttp/threat-intel/get-feed + response.decode_as: application/json + response.split: + ignore_empty_value: true + target: body.objects + transforms: + - set: + target: body.Content-Disposition + value: '[[.last_response.header.Get "Content-Disposition"]]' tags: - forwarded - mimecast-threat-intel-feed-malware-customer - - condition: ${kubernetes.hints.mimecast.threat_intel_malware_grid.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 + - condition: ${kubernetes.hints.mimecast.threat_intel_malware_grid.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_date: + value: '[[.first_event.created]]' data_stream: dataset: mimecast.threat_intel_malware_grid type: logs - fields_under_root: true interval: 5m - keep_null: true - program: "// This program is shared between threat_intel_malware_customer and\n// threat_intel_malware_grid. If it is changed here changes should be\n// reflected in the other data streams. Do not differentiate the logic\n// between these data streams lightly; use the state variable for this\n// unless absolutely required.\nstate.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token: \"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n {\n \"data\": state.?last_page.data.orValue([{\n ?\"start\": has(state.?cursor.token) ? optional.none() :\n optional.of(state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339))),\n ?\"end\": has(state.?cursor.token) ? optional.none() :\n optional.of(now.format(time_layout.RFC3339)),\n \"feedType\": state.feed_type,\n ?\"token\": state.?cursor.token,\n \"fileType\": \"stix\",\n }]),\n }.as(req,\n post_request(state.url.trim_right(\"/\") + state.path, \"application/json\", \n req.encode_json()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?\n {\n \"events\": body.objects.map(e, e.type == \"indicator\", {\"message\": e.encode_json()}),\n \"cursor\": {\n // The last timestamp may step past the last timestamp\n // seen for an indicator. We assume here that if another\n // type has a later timestamp, then the time at the API\n // has progressed past the last indicator and we do not\n // need to reach back that far.\n \"last\": ([now] + body.objects.map(e, timestamp(e.modified))).max().format(time_layout.RFC3339),\n ?\"token\": resp.?Header[\"X-Mc-Threat-Feed-Next-Token\"][?0],\n },\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": resp.?Header[\"X-Mc-Threat-Feed-Next-Token\"].hasValue(),\n }\n :\n // Mimecast can return failure states with a 200. This\n // is detected by a non-empty fail array at the root\n // of the response body. Don't attempt to parse this\n // out, just dump the whole body into the error message.\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \": \" + string(resp.Body), // We know this is not empty.\n },\n },\n \"want_more\": false,\n }\n )\n : resp.StatusCode == 429 ?\n // For reasons, Mimecast does not set X-RateLimit-* headers\n // until the rate limit has been exceeded, so treat 429 codes\n // as a sentinel to back off. We don't want to log errors and\n // we do not want to update the cursor, so return an empty\n // events array.\n {\n \"events\": [],\n // Log the rate limit excession at DEBUG level.\n \"rate_limited\": debug(\"rate_limit_exceeded\", bytes(resp.Body).decode_json().?fail[0].message.orValue(\"missing message\")),\n \"want_more\": false,\n }\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \": \" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" - redact: - fields: - - client_id - - client_secret - - token.access_token - resource.url: https://api.services.mimecast.com - state: - client_id: null - client_secret: null - feed_type: malware_grid - look_back: 24h - page_size: null - path: /api/ttp/threat-intel/get-feed + request.method: POST + request.transforms: + - set: + default: '[{"feedType": "malware_grid","fileType": "stix","compress": false,"end": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "start":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' + target: body.data + value: '[{"feedType": "malware_grid","fileType": "stix","compress": false,"token": "[[.last_response.header.Get "x-mc-threat-feed-next-token"]]", "end": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "start":"[[formatDate (.cursor.next_date) "2006-01-02T15:04:05-0700"]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/threat-intel/get-feed:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + request.url: https://eu-api.mimecast.com/api/ttp/threat-intel/get-feed + response.decode_as: application/json + response.split: + ignore_empty_value: true + target: body.objects + transforms: + - set: + target: body.Content-Disposition + value: '[[.last_response.header.Get "Content-Disposition"]]' tags: - forwarded - mimecast-threat-intel-feed-malware-grid - - condition: ${kubernetes.hints.mimecast.ttp_ap_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true - config_version: 2 + - condition: ${kubernetes.hints.mimecast.ttp_ap_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_date: + value: '[[.first_event.date]]' data_stream: dataset: mimecast.ttp_ap_logs type: logs + interval: 5m + request.method: POST + request.transforms: + - set: + default: '[{"oldestFirst": false, "route": "all", "result":"all","to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' + target: body.data + value: '[{"oldestFirst": false, "route": "all", "result":"all","to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[.cursor.next_date]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/attachment/get-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + request.url: https://eu-api.mimecast.com/api/ttp/attachment/get-logs + response.decode_as: application/json + response.pagination: + - delete: + target: body.data + - set: + fail_on_template_error: true + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + response.split: + ignore_empty_value: true + split: + target: body.attachmentLogs + target: body.data + tags: + - forwarded + - mimecast-ttp-ap + - condition: ${kubernetes.hints.mimecast.ttp_ip_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_date: + value: '[[.first_event.eventTime]]' + data_stream: + dataset: mimecast.ttp_ip_logs + type: logs + interval: 5m + request.method: POST + request.transforms: + - set: + default: '[{"oldestFirst": false,"to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' + target: body.data + value: '[{"oldestFirst": false,"to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[.cursor.next_date]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/impersonation/get-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + request.url: https://eu-api.mimecast.com/api/ttp/impersonation/get-logs + response.decode_as: application/json + response.pagination: + - delete: + target: body.data + - set: + fail_on_template_error: true + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + response.split: + ignore_empty_value: true + split: + target: body.impersonationLogs + target: body.data + tags: + - forwarded + - mimecast-ttp-ip + - condition: ${kubernetes.hints.mimecast.ttp_url_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true + config_version: "2" + cursor: + next_date: + value: '[[.first_event.date]]' + data_stream: + dataset: mimecast.ttp_url_logs + type: logs + interval: 5m + request.method: POST + request.transforms: + - set: + default: '[{"oldestFirst": false,"scanResult": "all","route":"all","to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' + target: body.data + value: '[{"oldestFirst": false,"scanResult": "all","route":"all","to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[.cursor.next_date]]"}]' + value_type: json + - set: + target: header.x-mc-app-id + value: null + - set: + target: header.x-mc-date + value: '[[formatDate (now) "RFC1123"]]' + - set: + target: header.x-mc-req-id + value: '[[uuid]]' + - set: + fail_on_template_error: true + target: header.Authorization + value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/url/get-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] + request.url: https://eu-api.mimecast.com/api/ttp/url/get-logs + response.decode_as: application/json + response.pagination: + - delete: + target: body.data + - set: + fail_on_template_error: true + target: body.meta.pagination.pageToken + value: '[[.last_response.body.meta.pagination.next]]' + response.split: + ignore_empty_value: true + split: + target: body.clickLogs + target: body.data + tags: + - forwarded + - mimecast-ttp-url + data_stream.namespace: default + - name: cel-mimecast + id: cel-mimecast-${kubernetes.hints.container_id} + type: cel + use_output: default + streams: + - condition: ${kubernetes.hints.mimecast.archive_search_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 + data_stream: + dataset: mimecast.archive_search_logs + type: logs fields_under_root: true interval: 5m keep_null: true @@ -750,76 +562,133 @@ inputs: state: client_id: null client_secret: null - data_path: attachmentLogs - end_field: to + data_path: logs + end_field: end look_back: 24h page_size: 100 - path: /api/ttp/attachment/get-logs - start_field: from - time_field: date + path: /api/archive/get-archive-search-logs + start_field: start + time_field: createTime tags: - forwarded - - mimecast-ttp-ap - - condition: ${kubernetes.hints.mimecast.ttp_ip_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + - mimecast-archive-search-logs + - condition: ${kubernetes.hints.mimecast.audit_events.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true config_version: 2 data_stream: - dataset: mimecast.ttp_ip_logs + dataset: mimecast.audit_events type: logs fields_under_root: true interval: 5m keep_null: true - program: | - // This program is shared amongst archive_search_logs, dlp_logs, - // message_release_logs, ttp_ap_logs, ttp_ip_logs, and ttp_url_logs. - // If it is changed here changes should be reflected in the other - // data streams. Do not differentiate the logic between these data - // streams lightly; use the state variable for this unless absolutely - // required. - state.with( - ( - (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? - // The token we have is still valid. - state.token - : - // Get a new token. - post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", - { - "client_id": [state.client_id], - "client_secret": [state.client_secret], - "grant_type": ["client_credentials"], - }.format_query() - ).do_request().as(auth, auth.StatusCode == 200 ? - bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ - // Include 60s grace period to avoid attempting to make - // a request with a stale authentication token. - "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), - })) - : - { - "events": { - "error": { - "code": string(auth.StatusCode), - "id": string(auth.Status), - "message": "POST /oauth/token: "+( - size(auth.Body) != 0 ? - string(auth.Body) - : - string(auth.Status) + ' (' + string(auth.StatusCode) + ')' - ), - }, - }, - "want_more": false, - } - ) - ).as(token, !has(token.access_token) ? token : - { - "data": state.?last_page.data.orValue([{ - state.start_field: state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339)), - state.end_field: now.format(time_layout.RFC3339), - }]), - }.as(req, - post_request(state.url.trim_right("/") + state.path, "application/json", - { + program: "state.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token:\"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n {\n \"data\": state.?last_page.data.orValue([{\n state.start_field: state.?cursor.last.orValue(now - duration(state.look_back)).format(time_layout.RFC3339),\n state.end_field: now.format(time_layout.RFC3339),\n }]),\n }.as(req,\n post_request(state.url.trim_right(\"/\") + state.path, \"application/json\", \n {\n \"meta\": {\n \"pagination\": {\n \"pageSize\": state.page_size,\n ?\"pageToken\": state.?last_page.next,\n }\n },\n \"data\": req.data,\n }.encode_json()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?\n {\n \"events\": body.data.map(e, {\"message\": e.encode_json()}),\n \"cursor\": {\n \"last\": ([now] + body.data.map(e, e[state.time_field].parse_time([\"2006-01-02T15:04:05-0700\", time_layout.RFC3339]))).max(),\n },\n ?\"last_page\": has(body.?meta.pagination.next) && size(body.data) != 0 ?\n optional.of({\n ?\"next\": body.?meta.pagination.next,\n \"data\": req.data,\n })\n :\n optional.none(),\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": has(body.?meta.pagination.next) && size(body.data) != 0,\n }\n :\n // Mimecast can return failure states with a 200. This\n // is detected by a non-empty fail array at the root\n // of the response body. Don't attempt to parse this\n // out, just dump the whole body into the error message.\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \":\" + string(resp.Body), // We know this is not empty.\n },\n },\n \"want_more\": false,\n }\n )\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \":\" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" + redact: + fields: + - client_id + - client_secret + - token.access_token + resource.url: https://api.services.mimecast.com + state: + client_id: null + client_secret: null + end_field: endDateTime + look_back: 24h + page_size: 100 + path: /api/audit/get-audit-events + start_field: startDateTime + time_field: eventTime + tags: + - forwarded + - mimecast-audit-events + - condition: ${kubernetes.hints.mimecast.cloud_integrated_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 + data_stream: + dataset: mimecast.cloud_integrated_logs + type: logs + fields_under_root: true + interval: 5m + keep_null: true + program: "// This program is shared between cloud_integrated_logs and siem_logs\n// If it is changed here changes should be reflected in the other data\n// streams. Do not differentiate the logic between these data streams\n// lightly; use the state variable for this unless absolutely required.\nstate.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token: \"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n state.?cursor.work_list.orValue(state.types.map(t, {\"type\": t})).as(work_list, size(work_list) == 0 ?\n state.types.map(t, {\"type\": t})\n :\n work_list\n ).as(work_list,\n get_request(\n state.url.trim_right(\"/\") + state.path + \"?\" + {\n \"type\": [work_list[0].type],\n ?\"nextPage\": work_list[0].?next.optMap(next, [next]),\n ?\"dateRangeStartsAt\": state.?start.optMap(start, [start.format(\"2006-01-02\")]),\n ?\"dateRangeEndsAt\": state.?end.optMap(end, [end.format(\"2006-01-02\")]),\n ?\"pageSize\": state.?page_size.optMap(size, [string(int(size))]),\n }.format_query()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body,\n {\n \"events\": body.value.map(b, has(b.url),\n get(b.url).as(batch, batch.StatusCode == 200 ?\n bytes(batch.Body).mime(\"application/gzip\").mime(\"application/x-ndjson\").map(e,\n {\n \"message\": dyn(e.encode_json()),\n }\n )\n :\n [{\n \"error\": {\n \"code\": string(batch.StatusCode),\n \"id\": string(batch.Status),\n \"message\": \"GET \" + b.url + \": \" + (\n size(batch.Body) != 0 ?\n string(batch.Body)\n :\n string(batch.Status) + ' (' + string(batch.StatusCode) + ')'\n ),\n },\n }]\n )\n ).flatten(),\n \"cursor\": {\n \"work_list\": (\n \"@nextPage\" in body && size(body.value) != 0 ?\n [work_list[0].with({\"next\": body[\"@nextPage\"]})]\n :\n []\n ) + tail(work_list),\n },\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": \"@nextPage\" in body && size(body.value) != 0,\n }.as(to_publish, to_publish.with({\n \"want_more\": to_publish.want_more || size(to_publish.cursor.work_list) != 0,\n }))\n ).as(state, \n // Check whether we still need to get more, but have\n // no event for this type. If we do, populate events\n // with a place-holder to be discarded by the ingest\n // pipeline.\n state.want_more && size(state.events) == 0 ?\n state.with({\"events\": [{\"message\": \"want_more\"}]})\n :\n state\n )\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"GET \" + state.path + \": \" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" + redact: + fields: + - client_id + - client_secret + - token.access_token + resource.url: https://api.services.mimecast.com + state: + client_id: null + client_secret: null + end_field: dateRangeEndsAt + look_back: 24h + page_size: null + path: /siem/v1/batch/events/ci + start_field: dateRangeStartsAt + types: + - entities + - mailflow + - urlclick + tags: + - forwarded + - mimecast-cloud-integrated-logs + - condition: ${kubernetes.hints.mimecast.dlp_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 + data_stream: + dataset: mimecast.dlp_logs + type: logs + fields_under_root: true + interval: 5m + keep_null: true + program: | + // This program is shared amongst archive_search_logs, dlp_logs, + // message_release_logs, ttp_ap_logs, ttp_ip_logs, and ttp_url_logs. + // If it is changed here changes should be reflected in the other + // data streams. Do not differentiate the logic between these data + // streams lightly; use the state variable for this unless absolutely + // required. + state.with( + ( + (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? + // The token we have is still valid. + state.token + : + // Get a new token. + post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", + { + "client_id": [state.client_id], + "client_secret": [state.client_secret], + "grant_type": ["client_credentials"], + }.format_query() + ).do_request().as(auth, auth.StatusCode == 200 ? + bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ + // Include 60s grace period to avoid attempting to make + // a request with a stale authentication token. + "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), + })) + : + { + "events": { + "error": { + "code": string(auth.StatusCode), + "id": string(auth.Status), + "message": "POST /oauth/token: "+( + size(auth.Body) != 0 ? + string(auth.Body) + : + string(auth.Status) + ' (' + string(auth.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ).as(token, !has(token.access_token) ? token : + { + "data": state.?last_page.data.orValue([{ + state.start_field: state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339)), + state.end_field: now.format(time_layout.RFC3339), + }]), + }.as(req, + post_request(state.url.trim_right("/") + state.path, "application/json", + { "meta": { "pagination": { "pageSize": state.page_size, @@ -903,20 +772,20 @@ inputs: state: client_id: null client_secret: null - data_path: impersonationLogs + data_path: dlpLogs end_field: to look_back: 24h page_size: 100 - path: /api/ttp/impersonation/get-logs + path: /api/dlp/get-logs start_field: from time_field: eventTime tags: - forwarded - - mimecast-ttp-ip - - condition: ${kubernetes.hints.mimecast.ttp_url_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + - mimecast-dlp-logs + - condition: ${kubernetes.hints.mimecast.message_release_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true config_version: 2 data_stream: - dataset: mimecast.ttp_url_logs + dataset: mimecast.message_release_logs type: logs fields_under_root: true interval: 5m @@ -1056,451 +925,582 @@ inputs: state: client_id: null client_secret: null - data_path: clickLogs - end_field: to + data_path: heldReleaseLogs + end_field: end look_back: 24h page_size: 100 - path: /api/ttp/url/get-logs - start_field: from - time_field: date + path: /api/gateway/get-held-release-logs + start_field: start + time_field: released tags: - forwarded - - mimecast-ttp-url - data_stream.namespace: default - - name: filestream-mimecast - id: filestream-mimecast-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.mimecast.container_logs.enabled} == true - data_stream: - dataset: mimecast.container_logs - type: logs - exclude_files: [] - exclude_lines: [] - parsers: - - container: - format: auto - stream: all - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - symlinks: true - tags: [] - data_stream.namespace: default - - name: httpjson-mimecast - id: httpjson-mimecast-${kubernetes.hints.container_id} - type: httpjson - use_output: default - streams: - - condition: ${kubernetes.hints.mimecast.archive_search_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_date: - value: '[[.first_event.createTime]]' + - mimecast-message-release-logs + - condition: ${kubernetes.hints.mimecast.siem_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 data_stream: - dataset: mimecast.archive_search_logs + dataset: mimecast.siem_logs type: logs + fields_under_root: true interval: 5m - request.method: POST - request.transforms: - - set: - target: body.meta.pagination.pageSize - value: 100 - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/archive/get-archive-search-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - request.url: https://eu-api.mimecast.com/api/archive/get-archive-search-logs - response.decode_as: application/json - response.pagination: - - delete: - target: body.data - - set: - fail_on_template_error: true - target: body.meta.pagination.pageToken - value: |- - [[- if index .last_response.body.meta.pagination "next" -]] - [[- .last_response.body.meta.pagination.next -]] - [[- end -]] - response.split: - ignore_empty_value: true - split: - ignore_empty_value: true - keep_parent: false - target: body.logs - target: body.data + keep_null: true + program: "// This program is shared between cloud_integrated_logs and siem_logs\n// If it is changed here changes should be reflected in the other data\n// streams. Do not differentiate the logic between these data streams\n// lightly; use the state variable for this unless absolutely required.\nstate.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token: \"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n state.?cursor.work_list.orValue(state.types.map(t, {\"type\": t})).as(work_list, size(work_list) == 0 ?\n state.types.map(t, {\"type\": t})\n :\n work_list\n ).as(work_list,\n get_request(\n state.url.trim_right(\"/\") + state.path + \"?\" + {\n \"type\": [work_list[0].type],\n ?\"nextPage\": work_list[0].?next.optMap(next, [next]),\n ?\"dateRangeStartsAt\": state.?start.optMap(start, [start.format(\"2006-01-02\")]),\n ?\"dateRangeEndsAt\": state.?end.optMap(end, [end.format(\"2006-01-02\")]),\n ?\"pageSize\": state.?page_size.optMap(size, [string(int(size))]),\n }.format_query()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body,\n {\n \"events\": body.value.map(b, has(b.url),\n get(b.url).as(batch, batch.StatusCode == 200 ?\n bytes(batch.Body).mime(\"application/gzip\").mime(\"application/x-ndjson\").map(e,\n {\n \"message\": dyn(e.encode_json()),\n }\n )\n :\n [{\n \"error\": {\n \"code\": string(batch.StatusCode),\n \"id\": string(batch.Status),\n \"message\": \"GET \" + b.url + \": \" + (\n size(batch.Body) != 0 ?\n string(batch.Body)\n :\n string(batch.Status) + ' (' + string(batch.StatusCode) + ')'\n ),\n },\n }]\n )\n ).flatten(),\n \"cursor\": {\n \"work_list\": (\n \"@nextPage\" in body && size(body.value) != 0 ?\n [work_list[0].with({\"next\": body[\"@nextPage\"]})]\n :\n []\n ) + tail(work_list),\n },\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": \"@nextPage\" in body && size(body.value) != 0,\n }.as(to_publish, to_publish.with({\n \"want_more\": to_publish.want_more || size(to_publish.cursor.work_list) != 0,\n }))\n ).as(state, \n // Check whether we still need to get more, but have\n // no event for this type. If we do, populate events\n // with a place-holder to be discarded by the ingest\n // pipeline.\n state.want_more && size(state.events) == 0 ?\n state.with({\"events\": [{\"message\": \"want_more\"}]})\n :\n state\n )\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"GET \" + state.path + \": \" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" + redact: + fields: + - client_id + - client_secret + - token.access_token + resource.url: https://api.services.mimecast.com + state: + client_id: null + client_secret: null + end_field: dateRangeEndsAt + look_back: 24h + page_size: null + path: /siem/v1/batch/events/cg + start_field: dateRangeStartsAt + types: + - av + - delivery + - internal email protect + - impersonation protect + - journal + - process + - receipt + - attachment protect + - spam + - url protect tags: - forwarded - - mimecast-archive-search-logs - - condition: ${kubernetes.hints.mimecast.audit_events.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_date: - value: '[[.first_event.eventTime]]' + - mimecast-siem-logs + - condition: ${kubernetes.hints.mimecast.threat_intel_malware_customer.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 data_stream: - dataset: mimecast.audit_events + dataset: mimecast.threat_intel_malware_customer type: logs + fields_under_root: true interval: 5m - request.method: POST - request.transforms: - - set: - target: body.meta.pagination.pageSize - value: 100 - - set: - default: '[{"endDateTime": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "startDateTime":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' - target: body.data - value: '[{"endDateTime": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "startDateTime":"[[.cursor.next_date]]"}]' - value_type: json - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/audit/get-audit-events:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - request.url: https://eu-api.mimecast.com/api/audit/get-audit-events - response.decode_as: application/json - response.pagination: - - delete: - target: body.data - - set: - fail_on_template_error: true - target: body.meta.pagination.pageToken - value: |- - [[- if index .last_response.body.meta.pagination "next" -]] - [[- .last_response.body.meta.pagination.next -]] - [[- end -]] - response.split: - ignore_empty_value: true - target: body.data - tags: - - forwarded - - mimecast-audit-events - - condition: ${kubernetes.hints.mimecast.dlp_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_date: - value: '[[.first_event.eventTime]]' - data_stream: - dataset: mimecast.dlp_logs - type: logs - interval: 5m - request.method: POST - request.transforms: - - set: - default: '[{"to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' - target: body.data - value: '[{"to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[.cursor.next_date]]"}]' - value_type: json - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/dlp/get-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - request.url: https://eu-api.mimecast.com/api/dlp/get-logs - response.decode_as: application/json - response.pagination: - - delete: - target: body.data - - set: - fail_on_template_error: true - target: body.meta.pagination.pageToken - value: '[[.last_response.body.meta.pagination.next]]' - response.split: - ignore_empty_value: true - split: - target: body.dlpLogs - target: body.data - tags: - - forwarded - - mimecast-dlp-logs - - condition: ${kubernetes.hints.mimecast.siem_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_token: - value: '[[.last_response.header.Get "mc-siem-token"]]' - data_stream: - dataset: mimecast.siem_logs - type: logs - interval: 5m - request.method: POST - request.transforms: - - set: - default: '[{"type":"MTA","fileFormat":"json", "compress":true}]' - target: body.data - value: '[{"type":"MTA","fileFormat":"json", "compress":true, "token": "[[.cursor.next_token]]"}]' - value_type: json - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/audit/get-siem-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - - set: - target: header.Accept - value: '*/*' - request.url: https://eu-api.mimecast.com/api/audit/get-siem-logs - response.decode_as: application/zip - response.split: - ignore_empty_value: true - target: body.data - transforms: - - set: - target: body.Content-Disposition - value: '[[.last_response.header.Get "Content-Disposition"]]' - tags: - - forwarded - - mimecast-siem-logs - - condition: ${kubernetes.hints.mimecast.threat_intel_malware_customer.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_date: - value: '[[.first_event.created]]' - data_stream: - dataset: mimecast.threat_intel_malware_customer - type: logs - interval: 5m - request.method: POST - request.transforms: - - set: - default: '[{"feedType": "malware_customer","fileType": "stix","compress": false,"end": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "start":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' - target: body.data - value: '[{"feedType": "malware_customer","fileType": "stix","compress": false,"token": "[[.last_response.header.Get "x-mc-threat-feed-next-token"]]", "end": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "start":"[[formatDate (.cursor.next_date) "2006-01-02T15:04:05-0700"]]"}]' - value_type: json - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/threat-intel/get-feed:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - request.url: https://eu-api.mimecast.com/api/ttp/threat-intel/get-feed - response.decode_as: application/json - response.split: - ignore_empty_value: true - target: body.objects - transforms: - - set: - target: body.Content-Disposition - value: '[[.last_response.header.Get "Content-Disposition"]]' + keep_null: true + program: "// This program is shared between threat_intel_malware_customer and\n// threat_intel_malware_grid. If it is changed here changes should be\n// reflected in the other data streams. Do not differentiate the logic\n// between these data streams lightly; use the state variable for this\n// unless absolutely required.\nstate.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token: \"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n {\n \"data\": state.?last_page.data.orValue([{\n ?\"start\": has(state.?cursor.token) ? optional.none() :\n optional.of(state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339))),\n ?\"end\": has(state.?cursor.token) ? optional.none() :\n optional.of(now.format(time_layout.RFC3339)),\n \"feedType\": state.feed_type,\n ?\"token\": state.?cursor.token,\n \"fileType\": \"stix\",\n }]),\n }.as(req,\n post_request(state.url.trim_right(\"/\") + state.path, \"application/json\", \n req.encode_json()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?\n {\n \"events\": body.objects.map(e, e.type == \"indicator\", {\"message\": e.encode_json()}),\n \"cursor\": {\n // The last timestamp may step past the last timestamp\n // seen for an indicator. We assume here that if another\n // type has a later timestamp, then the time at the API\n // has progressed past the last indicator and we do not\n // need to reach back that far.\n \"last\": ([now] + body.objects.map(e, timestamp(e.modified))).max().format(time_layout.RFC3339),\n ?\"token\": resp.?Header[\"X-Mc-Threat-Feed-Next-Token\"][?0],\n },\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": resp.?Header[\"X-Mc-Threat-Feed-Next-Token\"].hasValue(),\n }\n :\n // Mimecast can return failure states with a 200. This\n // is detected by a non-empty fail array at the root\n // of the response body. Don't attempt to parse this\n // out, just dump the whole body into the error message.\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \": \" + string(resp.Body), // We know this is not empty.\n },\n },\n \"want_more\": false,\n }\n )\n : resp.StatusCode == 429 ?\n // For reasons, Mimecast does not set X-RateLimit-* headers\n // until the rate limit has been exceeded, so treat 429 codes\n // as a sentinel to back off. We don't want to log errors and\n // we do not want to update the cursor, so return an empty\n // events array.\n {\n \"events\": [],\n // Log the rate limit excession at DEBUG level.\n \"rate_limited\": debug(\"rate_limit_exceeded\", bytes(resp.Body).decode_json().?fail[0].message.orValue(\"missing message\")),\n \"want_more\": false,\n }\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \": \" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" + redact: + fields: + - client_id + - client_secret + - token.access_token + resource.url: https://api.services.mimecast.com + state: + client_id: null + client_secret: null + feed_type: malware_customer + look_back: 24h + page_size: null + path: /api/ttp/threat-intel/get-feed tags: - forwarded - mimecast-threat-intel-feed-malware-customer - - condition: ${kubernetes.hints.mimecast.threat_intel_malware_grid.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_date: - value: '[[.first_event.created]]' + - condition: ${kubernetes.hints.mimecast.threat_intel_malware_grid.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 data_stream: dataset: mimecast.threat_intel_malware_grid type: logs + fields_under_root: true interval: 5m - request.method: POST - request.transforms: - - set: - default: '[{"feedType": "malware_grid","fileType": "stix","compress": false,"end": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "start":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' - target: body.data - value: '[{"feedType": "malware_grid","fileType": "stix","compress": false,"token": "[[.last_response.header.Get "x-mc-threat-feed-next-token"]]", "end": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "start":"[[formatDate (.cursor.next_date) "2006-01-02T15:04:05-0700"]]"}]' - value_type: json - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/threat-intel/get-feed:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - request.url: https://eu-api.mimecast.com/api/ttp/threat-intel/get-feed - response.decode_as: application/json - response.split: - ignore_empty_value: true - target: body.objects - transforms: - - set: - target: body.Content-Disposition - value: '[[.last_response.header.Get "Content-Disposition"]]' + keep_null: true + program: "// This program is shared between threat_intel_malware_customer and\n// threat_intel_malware_grid. If it is changed here changes should be\n// reflected in the other data streams. Do not differentiate the logic\n// between these data streams lightly; use the state variable for this\n// unless absolutely required.\nstate.with(\n (\n (has(state.?token.expires) && now() < timestamp(state.token.expires)) ?\n // The token we have is still valid.\n state.token\n :\n // Get a new token.\n post_request(state.url.trim_right(\"/\") + \"/oauth/token\", \"application/x-www-form-urlencoded\",\n {\n \"client_id\": [state.client_id],\n \"client_secret\": [state.client_secret],\n \"grant_type\": [\"client_credentials\"],\n }.format_query()\n ).do_request().as(auth, auth.StatusCode == 200 ?\n bytes(auth.Body).decode_json().as(auth_body, auth_body.with({\n // Include 60s grace period to avoid attempting to make\n // a request with a stale authentication token.\n \"expires\": now()+duration(string(int(auth_body.expires_in)-60)+\"s\"),\n }))\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(auth.StatusCode),\n \"id\": string(auth.Status),\n \"message\": \"POST /oauth/token: \"+(\n size(auth.Body) != 0 ?\n string(auth.Body)\n :\n string(auth.Status) + ' (' + string(auth.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n ).as(token, !has(token.access_token) ? token :\n {\n \"data\": state.?last_page.data.orValue([{\n ?\"start\": has(state.?cursor.token) ? optional.none() :\n optional.of(state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339))),\n ?\"end\": has(state.?cursor.token) ? optional.none() :\n optional.of(now.format(time_layout.RFC3339)),\n \"feedType\": state.feed_type,\n ?\"token\": state.?cursor.token,\n \"fileType\": \"stix\",\n }]),\n }.as(req,\n post_request(state.url.trim_right(\"/\") + state.path, \"application/json\", \n req.encode_json()\n ).with({\n \"Header\": {\n \"Authorization\": [\"Bearer \" + token.access_token], \n \"Accept\": [\"application/json\"],\n \"Content-Type\": [\"application/json\"],\n }\n }).do_request().as(resp, resp.StatusCode == 200 ?\n bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ?\n {\n \"events\": body.objects.map(e, e.type == \"indicator\", {\"message\": e.encode_json()}),\n \"cursor\": {\n // The last timestamp may step past the last timestamp\n // seen for an indicator. We assume here that if another\n // type has a later timestamp, then the time at the API\n // has progressed past the last indicator and we do not\n // need to reach back that far.\n \"last\": ([now] + body.objects.map(e, timestamp(e.modified))).max().format(time_layout.RFC3339),\n ?\"token\": resp.?Header[\"X-Mc-Threat-Feed-Next-Token\"][?0],\n },\n \"token\": {\n \"access_token\": token.access_token,\n \"expires\": token.expires,\n },\n \"want_more\": resp.?Header[\"X-Mc-Threat-Feed-Next-Token\"].hasValue(),\n }\n :\n // Mimecast can return failure states with a 200. This\n // is detected by a non-empty fail array at the root\n // of the response body. Don't attempt to parse this\n // out, just dump the whole body into the error message.\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \": \" + string(resp.Body), // We know this is not empty.\n },\n },\n \"want_more\": false,\n }\n )\n : resp.StatusCode == 429 ?\n // For reasons, Mimecast does not set X-RateLimit-* headers\n // until the rate limit has been exceeded, so treat 429 codes\n // as a sentinel to back off. We don't want to log errors and\n // we do not want to update the cursor, so return an empty\n // events array.\n {\n \"events\": [],\n // Log the rate limit excession at DEBUG level.\n \"rate_limited\": debug(\"rate_limit_exceeded\", bytes(resp.Body).decode_json().?fail[0].message.orValue(\"missing message\")),\n \"want_more\": false,\n }\n :\n {\n \"events\": {\n \"error\": {\n \"code\": string(resp.StatusCode),\n \"id\": string(resp.Status),\n \"message\": \"POST \" + state.path + \": \" + (\n size(resp.Body) != 0 ?\n string(resp.Body)\n :\n string(resp.Status) + ' (' + string(resp.StatusCode) + ')'\n ),\n },\n },\n \"want_more\": false,\n }\n )\n )\n )\n)\n" + redact: + fields: + - client_id + - client_secret + - token.access_token + resource.url: https://api.services.mimecast.com + state: + client_id: null + client_secret: null + feed_type: malware_grid + look_back: 24h + page_size: null + path: /api/ttp/threat-intel/get-feed tags: - forwarded - mimecast-threat-intel-feed-malware-grid - - condition: ${kubernetes.hints.mimecast.ttp_ap_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_date: - value: '[[.first_event.date]]' + - condition: ${kubernetes.hints.mimecast.ttp_ap_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 data_stream: dataset: mimecast.ttp_ap_logs type: logs + fields_under_root: true interval: 5m - request.method: POST - request.transforms: - - set: - default: '[{"oldestFirst": false, "route": "all", "result":"all","to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' - target: body.data - value: '[{"oldestFirst": false, "route": "all", "result":"all","to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[.cursor.next_date]]"}]' - value_type: json - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/attachment/get-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - request.url: https://eu-api.mimecast.com/api/ttp/attachment/get-logs - response.decode_as: application/json - response.pagination: - - delete: - target: body.data - - set: - fail_on_template_error: true - target: body.meta.pagination.pageToken - value: '[[.last_response.body.meta.pagination.next]]' - response.split: - ignore_empty_value: true - split: - target: body.attachmentLogs - target: body.data + keep_null: true + program: | + // This program is shared amongst archive_search_logs, dlp_logs, + // message_release_logs, ttp_ap_logs, ttp_ip_logs, and ttp_url_logs. + // If it is changed here changes should be reflected in the other + // data streams. Do not differentiate the logic between these data + // streams lightly; use the state variable for this unless absolutely + // required. + state.with( + ( + (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? + // The token we have is still valid. + state.token + : + // Get a new token. + post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", + { + "client_id": [state.client_id], + "client_secret": [state.client_secret], + "grant_type": ["client_credentials"], + }.format_query() + ).do_request().as(auth, auth.StatusCode == 200 ? + bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ + // Include 60s grace period to avoid attempting to make + // a request with a stale authentication token. + "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), + })) + : + { + "events": { + "error": { + "code": string(auth.StatusCode), + "id": string(auth.Status), + "message": "POST /oauth/token: "+( + size(auth.Body) != 0 ? + string(auth.Body) + : + string(auth.Status) + ' (' + string(auth.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ).as(token, !has(token.access_token) ? token : + { + "data": state.?last_page.data.orValue([{ + state.start_field: state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339)), + state.end_field: now.format(time_layout.RFC3339), + }]), + }.as(req, + post_request(state.url.trim_right("/") + state.path, "application/json", + { + "meta": { + "pagination": { + "pageSize": state.page_size, + ?"pageToken": state.?last_page.next, + } + }, + "data": req.data, + }.encode_json() + ).with({ + "Header": { + "Authorization": ["Bearer " + token.access_token], + "Accept": ["application/json"], + "Content-Type": ["application/json"], + } + }).do_request().as(resp, resp.StatusCode == 200 ? + bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ? + { + "events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(), + "cursor": { + "last": ([now] + body.data.map(e, + e[state.data_path].map(l, + l[state.time_field].parse_time(["2006-01-02T15:04:05-0700", time_layout.RFC3339]) + ) + ).flatten()).max().format(time_layout.RFC3339), + }, + ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ? + optional.of({ + ?"next": body.?meta.pagination.next, + "data": req.data, + }) + : + optional.none(), + "token": { + "access_token": token.access_token, + "expires": token.expires, + }, + "want_more": has(body.?meta.pagination.next) && size(body.data) != 0, + } + : + // Mimecast can return failure states with a 200. This + // is detected by a non-empty fail array at the root + // of the response body. Don't attempt to parse this + // out, just dump the whole body into the error message. + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "POST " + state.path + ":" + string(resp.Body), // We know this is not empty. + }, + }, + "want_more": false, + } + ) + : + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "POST " + state.path + ": " + ( + size(resp.Body) != 0 ? + string(resp.Body) + : + string(resp.Status) + ' (' + string(resp.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ) + ) + ) + redact: + fields: + - client_id + - client_secret + - token.access_token + resource.url: https://api.services.mimecast.com + state: + client_id: null + client_secret: null + data_path: attachmentLogs + end_field: to + look_back: 24h + page_size: 100 + path: /api/ttp/attachment/get-logs + start_field: from + time_field: date tags: - forwarded - mimecast-ttp-ap - - condition: ${kubernetes.hints.mimecast.ttp_ip_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_date: - value: '[[.first_event.eventTime]]' + - condition: ${kubernetes.hints.mimecast.ttp_ip_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 data_stream: dataset: mimecast.ttp_ip_logs type: logs + fields_under_root: true interval: 5m - request.method: POST - request.transforms: - - set: - default: '[{"oldestFirst": false,"to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' - target: body.data - value: '[{"oldestFirst": false,"to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[.cursor.next_date]]"}]' - value_type: json - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/impersonation/get-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - request.url: https://eu-api.mimecast.com/api/ttp/impersonation/get-logs - response.decode_as: application/json - response.pagination: - - delete: - target: body.data - - set: - fail_on_template_error: true - target: body.meta.pagination.pageToken - value: '[[.last_response.body.meta.pagination.next]]' - response.split: - ignore_empty_value: true - split: - target: body.impersonationLogs - target: body.data + keep_null: true + program: | + // This program is shared amongst archive_search_logs, dlp_logs, + // message_release_logs, ttp_ap_logs, ttp_ip_logs, and ttp_url_logs. + // If it is changed here changes should be reflected in the other + // data streams. Do not differentiate the logic between these data + // streams lightly; use the state variable for this unless absolutely + // required. + state.with( + ( + (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? + // The token we have is still valid. + state.token + : + // Get a new token. + post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", + { + "client_id": [state.client_id], + "client_secret": [state.client_secret], + "grant_type": ["client_credentials"], + }.format_query() + ).do_request().as(auth, auth.StatusCode == 200 ? + bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ + // Include 60s grace period to avoid attempting to make + // a request with a stale authentication token. + "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), + })) + : + { + "events": { + "error": { + "code": string(auth.StatusCode), + "id": string(auth.Status), + "message": "POST /oauth/token: "+( + size(auth.Body) != 0 ? + string(auth.Body) + : + string(auth.Status) + ' (' + string(auth.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ).as(token, !has(token.access_token) ? token : + { + "data": state.?last_page.data.orValue([{ + state.start_field: state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339)), + state.end_field: now.format(time_layout.RFC3339), + }]), + }.as(req, + post_request(state.url.trim_right("/") + state.path, "application/json", + { + "meta": { + "pagination": { + "pageSize": state.page_size, + ?"pageToken": state.?last_page.next, + } + }, + "data": req.data, + }.encode_json() + ).with({ + "Header": { + "Authorization": ["Bearer " + token.access_token], + "Accept": ["application/json"], + "Content-Type": ["application/json"], + } + }).do_request().as(resp, resp.StatusCode == 200 ? + bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ? + { + "events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(), + "cursor": { + "last": ([now] + body.data.map(e, + e[state.data_path].map(l, + l[state.time_field].parse_time(["2006-01-02T15:04:05-0700", time_layout.RFC3339]) + ) + ).flatten()).max().format(time_layout.RFC3339), + }, + ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ? + optional.of({ + ?"next": body.?meta.pagination.next, + "data": req.data, + }) + : + optional.none(), + "token": { + "access_token": token.access_token, + "expires": token.expires, + }, + "want_more": has(body.?meta.pagination.next) && size(body.data) != 0, + } + : + // Mimecast can return failure states with a 200. This + // is detected by a non-empty fail array at the root + // of the response body. Don't attempt to parse this + // out, just dump the whole body into the error message. + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "POST " + state.path + ":" + string(resp.Body), // We know this is not empty. + }, + }, + "want_more": false, + } + ) + : + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "POST " + state.path + ": " + ( + size(resp.Body) != 0 ? + string(resp.Body) + : + string(resp.Status) + ' (' + string(resp.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ) + ) + ) + redact: + fields: + - client_id + - client_secret + - token.access_token + resource.url: https://api.services.mimecast.com + state: + client_id: null + client_secret: null + data_path: impersonationLogs + end_field: to + look_back: 24h + page_size: 100 + path: /api/ttp/impersonation/get-logs + start_field: from + time_field: eventTime tags: - forwarded - mimecast-ttp-ip - - condition: ${kubernetes.hints.mimecast.ttp_url_logs.enabled} == true or ${kubernetes.hints.mimecast.enabled} == true - config_version: "2" - cursor: - next_date: - value: '[[.first_event.date]]' + - condition: ${kubernetes.hints.mimecast.ttp_url_logs.enabled} == true and ${kubernetes.hints.mimecast.enabled} == true + config_version: 2 data_stream: dataset: mimecast.ttp_url_logs type: logs + fields_under_root: true interval: 5m - request.method: POST - request.transforms: - - set: - default: '[{"oldestFirst": false,"scanResult": "all","route":"all","to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05-0700"]]"}]' - target: body.data - value: '[{"oldestFirst": false,"scanResult": "all","route":"all","to": "[[formatDate (now) "2006-01-02T15:04:05-0700"]]", "from":"[[.cursor.next_date]]"}]' - value_type: json - - set: - target: header.x-mc-app-id - value: null - - set: - target: header.x-mc-date - value: '[[formatDate (now) "RFC1123"]]' - - set: - target: header.x-mc-req-id - value: '[[uuid]]' - - set: - fail_on_template_error: true - target: header.Authorization - value: MC :[[hmacBase64 "sha1" (base64Decode "") (sprintf "%s:%s:/api/ttp/url/get-logs:" (.header.Get "x-mc-date") (.header.Get "x-mc-req-id"))]] - request.url: https://eu-api.mimecast.com/api/ttp/url/get-logs - response.decode_as: application/json - response.pagination: - - delete: - target: body.data - - set: - fail_on_template_error: true - target: body.meta.pagination.pageToken - value: '[[.last_response.body.meta.pagination.next]]' - response.split: - ignore_empty_value: true - split: - target: body.clickLogs - target: body.data + keep_null: true + program: | + // This program is shared amongst archive_search_logs, dlp_logs, + // message_release_logs, ttp_ap_logs, ttp_ip_logs, and ttp_url_logs. + // If it is changed here changes should be reflected in the other + // data streams. Do not differentiate the logic between these data + // streams lightly; use the state variable for this unless absolutely + // required. + state.with( + ( + (has(state.?token.expires) && now() < timestamp(state.token.expires)) ? + // The token we have is still valid. + state.token + : + // Get a new token. + post_request(state.url.trim_right("/") + "/oauth/token", "application/x-www-form-urlencoded", + { + "client_id": [state.client_id], + "client_secret": [state.client_secret], + "grant_type": ["client_credentials"], + }.format_query() + ).do_request().as(auth, auth.StatusCode == 200 ? + bytes(auth.Body).decode_json().as(auth_body, auth_body.with({ + // Include 60s grace period to avoid attempting to make + // a request with a stale authentication token. + "expires": now()+duration(string(int(auth_body.expires_in)-60)+"s"), + })) + : + { + "events": { + "error": { + "code": string(auth.StatusCode), + "id": string(auth.Status), + "message": "POST /oauth/token: "+( + size(auth.Body) != 0 ? + string(auth.Body) + : + string(auth.Status) + ' (' + string(auth.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ).as(token, !has(token.access_token) ? token : + { + "data": state.?last_page.data.orValue([{ + state.start_field: state.?cursor.last.orValue((now - duration(state.look_back)).format(time_layout.RFC3339)), + state.end_field: now.format(time_layout.RFC3339), + }]), + }.as(req, + post_request(state.url.trim_right("/") + state.path, "application/json", + { + "meta": { + "pagination": { + "pageSize": state.page_size, + ?"pageToken": state.?last_page.next, + } + }, + "data": req.data, + }.encode_json() + ).with({ + "Header": { + "Authorization": ["Bearer " + token.access_token], + "Accept": ["application/json"], + "Content-Type": ["application/json"], + } + }).do_request().as(resp, resp.StatusCode == 200 ? + bytes(resp.Body).decode_json().as(body, body.?fail.orValue([]).size() == 0 ? + { + "events": body.data.map(e, e[state.data_path].map(l, {"message": l.encode_json()})).flatten(), + "cursor": { + "last": ([now] + body.data.map(e, + e[state.data_path].map(l, + l[state.time_field].parse_time(["2006-01-02T15:04:05-0700", time_layout.RFC3339]) + ) + ).flatten()).max().format(time_layout.RFC3339), + }, + ?"last_page": has(body.?meta.pagination.next) && size(body.data) != 0 ? + optional.of({ + ?"next": body.?meta.pagination.next, + "data": req.data, + }) + : + optional.none(), + "token": { + "access_token": token.access_token, + "expires": token.expires, + }, + "want_more": has(body.?meta.pagination.next) && size(body.data) != 0, + } + : + // Mimecast can return failure states with a 200. This + // is detected by a non-empty fail array at the root + // of the response body. Don't attempt to parse this + // out, just dump the whole body into the error message. + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "POST " + state.path + ":" + string(resp.Body), // We know this is not empty. + }, + }, + "want_more": false, + } + ) + : + { + "events": { + "error": { + "code": string(resp.StatusCode), + "id": string(resp.Status), + "message": "POST " + state.path + ": " + ( + size(resp.Body) != 0 ? + string(resp.Body) + : + string(resp.Status) + ' (' + string(resp.StatusCode) + ')' + ), + }, + }, + "want_more": false, + } + ) + ) + ) + ) + redact: + fields: + - client_id + - client_secret + - token.access_token + resource.url: https://api.services.mimecast.com + state: + client_id: null + client_secret: null + data_path: clickLogs + end_field: to + look_back: 24h + page_size: 100 + path: /api/ttp/url/get-logs + start_field: from + time_field: date tags: - forwarded - mimecast-ttp-url data_stream.namespace: default + - name: filestream-mimecast + id: filestream-mimecast-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.mimecast.container_logs.enabled} == true + data_stream: + dataset: mimecast.container_logs + type: logs + exclude_files: [] + exclude_lines: [] + parsers: + - container: + format: auto + stream: all + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + symlinks: true + tags: [] + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml index b0bd0b07245..487da80360e 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/mongodb.yml @@ -42,7 +42,7 @@ inputs: password: ${kubernetes.hints.mongodb.collstats.password|kubernetes.hints.mongodb.password|''} period: ${kubernetes.hints.mongodb.collstats.period|kubernetes.hints.mongodb.period|'10s'} ssl.certificate: null - ssl.enabled: false + ssl.enabled: null ssl.key: null ssl.verification_mode: null username: ${kubernetes.hints.mongodb.collstats.username|kubernetes.hints.mongodb.username|''} @@ -57,7 +57,7 @@ inputs: password: ${kubernetes.hints.mongodb.dbstats.password|kubernetes.hints.mongodb.password|''} period: ${kubernetes.hints.mongodb.dbstats.period|kubernetes.hints.mongodb.period|'10s'} ssl.certificate: null - ssl.enabled: false + ssl.enabled: null ssl.key: null ssl.verification_mode: null username: ${kubernetes.hints.mongodb.dbstats.username|kubernetes.hints.mongodb.username|''} diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml index b21edd74269..3316d14994f 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/mysql.yml @@ -1,4 +1,43 @@ inputs: + - name: mysql/metrics-mysql + id: mysql/metrics-mysql-${kubernetes.hints.container_id} + type: mysql/metrics + use_output: default + streams: + - condition: ${kubernetes.hints.mysql.galera_status.enabled} == true and ${kubernetes.hints.mysql.enabled} == true + data_stream: + dataset: mysql.galera_status + type: metrics + hosts: + - ${kubernetes.hints.mysql.galera_status.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} + metricsets: + - galera_status + password: ${kubernetes.hints.mysql.galera_status.password|kubernetes.hints.mysql.password|'test'} + period: ${kubernetes.hints.mysql.galera_status.period|kubernetes.hints.mysql.period|'10s'} + username: ${kubernetes.hints.mysql.galera_status.username|kubernetes.hints.mysql.username|'root'} + - condition: ${kubernetes.hints.mysql.performance.enabled} == true or ${kubernetes.hints.mysql.enabled} == true + data_stream: + dataset: mysql.performance + type: metrics + hosts: + - ${kubernetes.hints.mysql.performance.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} + metricsets: + - performance + password: ${kubernetes.hints.mysql.performance.password|kubernetes.hints.mysql.password|'test'} + period: ${kubernetes.hints.mysql.performance.period|kubernetes.hints.mysql.period|'10s'} + username: ${kubernetes.hints.mysql.performance.username|kubernetes.hints.mysql.username|'root'} + - condition: ${kubernetes.hints.mysql.status.enabled} == true or ${kubernetes.hints.mysql.enabled} == true + data_stream: + dataset: mysql.status + type: metrics + hosts: + - ${kubernetes.hints.mysql.status.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} + metricsets: + - status + password: ${kubernetes.hints.mysql.status.password|kubernetes.hints.mysql.password|'test'} + period: ${kubernetes.hints.mysql.status.period|kubernetes.hints.mysql.period|'10s'} + username: ${kubernetes.hints.mysql.status.username|kubernetes.hints.mysql.username|'root'} + data_stream.namespace: default - name: filestream-mysql id: filestream-mysql-${kubernetes.hints.container_id} type: filestream @@ -56,42 +95,3 @@ inputs: enabled: true symlinks: true data_stream.namespace: default - - name: mysql/metrics-mysql - id: mysql/metrics-mysql-${kubernetes.hints.container_id} - type: mysql/metrics - use_output: default - streams: - - condition: ${kubernetes.hints.mysql.galera_status.enabled} == true and ${kubernetes.hints.mysql.enabled} == true - data_stream: - dataset: mysql.galera_status - type: metrics - hosts: - - ${kubernetes.hints.mysql.galera_status.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} - metricsets: - - galera_status - password: ${kubernetes.hints.mysql.galera_status.password|kubernetes.hints.mysql.password|'test'} - period: ${kubernetes.hints.mysql.galera_status.period|kubernetes.hints.mysql.period|'10s'} - username: ${kubernetes.hints.mysql.galera_status.username|kubernetes.hints.mysql.username|'root'} - - condition: ${kubernetes.hints.mysql.performance.enabled} == true or ${kubernetes.hints.mysql.enabled} == true - data_stream: - dataset: mysql.performance - type: metrics - hosts: - - ${kubernetes.hints.mysql.performance.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} - metricsets: - - performance - password: ${kubernetes.hints.mysql.performance.password|kubernetes.hints.mysql.password|'test'} - period: ${kubernetes.hints.mysql.performance.period|kubernetes.hints.mysql.period|'10s'} - username: ${kubernetes.hints.mysql.performance.username|kubernetes.hints.mysql.username|'root'} - - condition: ${kubernetes.hints.mysql.status.enabled} == true or ${kubernetes.hints.mysql.enabled} == true - data_stream: - dataset: mysql.status - type: metrics - hosts: - - ${kubernetes.hints.mysql.status.host|kubernetes.hints.mysql.host|'tcp(127.0.0.1:3306)/'} - metricsets: - - status - password: ${kubernetes.hints.mysql.status.password|kubernetes.hints.mysql.password|'test'} - period: ${kubernetes.hints.mysql.status.period|kubernetes.hints.mysql.period|'10s'} - username: ${kubernetes.hints.mysql.status.username|kubernetes.hints.mysql.username|'root'} - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml index c75da289568..cefc4f03beb 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/nats.yml @@ -1,31 +1,4 @@ inputs: - - name: filestream-nats - id: filestream-nats-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.nats.log.enabled} == true or ${kubernetes.hints.nats.enabled} == true - data_stream: - dataset: nats.log - type: logs - exclude_files: - - .gz$ - file_identity: - fingerprint: null - parsers: - - container: - format: auto - stream: ${kubernetes.hints.nats.log.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - fingerprint: - enabled: true - symlinks: true - tags: - - nats-log - data_stream.namespace: default - name: nats/metrics-nats id: nats/metrics-nats-${kubernetes.hints.container_id} type: nats/metrics @@ -86,3 +59,30 @@ inputs: - subscriptions period: ${kubernetes.hints.nats.subscriptions.period|kubernetes.hints.nats.period|'10s'} data_stream.namespace: default + - name: filestream-nats + id: filestream-nats-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.nats.log.enabled} == true or ${kubernetes.hints.nats.enabled} == true + data_stream: + dataset: nats.log + type: logs + exclude_files: + - .gz$ + file_identity: + fingerprint: null + parsers: + - container: + format: auto + stream: ${kubernetes.hints.nats.log.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + fingerprint: + enabled: true + symlinks: true + tags: + - nats-log + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/panw.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/panw.yml index 76bf6396568..8fd2a91b723 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/panw.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/panw.yml @@ -1,41 +1,4 @@ inputs: - - name: tcp-panw - id: tcp-panw-${kubernetes.hints.container_id} - type: tcp - use_output: default - streams: - - condition: ${kubernetes.hints.panw.panos.enabled} == true or ${kubernetes.hints.panw.enabled} == true - data_stream: - dataset: panw.panos - type: logs - framing: rfc6587 - host: localhost:9001 - max_message_size: 50KiB - processors: - - add_locale: null - - copy_fields: - fields: - - from: '@timestamp' - to: event.created - - syslog: - field: message - format: auto - timezone: Local - - add_fields: - fields: - internal_zones: - - trust - tz_offset: Local - target: _conf - - add_fields: - fields: - external_zones: - - untrust - target: _conf - tags: - - panw-panos - - forwarded - data_stream.namespace: default - name: udp-panw id: udp-panw-${kubernetes.hints.container_id} type: udp @@ -114,3 +77,40 @@ inputs: - panw-panos - forwarded data_stream.namespace: default + - name: tcp-panw + id: tcp-panw-${kubernetes.hints.container_id} + type: tcp + use_output: default + streams: + - condition: ${kubernetes.hints.panw.panos.enabled} == true or ${kubernetes.hints.panw.enabled} == true + data_stream: + dataset: panw.panos + type: logs + framing: rfc6587 + host: localhost:9001 + max_message_size: 50KiB + processors: + - add_locale: null + - copy_fields: + fields: + - from: '@timestamp' + to: event.created + - syslog: + field: message + format: auto + timezone: Local + - add_fields: + fields: + internal_zones: + - trust + tz_offset: Local + target: _conf + - add_fields: + fields: + external_zones: + - untrust + target: _conf + tags: + - panw-panos + - forwarded + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/pfsense.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/pfsense.yml index f21316d7581..96708c77e38 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/pfsense.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/pfsense.yml @@ -1,4 +1,24 @@ inputs: + - name: tcp-pfsense + id: tcp-pfsense-${kubernetes.hints.container_id} + type: tcp + use_output: default + streams: + - condition: ${kubernetes.hints.pfsense.log.enabled} == true and ${kubernetes.hints.pfsense.enabled} == true + data_stream: + dataset: pfsense.log + type: logs + host: localhost:9001 + processors: + - add_locale: null + - add_fields: + fields: + tz_offset: local + target: _tmp + tags: + - pfsense + - forwarded + data_stream.namespace: default - name: filestream-pfsense id: filestream-pfsense-${kubernetes.hints.container_id} type: filestream @@ -43,23 +63,3 @@ inputs: - pfsense - forwarded data_stream.namespace: default - - name: tcp-pfsense - id: tcp-pfsense-${kubernetes.hints.container_id} - type: tcp - use_output: default - streams: - - condition: ${kubernetes.hints.pfsense.log.enabled} == true and ${kubernetes.hints.pfsense.enabled} == true - data_stream: - dataset: pfsense.log - type: logs - host: localhost:9001 - processors: - - add_locale: null - - add_fields: - fields: - tz_offset: local - target: _tmp - tags: - - pfsense - - forwarded - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/postgresql.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/postgresql.yml index 8099386204d..ed979e12fc7 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/postgresql.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/postgresql.yml @@ -1,35 +1,4 @@ inputs: - - name: filestream-postgresql - id: filestream-postgresql-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.postgresql.log.enabled} == true or ${kubernetes.hints.postgresql.enabled} == true - data_stream: - dataset: postgresql.log - type: logs - exclude_files: - - .gz$ - file_identity: - fingerprint: null - multiline: - match: after - negate: true - pattern: '^\d{4}-\d{2}-\d{2} ' - parsers: - - container: - format: auto - stream: ${kubernetes.hints.postgresql.log.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - fingerprint: - enabled: true - symlinks: true - tags: - - postgresql-log - data_stream.namespace: default - name: postgresql/metrics-postgresql id: postgresql/metrics-postgresql-${kubernetes.hints.container_id} type: postgresql/metrics @@ -80,3 +49,34 @@ inputs: period: ${kubernetes.hints.postgresql.statement.period|kubernetes.hints.postgresql.period|'10s'} username: ${kubernetes.hints.postgresql.statement.username|kubernetes.hints.postgresql.username|''} data_stream.namespace: default + - name: filestream-postgresql + id: filestream-postgresql-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.postgresql.log.enabled} == true or ${kubernetes.hints.postgresql.enabled} == true + data_stream: + dataset: postgresql.log + type: logs + exclude_files: + - .gz$ + file_identity: + fingerprint: null + multiline: + match: after + negate: true + pattern: '^\d{4}-\d{2}-\d{2} ' + parsers: + - container: + format: auto + stream: ${kubernetes.hints.postgresql.log.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + fingerprint: + enabled: true + symlinks: true + tags: + - postgresql-log + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/prometheus.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/prometheus.yml index 43c1ade916a..e5b613a4804 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/prometheus.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/prometheus.yml @@ -59,9 +59,9 @@ inputs: - remote_write port: 9201 rate_counters: true - ssl.certificate: null + ssl.certificate: /etc/pki/server/cert.pem ssl.enabled: null - ssl.key: /etc/pki/server/cert.key + ssl.key: null types_patterns.exclude: null types_patterns.include: null use_types: true diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/redis.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/redis.yml index a1321a6880c..c8a26e160f4 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/redis.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/redis.yml @@ -1,4 +1,33 @@ inputs: + - name: filestream-redis + id: filestream-redis-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.redis.log.enabled} == true or ${kubernetes.hints.redis.enabled} == true + data_stream: + dataset: redis.log + type: logs + exclude_files: + - .gz$ + exclude_lines: + - ^\s+[\-`('.|_] + file_identity: + fingerprint: null + parsers: + - container: + format: auto + stream: ${kubernetes.hints.redis.log.stream|'all'} + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + fingerprint: + enabled: true + symlinks: true + tags: + - redis-log + data_stream.namespace: default - name: redis-redis id: redis-redis-${kubernetes.hints.container_id} type: redis @@ -60,32 +89,3 @@ inputs: password: ${kubernetes.hints.redis.keyspace.password|kubernetes.hints.redis.password|''} period: ${kubernetes.hints.redis.keyspace.period|kubernetes.hints.redis.period|'10s'} data_stream.namespace: default - - name: filestream-redis - id: filestream-redis-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.redis.log.enabled} == true or ${kubernetes.hints.redis.enabled} == true - data_stream: - dataset: redis.log - type: logs - exclude_files: - - .gz$ - exclude_lines: - - ^\s+[\-`('.|_] - file_identity: - fingerprint: null - parsers: - - container: - format: auto - stream: ${kubernetes.hints.redis.log.stream|'all'} - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - fingerprint: - enabled: true - symlinks: true - tags: - - redis-log - data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/sentinel_one.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/sentinel_one.yml index 7e7c6e3de88..8557717a5db 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/sentinel_one.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/sentinel_one.yml @@ -1,26 +1,4 @@ inputs: - - name: filestream-sentinel_one - id: filestream-sentinel_one-${kubernetes.hints.container_id} - type: filestream - use_output: default - streams: - - condition: ${kubernetes.hints.sentinel_one.container_logs.enabled} == true - data_stream: - dataset: sentinel_one.container_logs - type: logs - exclude_files: [] - exclude_lines: [] - parsers: - - container: - format: auto - stream: all - paths: - - /var/log/containers/*${kubernetes.hints.container_id}.log - prospector: - scanner: - symlinks: true - tags: [] - data_stream.namespace: default - name: httpjson-sentinel_one id: httpjson-sentinel_one-${kubernetes.hints.container_id} type: httpjson @@ -217,3 +195,25 @@ inputs: - forwarded - sentinel_one-threat data_stream.namespace: default + - name: filestream-sentinel_one + id: filestream-sentinel_one-${kubernetes.hints.container_id} + type: filestream + use_output: default + streams: + - condition: ${kubernetes.hints.sentinel_one.container_logs.enabled} == true + data_stream: + dataset: sentinel_one.container_logs + type: logs + exclude_files: [] + exclude_lines: [] + parsers: + - container: + format: auto + stream: all + paths: + - /var/log/containers/*${kubernetes.hints.container_id}.log + prospector: + scanner: + symlinks: true + tags: [] + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/synthetics.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/synthetics.yml index 5127a4ba11d..0cd225db65e 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/synthetics.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/synthetics.yml @@ -1,55 +1,4 @@ inputs: - - name: synthetics/http-synthetics - id: synthetics/http-synthetics-${kubernetes.hints.container_id} - type: synthetics/http - use_output: default - streams: - - __ui: null - check.request.method: null - condition: ${kubernetes.hints.synthetics.http.enabled} == true and ${kubernetes.hints.synthetics.enabled} == true - data_stream: - dataset: http - type: synthetics - enabled: true - ipv4: true - ipv6: true - max_attempts: 2 - max_redirects: null - name: null - password: ${kubernetes.hints.synthetics.http.password|kubernetes.hints.synthetics.password|''} - response.include_body: null - response.include_headers: null - run_from.geo.name: Fleet managed - run_from.id: fleet_managed - schedule: '@every 3m' - timeout: ${kubernetes.hints.synthetics.http.timeout|kubernetes.hints.synthetics.timeout|''} - type: http - urls: null - username: ${kubernetes.hints.synthetics.http.username|kubernetes.hints.synthetics.username|''} - data_stream.namespace: default - - name: synthetics/tcp-synthetics - id: synthetics/tcp-synthetics-${kubernetes.hints.container_id} - type: synthetics/tcp - use_output: default - streams: - - __ui: null - condition: ${kubernetes.hints.synthetics.tcp.enabled} == true and ${kubernetes.hints.synthetics.enabled} == true - data_stream: - dataset: tcp - type: synthetics - enabled: true - hosts: ${kubernetes.hints.synthetics.tcp.host|kubernetes.hints.synthetics.host|''} - ipv4: true - ipv6: true - max_attempts: 2 - name: null - proxy_use_local_resolver: false - run_from.geo.name: Fleet managed - run_from.id: fleet_managed - schedule: '@every 3m' - timeout: ${kubernetes.hints.synthetics.tcp.timeout|kubernetes.hints.synthetics.timeout|''} - type: tcp - data_stream.namespace: default - name: synthetics/icmp-synthetics id: synthetics/icmp-synthetics-${kubernetes.hints.container_id} type: synthetics/icmp @@ -133,3 +82,54 @@ inputs: symlinks: true tags: [] data_stream.namespace: default + - name: synthetics/http-synthetics + id: synthetics/http-synthetics-${kubernetes.hints.container_id} + type: synthetics/http + use_output: default + streams: + - __ui: null + check.request.method: null + condition: ${kubernetes.hints.synthetics.http.enabled} == true and ${kubernetes.hints.synthetics.enabled} == true + data_stream: + dataset: http + type: synthetics + enabled: true + ipv4: true + ipv6: true + max_attempts: 2 + max_redirects: null + name: null + password: ${kubernetes.hints.synthetics.http.password|kubernetes.hints.synthetics.password|''} + response.include_body: null + response.include_headers: null + run_from.geo.name: Fleet managed + run_from.id: fleet_managed + schedule: '@every 3m' + timeout: ${kubernetes.hints.synthetics.http.timeout|kubernetes.hints.synthetics.timeout|''} + type: http + urls: null + username: ${kubernetes.hints.synthetics.http.username|kubernetes.hints.synthetics.username|''} + data_stream.namespace: default + - name: synthetics/tcp-synthetics + id: synthetics/tcp-synthetics-${kubernetes.hints.container_id} + type: synthetics/tcp + use_output: default + streams: + - __ui: null + condition: ${kubernetes.hints.synthetics.tcp.enabled} == true and ${kubernetes.hints.synthetics.enabled} == true + data_stream: + dataset: tcp + type: synthetics + enabled: true + hosts: ${kubernetes.hints.synthetics.tcp.host|kubernetes.hints.synthetics.host|''} + ipv4: true + ipv6: true + max_attempts: 2 + name: null + proxy_use_local_resolver: false + run_from.geo.name: Fleet managed + run_from.id: fleet_managed + schedule: '@every 3m' + timeout: ${kubernetes.hints.synthetics.tcp.timeout|kubernetes.hints.synthetics.timeout|''} + type: tcp + data_stream.namespace: default diff --git a/deploy/kubernetes/elastic-agent-standalone/templates.d/udp.yml b/deploy/kubernetes/elastic-agent-standalone/templates.d/udp.yml index 1204c4e7e9c..bc21b484f27 100644 --- a/deploy/kubernetes/elastic-agent-standalone/templates.d/udp.yml +++ b/deploy/kubernetes/elastic-agent-standalone/templates.d/udp.yml @@ -1,4 +1,17 @@ inputs: + - name: udp-udp + id: udp-udp-${kubernetes.hints.container_id} + type: udp + use_output: default + streams: + - condition: ${kubernetes.hints.udp.generic.enabled} == true or ${kubernetes.hints.udp.enabled} == true + data_stream: + dataset: udp.generic + type: logs + host: localhost:8080 + max_message_size: 10KiB + timeout: ${kubernetes.hints.udp.generic.timeout|kubernetes.hints.udp.timeout|''} + data_stream.namespace: default - name: filestream-udp id: filestream-udp-${kubernetes.hints.container_id} type: filestream @@ -21,16 +34,3 @@ inputs: symlinks: true tags: [] data_stream.namespace: default - - name: udp-udp - id: udp-udp-${kubernetes.hints.container_id} - type: udp - use_output: default - streams: - - condition: ${kubernetes.hints.udp.generic.enabled} == true or ${kubernetes.hints.udp.enabled} == true - data_stream: - dataset: udp.generic - type: logs - host: localhost:8080 - max_message_size: 10KiB - timeout: ${kubernetes.hints.udp.generic.timeout|kubernetes.hints.udp.timeout|''} - data_stream.namespace: default