ti_misp: Fix ECS date mapping on threat fields. (#10638)

Fix ECS date mapping for threat fields. ecs@mappings component template is missing threat fields mapped as date. Example: fields such as first_seen, last_seen, modified_at are being mapped as keyword in transform's source datastream-backed indices. The transform's destination indices are not effected as they are not datastream-backed and mappings are explicitly defined as date. This causes field type conflicts. - Explicitly add ECS threat fields that are of type date into source data-stream backed fields. - Fix first_seen, last_seen date processors with microseconds. - Add missing first_seen, last_seen fields.
elastic · Jul 30, 2024 · b8d328c · b8d328c
1 parent 1ef788e
commit b8d328c
Show file tree

Hide file tree

Showing 11 changed files with 90 additions and 19 deletions.
diff --git a/packages/ti_misp/_dev/deploy/docker/files/config.yml b/packages/ti_misp/_dev/deploy/docker/files/config.yml
@@ -414,8 +414,8 @@ rules:
                   "comment": "",
                   "deleted": false,
                   "disable_correlation": false,
-                  "first_seen": null,
-                  "last_seen": null,
+                  "first_seen": "1701984000000000",
+                  "last_seen": "1715694000000000",
                   "value": "https://gist.githubusercontent.com/andrewsmhay/de1cdc63d04c2bbf8c12/raw/f20402cf5a0c646c63c4521f60587703fe654443/iplist",
                   "Event": {
                     "org_id": "1",
@@ -441,8 +441,8 @@ rules:
                   "comment": "",
                   "deleted": false,
                   "disable_correlation": false,
-                  "first_seen": null,
-                  "last_seen": null,
+                  "first_seen": "1714694000000000",
+                  "last_seen": "1720694000000000",
                   "value": "Shellshock",
                   "Event": {
                     "org_id": "1",

diff --git a/packages/ti_misp/changelog.yml b/packages/ti_misp/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.35.1"
+  changes:
+    - description: Fix ECS date mapping on threat fields.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10638
 - version: "1.35.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/ti_misp/data_stream/threat/fields/ecs.yml b/packages/ti_misp/data_stream/threat/fields/ecs.yml
@@ -0,0 +1,6 @@
+- external: ecs
+  name: threat.indicator.first_seen
+- external: ecs
+  name: threat.indicator.last_seen
+- external: ecs
+  name: threat.indicator.modified_at
diff --git a/...ata_stream/threat_attributes/_dev/test/pipeline/test-misp-sample-ndjson.log-expected.json b/...ata_stream/threat_attributes/_dev/test/pipeline/test-misp-sample-ndjson.log-expected.json
@@ -38,7 +38,9 @@
                     "distribution": 0,
                     "event_id": "12345",
                     "event_uuid": "c99506a6-1255-4b71-afa5-7b8ba48c3b1b",
+                    "first_seen": "1581984000000",
                     "id": "12345",
+                    "last_seen": "1581984000000",
                     "object_id": "12345",
                     "object_relation": "sensor",
                     "sharing_group_id": "1",
@@ -121,8 +123,8 @@
                             "md5": "127.0.0.1"
                         }
                     },
-                    "first_seen": "1581984000000000",
-                    "last_seen": "1581984000000000",
+                    "first_seen": "2020-02-18T00:00:00.000Z",
+                    "last_seen": "2020-02-18T00:00:00.000Z",
                     "marking": {
                         "tlp": [
                             "WHITE"

diff --git a/packages/ti_misp/data_stream/threat_attributes/elasticsearch/ingest_pipeline/default.yml b/packages/ti_misp/data_stream/threat_attributes/elasticsearch/ingest_pipeline/default.yml
@@ -199,14 +199,38 @@ processors:
   - set:
       field: threat.feed.name
       value: "MISP"
-  - rename:
+  - script:
+      description: Convert microseconds to millseconds.
+      lang: painless
+      source: |
+        if (ctx.misp?.attribute?.first_seen != null && ctx.misp.attribute.first_seen.length() > 3) {
+            ctx.misp.attribute.first_seen = ctx.misp.attribute.first_seen.substring(0, ctx.misp.attribute.first_seen.length() - 3)
+        }
+        if (ctx.misp?.attribute?.last_seen != null && ctx.misp.attribute.last_seen.length() > 3) {
+            ctx.misp.attribute.last_seen = ctx.misp.attribute.last_seen.substring(0, ctx.misp.attribute.last_seen.length() - 3)
+        }
+  - date:
       field: misp.attribute.first_seen
       target_field: threat.indicator.first_seen
-      ignore_missing: true
-  - rename:
+      tag: date_attribute_first_seen
+      if: ctx.misp?.attribute?.first_seen != null
+      formats:
+        - UNIX_MS
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
+  - date:
       field: misp.attribute.last_seen
       target_field: threat.indicator.last_seen
-      ignore_missing: true
+      tag: date_attribute_last_seen
+      if: ctx.misp?.attribute?.last_seen != null
+      formats:
+        - UNIX_MS
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - convert:
       field: misp.event.analysis
       type: long

diff --git a/packages/ti_misp/data_stream/threat_attributes/fields/ecs.yml b/packages/ti_misp/data_stream/threat_attributes/fields/ecs.yml
@@ -1,2 +1,8 @@
 - name: threat.indicator.email.subject
   type: keyword
+- external: ecs
+  name: threat.indicator.first_seen
+- external: ecs
+  name: threat.indicator.last_seen
+- external: ecs
+  name: threat.indicator.modified_at
diff --git a/packages/ti_misp/data_stream/threat_attributes/fields/fields.yml b/packages/ti_misp/data_stream/threat_attributes/fields/fields.yml
@@ -199,6 +199,16 @@
           description: >
             Timestamp when the document is decayed. Not sent by the API. This is calculated inside the ingest pipeline.
 
+        - name: first_seen
+          type: keyword
+          description: >
+            The first time the attribute was seen.
+
+        - name: last_seen
+          type: keyword
+          description: >
+            The last time the attribute was seen.
+
     - name: object
       type: group
       description: >

diff --git a/packages/ti_misp/data_stream/threat_attributes/sample_event.json b/packages/ti_misp/data_stream/threat_attributes/sample_event.json
@@ -1,33 +1,33 @@
 {
     "@timestamp": "2014-10-03T07:14:05.000Z",
     "agent": {
-        "ephemeral_id": "fe458061-e572-49d2-91fc-cd784bf66d09",
-        "id": "e4354c0c-ca75-448a-b886-ec73a12bce07",
+        "ephemeral_id": "6b45096a-f41c-4410-879d-e04a56b22bb2",
+        "id": "0eb83218-5f40-45bd-8fb3-9423008f7b6f",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.11.0"
+        "version": "8.14.3"
     },
     "data_stream": {
         "dataset": "ti_misp.threat_attributes",
-        "namespace": "ep",
+        "namespace": "89460",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "e4354c0c-ca75-448a-b886-ec73a12bce07",
+        "id": "0eb83218-5f40-45bd-8fb3-9423008f7b6f",
         "snapshot": false,
-        "version": "8.11.0"
+        "version": "8.14.3"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "threat"
         ],
-        "created": "2023-12-21T08:19:07.017Z",
+        "created": "2024-07-29T13:33:33.711Z",
         "dataset": "ti_misp.threat_attributes",
-        "ingested": "2023-12-21T08:19:09Z",
+        "ingested": "2024-07-29T13:33:45Z",
         "kind": "enrichment",
         "original": "{\"Event\":{\"distribution\":\"3\",\"id\":\"1\",\"info\":\"OSINT ShellShock scanning IPs from OpenDNS\",\"org_id\":\"1\",\"orgc_id\":\"2\",\"uuid\":\"542e4c9c-cadc-4f8f-bb11-6d13950d210b\"},\"category\":\"External analysis\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"1\",\"first_seen\":null,\"id\":\"1\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1412320445\",\"to_ids\":false,\"type\":\"link\",\"uuid\":\"542e4cbd-ee78-4a57-bfb8-1fda950d210b\",\"value\":\"http://labs.opendns.com/2014/10/02/opendns-and-bash/\"}",
         "type": [

diff --git a/packages/ti_misp/docs/README.md b/packages/ti_misp/docs/README.md
@@ -103,6 +103,9 @@ The filters themselves are based on the [MISP API documentation](https://www.cir
 | misp.orgc.uuid | The Organization Community UUID in which the event object was reported from. | keyword |
 | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword |
 | threat.feed.name | Display friendly feed name. | constant_keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
 An example event for `threat` looks as following:
@@ -260,7 +263,9 @@ To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_misp
 | misp.attribute.distribution | How the attribute has been distributed, represented by integer numbers. | long |
 | misp.attribute.event_id | The local event ID of the attribute. | keyword |
 | misp.attribute.event_uuid | The local event UUID of the attribute. | keyword |
+| misp.attribute.first_seen | The first time the attribute was seen. | keyword |
 | misp.attribute.id | The ID of the attribute. | keyword |
+| misp.attribute.last_seen | The last time the attribute was seen. | keyword |
 | misp.attribute.object_id | The ID of the Object in which the attribute is attached. | keyword |
 | misp.attribute.object_relation | The type of relation the attribute has with the attribute object itself. | keyword |
 | misp.attribute.sharing_group_id | The group ID of the sharing group related to the specific attribute. | keyword |
@@ -305,5 +310,8 @@ To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_misp
 | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword |
 | threat.feed.name | Display friendly feed name | constant_keyword |
 | threat.indicator.email.subject |  | keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
 
diff --git a/packages/ti_misp/elasticsearch/transform/latest_ioc/fields/fields.yml b/packages/ti_misp/elasticsearch/transform/latest_ioc/fields/fields.yml
@@ -199,6 +199,16 @@
           description: >
             Timestamp when the document is decayed. Not sent by the API. This is calculated inside the ingest pipeline.
 
+        - name: first_seen
+          type: keyword
+          description: >
+            The first time the attribute was seen.
+
+        - name: last_seen
+          type: keyword
+          description: >
+            The last time the attribute was seen.
+
     - name: object
       type: group
       description: >

diff --git a/packages/ti_misp/manifest.yml b/packages/ti_misp/manifest.yml
@@ -1,6 +1,6 @@
 name: ti_misp
 title: MISP
-version: "1.35.0"
+version: "1.35.1"
 description: Ingest threat intelligence indicators from MISP platform with Elastic Agent.
 type: integration
 format_version: "3.0.2"