qualys_vmdr.asset_host_detection: Capture issue with XML parsing into…

… error.message (elastic#12009) Captures 2 scenarios into `error.message` to give better traceability. - Invalid XML - When XML response doesn't match with XSD defined in the input.
harnish-elastic · Dec 5, 2024 · 875416c · 875416c
1 parent 0b14a8a
commit 875416c
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 23 deletions.
diff --git a/packages/qualys_vmdr/changelog.yml b/packages/qualys_vmdr/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "5.5.0"
+  changes:
+    - description: Capture error with decode_xml.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12009
 - version: "5.4.0"
   changes:
     - description: Truncate very long field values.

diff --git a/packages/qualys_vmdr/data_stream/asset_host_detection/agent/stream/input.yml.hbs b/packages/qualys_vmdr/data_stream/asset_host_detection/agent/stream/input.yml.hbs
@@ -43,15 +43,50 @@ program: |
     }).do_request().as(resp,
       resp.StatusCode == 200
       ?
-        resp.Body.as(xml, bytes(xml).decode_xml('qualys_api_2_0').as(body, {
-            "events": body.doc.HOST_LIST_VM_DETECTION_OUTPUT.RESPONSE.HOST_LIST.HOST.map(h,
-              h.DETECTION_LIST.DETECTION.map(v, {
-                "message": h.with({"DETECTION_LIST": v}).encode_json(),
+        resp.Body.as(xml, try(bytes(xml).decode_xml('qualys_api_2_0'), "decode_xml_error").as(body, 
+          !has(body.decode_xml_error) 
+          ?
+            (body.?doc.HOST_LIST_VM_DETECTION_OUTPUT.RESPONSE.HOST_LIST.HOST.hasValue() 
+            ?
+              ({
+                "events": body.doc.HOST_LIST_VM_DETECTION_OUTPUT.RESPONSE.HOST_LIST.HOST.map(h,
+                  h.DETECTION_LIST.DETECTION.map(v, {
+                    "message": h.with({"DETECTION_LIST": v}).encode_json(),
+                  })
+                ).flatten(),
+                ?"pagination_url": body.?doc.HOST_LIST_VM_DETECTION_OUTPUT.RESPONSE.WARNING.URL,
+                "want_more": body.?doc.HOST_LIST_VM_DETECTION_OUTPUT.RESPONSE.WARNING.URL.hasValue(),
               })
-            ).flatten(),
-            ?"pagination_url": body.?doc.HOST_LIST_VM_DETECTION_OUTPUT.RESPONSE.WARNING.URL,
-            "want_more": body.?doc.HOST_LIST_VM_DETECTION_OUTPUT.RESPONSE.WARNING.URL.hasValue(),
-        }))
+            :
+              {
+                "events": {
+                  "error": {
+                    "message": "xsd and response mismatch:" + (
+                      size(resp.Body) != 0 ?
+                        string(resp.Body)
+                      :
+                        "XML response missing"
+                    ),
+                  },
+                },
+                "want_more": false,
+              }
+            )
+          :
+            {
+              "events": {
+                "error": {
+                  "message": "decode_xml error:" + string(body.decode_xml_error) + ":" + (
+                    size(resp.Body) != 0 ?
+                      string(resp.Body)
+                    :
+                      "XML response missing"
+                  ),
+                },
+              },
+              "want_more": false,
+            }
+        ))
       :
         {
           "events": {

diff --git a/packages/qualys_vmdr/data_stream/asset_host_detection/sample_event.json b/packages/qualys_vmdr/data_stream/asset_host_detection/sample_event.json
@@ -1,22 +1,22 @@
 {
-    "@timestamp": "2024-09-25T21:44:26.325Z",
+    "@timestamp": "2024-12-05T11:02:04.225Z",
     "agent": {
-        "ephemeral_id": "f8145b5b-4d53-444a-bd44-2f296cf357e6",
-        "id": "efcbf604-6e25-41db-a21e-22c8227e0663",
-        "name": "elastic-agent-93250",
+        "ephemeral_id": "9c04104d-1da0-4c98-b133-0aedefdc2680",
+        "id": "dccf1148-df50-4c35-a3d7-633418e936ff",
+        "name": "elastic-agent-88337",
         "type": "filebeat",
         "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "qualys_vmdr.asset_host_detection",
-        "namespace": "88572",
+        "namespace": "85068",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "efcbf604-6e25-41db-a21e-22c8227e0663",
+        "id": "dccf1148-df50-4c35-a3d7-633418e936ff",
         "snapshot": false,
         "version": "8.13.0"
     },
@@ -26,7 +26,7 @@
             "host"
         ],
         "dataset": "qualys_vmdr.asset_host_detection",
-        "ingested": "2024-09-25T21:44:29Z",
+        "ingested": "2024-12-05T11:02:07Z",
         "kind": "alert",
         "original": "{\"DETECTION_LIST\":{\"AFFECT_RUNNING_KERNEL\":\"0\",\"FIRST_FOUND_DATETIME\":\"2021-02-05T04:50:45Z\",\"IS_DISABLED\":\"0\",\"IS_IGNORED\":\"0\",\"LAST_FIXED_DATETIME\":\"2022-12-14T06:52:57Z\",\"LAST_FOUND_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_PROCESSED_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_TEST_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_UPDATE_DATETIME\":\"2024-03-08T20:15:41Z\",\"QDS\":{\"#text\":\"35\",\"severity\":\"LOW\"},\"QDS_FACTORS\":{\"QDS_FACTOR\":[{\"#text\":\"7.7\",\"name\":\"CVSS\"},{\"#text\":\"v3.x\",\"name\":\"CVSS_version\"},{\"#text\":\"0.00232\",\"name\":\"epss\"},{\"#text\":\"AV:N/AC:L/PR:L/UI:N/S:C/C:N/I:N/A:H\",\"name\":\"CVSS_vector\"}]},\"QID\":\"197595\",\"RESULTS\":\"Package Installed Version Required Version\\nlinux-cloud-tools-4.4.0 1074-aws_4.4.0-1074.84  1092\\nlinux-aws-tools-4.4.0 1074_4.4.0-1074.84  1092\\nlinux-aws-headers-4.4.0 1074_4.15.0-1126.135  1092\\nlinux-tools-4.4.0 1074-aws_4.4.0-1074.84  1092\\nlinux-aws-cloud-tools-4.4.0 1074_4.4.0-1074.84  1092\",\"SEVERITY\":\"3\",\"SSL\":\"0\",\"STATUS\":\"Active\",\"TIMES_FOUND\":\"5393\",\"TYPE\":\"Confirmed\",\"UNIQUE_VULN_ID\":\"5555555555\"},\"DNS\":\"\",\"DNS_DATA\":{\"DOMAIN\":\"\",\"FQDN\":\"\",\"HOSTNAME\":\"\"},\"ID\":\"12048633\",\"IP\":\"10.50.2.111\",\"LAST_PC_SCANNED_DATE\":\"2023-06-28T09:58:12Z\",\"LAST_SCAN_DATETIME\":\"2023-07-03T06:25:17Z\",\"LAST_VM_SCANNED_DATE\":\"2023-07-03T06:23:47Z\",\"LAST_VM_SCANNED_DURATION\":\"1113\",\"NETBIOS\":\"\",\"OS\":\"\",\"TRACKING_METHOD\":\"IP\"}",
         "type": [

diff --git a/packages/qualys_vmdr/docs/README.md b/packages/qualys_vmdr/docs/README.md
@@ -118,24 +118,24 @@ An example event for `asset_host_detection` looks as following:
 
 ```json
 {
-    "@timestamp": "2024-09-25T21:44:26.325Z",
+    "@timestamp": "2024-12-05T11:02:04.225Z",
     "agent": {
-        "ephemeral_id": "f8145b5b-4d53-444a-bd44-2f296cf357e6",
-        "id": "efcbf604-6e25-41db-a21e-22c8227e0663",
-        "name": "elastic-agent-93250",
+        "ephemeral_id": "9c04104d-1da0-4c98-b133-0aedefdc2680",
+        "id": "dccf1148-df50-4c35-a3d7-633418e936ff",
+        "name": "elastic-agent-88337",
         "type": "filebeat",
         "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "qualys_vmdr.asset_host_detection",
-        "namespace": "88572",
+        "namespace": "85068",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "efcbf604-6e25-41db-a21e-22c8227e0663",
+        "id": "dccf1148-df50-4c35-a3d7-633418e936ff",
         "snapshot": false,
         "version": "8.13.0"
     },
@@ -145,7 +145,7 @@ An example event for `asset_host_detection` looks as following:
             "host"
         ],
         "dataset": "qualys_vmdr.asset_host_detection",
-        "ingested": "2024-09-25T21:44:29Z",
+        "ingested": "2024-12-05T11:02:07Z",
         "kind": "alert",
         "original": "{\"DETECTION_LIST\":{\"AFFECT_RUNNING_KERNEL\":\"0\",\"FIRST_FOUND_DATETIME\":\"2021-02-05T04:50:45Z\",\"IS_DISABLED\":\"0\",\"IS_IGNORED\":\"0\",\"LAST_FIXED_DATETIME\":\"2022-12-14T06:52:57Z\",\"LAST_FOUND_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_PROCESSED_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_TEST_DATETIME\":\"2024-03-08T20:15:41Z\",\"LAST_UPDATE_DATETIME\":\"2024-03-08T20:15:41Z\",\"QDS\":{\"#text\":\"35\",\"severity\":\"LOW\"},\"QDS_FACTORS\":{\"QDS_FACTOR\":[{\"#text\":\"7.7\",\"name\":\"CVSS\"},{\"#text\":\"v3.x\",\"name\":\"CVSS_version\"},{\"#text\":\"0.00232\",\"name\":\"epss\"},{\"#text\":\"AV:N/AC:L/PR:L/UI:N/S:C/C:N/I:N/A:H\",\"name\":\"CVSS_vector\"}]},\"QID\":\"197595\",\"RESULTS\":\"Package Installed Version Required Version\\nlinux-cloud-tools-4.4.0 1074-aws_4.4.0-1074.84  1092\\nlinux-aws-tools-4.4.0 1074_4.4.0-1074.84  1092\\nlinux-aws-headers-4.4.0 1074_4.15.0-1126.135  1092\\nlinux-tools-4.4.0 1074-aws_4.4.0-1074.84  1092\\nlinux-aws-cloud-tools-4.4.0 1074_4.4.0-1074.84  1092\",\"SEVERITY\":\"3\",\"SSL\":\"0\",\"STATUS\":\"Active\",\"TIMES_FOUND\":\"5393\",\"TYPE\":\"Confirmed\",\"UNIQUE_VULN_ID\":\"5555555555\"},\"DNS\":\"\",\"DNS_DATA\":{\"DOMAIN\":\"\",\"FQDN\":\"\",\"HOSTNAME\":\"\"},\"ID\":\"12048633\",\"IP\":\"10.50.2.111\",\"LAST_PC_SCANNED_DATE\":\"2023-06-28T09:58:12Z\",\"LAST_SCAN_DATETIME\":\"2023-07-03T06:25:17Z\",\"LAST_VM_SCANNED_DATE\":\"2023-07-03T06:23:47Z\",\"LAST_VM_SCANNED_DURATION\":\"1113\",\"NETBIOS\":\"\",\"OS\":\"\",\"TRACKING_METHOD\":\"IP\"}",
         "type": [

diff --git a/packages/qualys_vmdr/manifest.yml b/packages/qualys_vmdr/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: qualys_vmdr
 title: Qualys VMDR
-version: "5.4.0"
+version: "5.5.0"
 description: Collect data from Qualys VMDR platform with Elastic Agent.
 type: integration
 categories: