[O11y][Airflow] Add system tests for the integration package (#8564)

* add system tests

* update changelog

* remove scheduler.critical_section_query_duration field

* update kibana version and stream image version

* update version and description

packages/airflow/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '2.3'
+services:
+  airflow:
+    image: docker.elastic.co/observability/stream:v0.13.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command: log --start-signal=SIGHUP --delay=5s --addr elastic-agent:8125 -p=udp /sample_logs/test-airflow.log

packages/airflow/_dev/deploy/docker/sample_logs/test-airflow.log

@@ -0,0 +1,9 @@
+dag_processing.total_parse_time:1.066219636995811|g
+dagbag_size:0|g
+dag_processing.import_errors:0|g
+scheduler.tasks.starving:0|g
+scheduler.tasks.executable:0|g
+scheduler.critical_section_duration:7.831787|ms
+executor.open_slots:32|g
+executor.queued_tasks:0|g
+executor.running_tasks:0|g

packages/airflow/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.6.0"
+  changes:
+    - description: Update to Kibana 8.11 to support enhanced statsd implementation, and add system test cases.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8564
 - version: "0.5.1"
   changes:
     - description: Add dimension field for container.id which was previously missed during package-spec v3 migration

packages/airflow/data_stream/statsd/_dev/test/system/test-default-config.yml

@@ -0,0 +1,4 @@
+service_notify_signal: SIGHUP
+vars:
+  host: 0.0.0.0
+  port: 8125

packages/airflow/data_stream/statsd/sample_event.json

@@ -0,0 +1,58 @@
+{
+    "@timestamp": "2023-11-28T06:26:54.238Z",
+    "agent": {
+        "ephemeral_id": "283d2103-181e-4e55-990c-d463765d591a",
+        "id": "208488b1-ba3d-4035-b968-4202e1fadc05",
+        "name": "docker-fleet-agent",
+        "type": "metricbeat",
+        "version": "8.11.0"
+    },
+    "airflow": {
+        "task_executable": {
+            "value": 0
+        }
+    },
+    "data_stream": {
+        "dataset": "airflow.statsd",
+        "namespace": "ep",
+        "type": "metrics"
+    },
+    "ecs": {
+        "version": "8.5.1"
+    },
+    "elastic_agent": {
+        "id": "208488b1-ba3d-4035-b968-4202e1fadc05",
+        "snapshot": false,
+        "version": "8.11.0"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "dataset": "airflow.statsd",
+        "ingested": "2023-11-28T06:26:55Z",
+        "module": "statsd"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "containerized": true,
+        "hostname": "docker-fleet-agent",
+        "id": "d7fd92f5e61644938d48518adcee73ad",
+        "ip": "172.20.0.7",
+        "mac": "02-42-AC-14-00-07",
+        "name": "docker-fleet-agent",
+        "os": {
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "3.10.0-1160.90.1.el7.x86_64",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.6 LTS (Focal Fossa)"
+        }
+    },
+    "metricset": {
+        "name": "server"
+    },
+    "service": {
+        "type": "statsd"
+    }
+}

packages/airflow/manifest.yml

@@ -1,14 +1,14 @@
 name: airflow
 title: Airflow
-version: "0.5.1"
+version: "0.6.0"
 description: Airflow Integration.
 type: integration
 format_version: "3.0.0"
 categories:
   - observability
 conditions:
   kibana:
-    version: "^8.9.0"
+    version: "^8.11.0"
   elastic:
     subscription: basic
 icons: