diff --git a/flight-data-ingester/README.md b/flight-data-ingester/README.md index 532215f..0179554 100644 --- a/flight-data-ingester/README.md +++ b/flight-data-ingester/README.md @@ -1,6 +1,12 @@ # GreptimeDB Flight -This project demonstrates GreptimeDB's ingesting geo-spatial data using the the `greptimedb-ingester-go` client. It selects the last 10 flights that departed in the last 30 minutes from the configured icao airport code. The ingester script utilizes the [OpenSky Network API](https://opensky-network.org/apidoc/) to fetch flight state data and inserts the flight metrics into GreptimeDB +This project demonstrates GreptimeDB's ability to ingest geo-spatial data using the +`greptimedb-ingester-go` client. It selects the last 10 flights that departed in +the last 30 minutes from the configured icao airport code. The ingester script +utilizes the [OpenSky Network API](https://opensky-network.org/apidoc/) to fetch +flight state data and inserts the flight metrics into GreptimeDB + +![screenshot](screenshot.png) ## How to run this demo @@ -13,7 +19,11 @@ cd demo-scene/flight-data-ingester docker compose up ``` -It can take a while for the first run to pull down images and also build necessary components. +It can take a while for the first run to pull down images and also build +necessary components. + +Open your browser at `http://localhost:3000` and find our preconfigured grafnaa +dashboard. The default user/pw is admin/admin ## How it works @@ -22,12 +32,17 @@ The topology is illustrated in this diagram. ```mermaid flowchart LR greptimedb[(GreptimeDB)] + grafana api{Open Sky Data} --> go-ingester go-ingester --> greptimedb + greptimedb --> grafana ``` -after GreptimeDB starts, we use the `ingester` script which uses the go client's [high level api](https://docs.greptime.com/user-guide/ingest-data/for-iot/grpc-sdks/go/#installation) to create the table and insert data. It's dead-simple to perform transformations and data munging on your struct and insert into target GreptimeDB columns by tagging your metric struct accordingly as seen in the `./ingester/dto.go` file . +after GreptimeDB starts, we use the `ingester` script which uses the go client's +[high level api](https://docs.greptime.com/user-guide/ingest-data/for-iot/grpc-sdks/go/#installation) +to create the table and insert data. It's dead-simple to perform transformations and data munging for insertion into target GretimeDB tables. +Create and tag your metric struct accordingly as seen in the `./ingester/dto.go` file. ## Note diff --git a/flight-data-ingester/docker-compose.yml b/flight-data-ingester/docker-compose.yml index bbd259a..3845bbf 100644 --- a/flight-data-ingester/docker-compose.yml +++ b/flight-data-ingester/docker-compose.yml @@ -1,6 +1,6 @@ services: greptimedb: - image: docker.io/greptime/greptimedb:v0.9.2 + image: docker.io/greptime/greptimedb-dev:dev-20240901-1725202129-e56709b5 command: standalone start --http-addr=0.0.0.0:4000 --rpc-addr=0.0.0.0:4001 --mysql-addr=0.0.0.0:4002 --postgres-addr 0.0.0.0:4003 ports: - 4000:4000 @@ -14,6 +14,8 @@ services: interval: 3s timeout: 3s retries: 5 + volumes: + - /tmp/greptimedb-demo:/tmp/greptimedb ingester: build: @@ -29,5 +31,14 @@ services: greptimedb: condition: service_started + grafana: + image: docker.io/grafana/grafana:11.2.0 + ports: + - 3000:3000 + networks: + - demo-network + volumes: + - ./grafana_provisioning:/etc/grafana/provisioning + networks: demo-network: diff --git a/flight-data-ingester/grafana_provisioning/dashboards/dashboards.yml b/flight-data-ingester/grafana_provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..3447546 --- /dev/null +++ b/flight-data-ingester/grafana_provisioning/dashboards/dashboards.yml @@ -0,0 +1,23 @@ +apiVersion: 1 + +providers: + # an unique provider name. Required + - name: 'greptimedb demo dashboards' + # Org id. Default to 1 + orgId: 1 + # name of the dashboard folder. + folder: '' + # folder UID. will be automatically generated if not specified + folderUid: '' + # provider type. Default to 'file' + type: file + # disable dashboard deletion + disableDeletion: false + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + options: + # path to dashboard files on disk. Required when using the 'file' type + path: /etc/grafana/provisioning/dashboards + # use folder names from filesystem to create folders in Grafana \ No newline at end of file diff --git a/flight-data-ingester/grafana_provisioning/dashboards/map.json b/flight-data-ingester/grafana_provisioning/dashboards/map.json new file mode 100644 index 0000000..a117792 --- /dev/null +++ b/flight-data-ingester/grafana_provisioning/dashboards/map.json @@ -0,0 +1,223 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "datasource": { + "default": true, + "type": "grafana-postgresql-datasource", + "uid": "edwaq5vjcr668c" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 21, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "opacity": 0.4, + "tooltip": false, + "type": "osm-standard" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": true, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": true + }, + "layers": [ + { + "config": { + "showLegend": true, + "style": { + "color": { + "field": "velocity", + "fixed": "dark-green" + }, + "opacity": 1, + "rotation": { + "fixed": 0, + "max": 360, + "min": -360, + "mode": "mod" + }, + "size": { + "fixed": 15, + "max": 15, + "min": 2 + }, + "symbol": { + "fixed": "img/icons/marker/plane.svg", + "mode": "fixed" + }, + "symbolAlign": { + "horizontal": "center", + "vertical": "center" + }, + "text": { + "field": "baro_altitude", + "fixed": "", + "mode": "field" + }, + "textConfig": { + "fontSize": 11, + "offsetX": 0, + "offsetY": 20, + "textAlign": "center", + "textBaseline": "middle" + } + } + }, + "filterData": { + "id": "byRefId", + "options": "A" + }, + "location": { + "mode": "auto" + }, + "name": "Flight Velocity", + "tooltip": true, + "type": "markers" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "id": "fit", + "lastOnly": false, + "lat": 0, + "layer": "Flight Velocity", + "lon": 0, + "padding": 10, + "zoom": 13 + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "edwaq5vjcr668c" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select icao24, longitude, latitude, baro_altitude, geo_altitude, velocity, geohash(latitude, longitude, 7) as geohash from icao24_state where icao24 = '$icao24' and $__timeFilter(ts) order by ts desc limit 500;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Flight Map", + "type": "geomap" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "aac87b", + "value": "aac87b" + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "edwaq5vjcr668c" + }, + "definition": "SELECT DISTINCT icao24 FROM icao24_state where $__timeFilter(ts);", + "hide": 0, + "includeAll": false, + "label": "icao24", + "multi": false, + "name": "icao24", + "options": [], + "query": "SELECT DISTINCT icao24 FROM icao24_state where $__timeFilter(ts);", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Flight Map", + "uid": "bdwetcv3dtssge", + "version": 1, + "weekStart": "" +} diff --git a/flight-data-ingester/grafana_provisioning/datasources/greptime_pg_ds.yml b/flight-data-ingester/grafana_provisioning/datasources/greptime_pg_ds.yml new file mode 100644 index 0000000..db62017 --- /dev/null +++ b/flight-data-ingester/grafana_provisioning/datasources/greptime_pg_ds.yml @@ -0,0 +1,9 @@ +apiVersion: 2 +datasources: + - name: greptimedb_pg + type: postgres + url: greptimedb:4003 + uid: edwaq5vjcr668c + jsonData: + database: public + sslmode: 'disable' # disable/require/verify-ca/verify-full diff --git a/flight-data-ingester/screenshot.png b/flight-data-ingester/screenshot.png new file mode 100644 index 0000000..397e851 Binary files /dev/null and b/flight-data-ingester/screenshot.png differ