diff --git a/.gitignore b/.gitignore index 394bcf2..61792e2 100644 --- a/.gitignore +++ b/.gitignore @@ -394,9 +394,11 @@ tags # End of https://www.toptal.com/developers/gitignore/api/git,vim,helm,rust,bazel,emacs,jetbrains+all,visualstudiocode,node,go +tmp +*.off + # ignore downloaded charts *.tgz -*.off # ignore report from MegaLinter report/ diff --git a/README.md b/README.md index 4f8d4db..d66cd55 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A set of components to provide a view of which version of services are deployed ## Architecture Overview -![cdviz architecture](doc/images/Drawing%202023-12-27%2016.04.47-cdviz-architecture.excalidraw.svg) +![cdviz architecture](docs/images/Drawing%202023-12-27%2016.04.47-cdviz-architecture.excalidraw.svg) ### cdviz-collector diff --git a/cdviz-collector/Cargo.toml b/cdviz-collector/Cargo.toml index 3a62c54..38658a3 100644 --- a/cdviz-collector/Cargo.toml +++ b/cdviz-collector/Cargo.toml @@ -50,7 +50,7 @@ sqlx = { version = "0.7", features = [ ], default-features = false, optional = true } thiserror = "1" time = "0.3" -tokio = { version = "1.36", features = ["full"] } +tokio = { version = "1.37", features = ["full"] } tracing = "0.1" tracing-opentelemetry-instrumentation-sdk = { version = "0.18" } @@ -61,7 +61,7 @@ cdevents-sdk = { git = "https://github.com/cdevents/sdk-rust", features = [ ] } proptest = "1" rstest = "0.18" -rustainers = "0.11" +rustainers = "0.12" tracing-subscriber = "0.3" [features] diff --git a/cdviz-collector/cdviz-collector.toml b/cdviz-collector/cdviz-collector.toml index bdf6389..c4544f3 100644 --- a/cdviz-collector/cdviz-collector.toml +++ b/cdviz-collector/cdviz-collector.toml @@ -2,7 +2,7 @@ type = "debug" [sinks.cdviz_db] -type = "postgresql" +type = "db" url = "postgresql://postgres:passwd@localhost:5432/cdviz" pool_connections_min = 1 pool_connections_max = 10 diff --git a/cdviz-collector/src/sinks/db.rs b/cdviz-collector/src/sinks/db.rs index 73091b7..dba75a2 100644 --- a/cdviz-collector/src/sinks/db.rs +++ b/cdviz-collector/src/sinks/db.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use sqlx::postgres::PgPoolOptions; use sqlx::PgPool; use time::OffsetDateTime; -use tracing::{info, Instrument}; +use tracing::Instrument; use super::Sink; @@ -38,7 +38,7 @@ impl TryFrom for DbSink { let pool_options = PgPoolOptions::new() .min_connections(config.pool_connections_min) .max_connections(config.pool_connections_max); - info!( + tracing::info!( max_connections = pool_options.get_max_connections(), min_connections = pool_options.get_min_connections(), acquire_timeout = ?pool_options.get_acquire_timeout(), @@ -118,7 +118,6 @@ async fn store_event(pg_pool: &PgPool, event: Event) -> Result<()> { .execute(pg_pool) .instrument(build_otel_span("INSERT INTO events")) .await?; - Ok(()) } @@ -233,8 +232,8 @@ mod tests { .await .unwrap() .get(0); - assert_eq!(count_n, (count + 1)); - count = count_n; + count += 1; + assert_eq!(count_n, count); } } } diff --git a/cdviz-collector/src/sinks/mod.rs b/cdviz-collector/src/sinks/mod.rs index 9e5d337..912ee3b 100644 --- a/cdviz-collector/src/sinks/mod.rs +++ b/cdviz-collector/src/sinks/mod.rs @@ -18,7 +18,7 @@ use http::HttpSink; #[serde(tag = "type")] pub(crate) enum Config { #[cfg(feature = "sink_db")] - #[serde(alias = "postgresql")] + #[serde(alias = "db")] Db(db::Config), #[serde(alias = "debug")] Debug(debug::Config), @@ -59,6 +59,7 @@ pub(crate) fn start(name: String, config: Config, rx: Receiver) -> Join let sink = SinkEnum::try_from(config)?; let mut rx = rx; while let Ok(msg) = rx.recv().await { + tracing::debug!(name, event_id = ?msg.cdevent.id(), "sending"); if let Err(err) = sink.send(&msg).await { tracing::warn!(name, ?err, "fail during sending of event"); } diff --git a/cdviz-db/migrations/20240401171152.sql b/cdviz-db/migrations/20240401171152.sql new file mode 100644 index 0000000..70d0b0b --- /dev/null +++ b/cdviz-db/migrations/20240401171152.sql @@ -0,0 +1,8 @@ +-- Create index "idx_subject" to table: "cdevents_lake" +CREATE INDEX "idx_subject" ON "cdevents_lake" ("subject"); +-- Create index "idx_timestamp" to table: "cdevents_lake" +CREATE INDEX "idx_timestamp" ON "cdevents_lake" ("timestamp"); +-- Set comment to column: "timestamp" on table: "cdevents_lake" +COMMENT ON COLUMN "cdevents_lake" ."timestamp" IS 'timestamp of cdevents extracted from context.timestamp in the json'; +-- Set comment to column: "version" on table: "cdevents_lake" +COMMENT ON COLUMN "cdevents_lake" ."version" IS 'the version of the suject s type, extracted from context.type. The verion number are split in 0 for major, 1 for minor, 2 for patch'; diff --git a/cdviz-db/migrations/atlas.sum b/cdviz-db/migrations/atlas.sum index 1d7941b..11431b6 100644 --- a/cdviz-db/migrations/atlas.sum +++ b/cdviz-db/migrations/atlas.sum @@ -1,2 +1,3 @@ -h1:iMxNr2Ku/G2PcNz5GvMGn9GSNPBbHpZZGqb5ZnXohJY= +h1:G8sqtBxbE8rQ0ZReTiVO01ugCUK0Z5U1/tn0i10a0H4= 20240316184734.sql h1:P4X1rAK3TUnBFjHwTZw+Lc7l3x9qkKXmTk8Y56GjMvI= +20240401171152.sql h1:Oh6ZIUfdo8j/LCUmA+YWmv28MIJYKsioxYIAOSA2MhI= diff --git a/cdviz-db/src/schema.sql b/cdviz-db/src/schema.sql index 3e91e14..87c7fcf 100644 --- a/cdviz-db/src/schema.sql +++ b/cdviz-db/src/schema.sql @@ -11,11 +11,15 @@ CREATE TABLE IF NOT EXISTS "cdevents_lake" ( COMMENT ON TABLE "cdevents_lake" IS 'table of stored cdevents without transformation'; COMMENT ON COLUMN "cdevents_lake"."imported_at" IS 'the timestamp when the cdevent was stored into the table'; +COMMENT ON COLUMN "cdevents_lake"."timestamp" IS 'timestamp of cdevents extracted from context.timestamp in the json'; COMMENT ON COLUMN "cdevents_lake"."payload" IS 'the full cdevent in json format'; COMMENT ON COLUMN "cdevents_lake"."subject" IS 'subject extracted from context.type in the json'; COMMENT ON COLUMN "cdevents_lake"."predicate" IS 'predicate of the subject, extracted from context.type in the json'; COMMENT ON COLUMN "cdevents_lake"."version" IS 'the version of the suject s type, extracted from context.type. The verion number are split in 0 for major, 1 for minor, 2 for patch'; +CREATE INDEX IF NOT EXISTS "idx_timestamp" ON "cdevents_lake"("timestamp"); +CREATE INDEX IF NOT EXISTS "idx_subject" ON "cdevents_lake"("subject"); + -- create a view based on fields in the json payload -- source: [Postgresql json column to view - Database Administrators Stack Exchange](https://dba.stackexchange.com/questions/151838/postgresql-json-column-to-view?newreg=ed0a9389843a45699bfb02559dd32038) -- DO $$ diff --git a/cdviz-grafana/dashboards/cdevents-activity.json b/cdviz-grafana/dashboards/cdevents-activity.json index 3dc2b40..29ef2c8 100644 --- a/cdviz-grafana/dashboards/cdevents-activity.json +++ b/cdviz-grafana/dashboards/cdevents-activity.json @@ -19,43 +19,229 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, "links": [], "liveNow": false, "panels": [ { "datasource": { - "type": "postgres", - "uid": "P3ABDE63318046581" + "type": "grafana-postgresql-datasource", + "uid": "cdviz-db" + }, + "description": "Count by Subject & Predicate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] }, "gridPos": { - "h": 8, - "w": 24, + "h": 6, + "w": 4, "x": 0, "y": 0 }, + "id": 2, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": false, + "values": ["value"] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["allValues"], + "fields": "/^count$/", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "cdviz-db" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as count, subject, predicate\nFROM cdevents_lake\nWHERE $__timeFilter(imported_at) AND subject IN ($subjects) AND predicate IN ($predicates)\nGROUP BY subject, predicate\nORDER BY subject, predicate\nLIMIT $limit", + "refId": "A", + "sql": { + "columns": [ + { + "name": "COUNT", + "parameters": [ + { + "name": "*", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "subject", + "type": "string" + }, + "type": "groupBy" + }, + { + "property": { + "name": "predicate", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "cdevents_lake" + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "cdviz-db" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": true, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "subject" + }, + "properties": [ + { + "id": "custom.width", + "value": 107 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "predicate" + }, + "properties": [ + { + "id": "custom.width", + "value": 127 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "imported_at" + }, + "properties": [ + { + "id": "custom.width", + "value": 177 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 206 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 6 + }, "id": 1, "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": true, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": true + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "timestamp" + } + ] }, - "pluginVersion": "10.2.2", + "pluginVersion": "10.2.6", "targets": [ { "datasource": { - "type": "postgres", - "uid": "P3ABDE63318046581" + "type": "grafana-postgresql-datasource", + "uid": "cdviz-db" }, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT \"timestamp\",\n \"payload\"\nFROM \"cdevents_lake\"\nWHERE $__timeFilter(timestamp) \nORDER BY \"timestamp\" DESC", + "rawSql": "SELECT \n \"imported_at\",\n \"timestamp\",\n \"subject\",\n \"predicate\",\n \"payload\"->'subject' as \"payload_subject\",\n \"payload\"->'context' as \"payload_context\"\nFROM cdevents_lake\nWHERE $__timeFilter(imported_at) AND subject IN ($subjects) AND predicate IN ($predicates)\nORDER BY imported_at DESC\nLIMIT $limit", "refId": "A", "sql": { "columns": [ @@ -77,23 +263,117 @@ } ], "title": "Events Logs", - "type": "logs" + "type": "table" } ], "refresh": "", - "schemaVersion": 38, + "schemaVersion": 39, "tags": [], "templating": { - "list": [] + "list": [ + { + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "cdviz-db" + }, + "definition": "SELECT DISTINCT \"subject\" FROM \"cdevents_lake\" WHERE \"subject\" LIKE '$__searchFilter' AND $__timeFilter(imported_at)", + "hide": 0, + "includeAll": true, + "label": "subjects", + "multi": true, + "name": "subjects", + "options": [], + "query": "SELECT DISTINCT \"subject\" FROM \"cdevents_lake\" WHERE \"subject\" LIKE '$__searchFilter' AND $__timeFilter(imported_at)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "cdviz-db" + }, + "definition": "SELECT DISTINCT \"predicate\" FROM \"cdevents_lake\" WHERE \"predicate\" LIKE '$__searchFilter' AND subject IN ($subjects) AND $__timeFilter(imported_at)", + "hide": 0, + "includeAll": true, + "label": "predicates", + "multi": true, + "name": "predicates", + "options": [], + "query": "SELECT DISTINCT \"predicate\" FROM \"cdevents_lake\" WHERE \"predicate\" LIKE '$__searchFilter' AND subject IN ($subjects) AND $__timeFilter(imported_at)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "50", + "value": "50" + }, + "description": "Maximum number of value retrieved", + "hide": 0, + "includeAll": false, + "label": "limit", + "multi": false, + "name": "limit", + "options": [ + { + "selected": true, + "text": "50", + "value": "50" + }, + { + "selected": false, + "text": "100", + "value": "100" + }, + { + "selected": false, + "text": "500", + "value": "500" + }, + { + "selected": false, + "text": "1000", + "value": "1000" + } + ], + "query": "50,100,500,1000", + "skipUrlSync": false, + "type": "custom" + }, + { + "hide": 2, + "name": "where", + "query": "$__timeFilter(imported_at) AND subject IN ($subjects) AND predicate IN ($predicates)", + "skipUrlSync": false, + "type": "constant" + } + ] }, "time": { - "from": "now-6h", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "cdevents activity", "uid": "a6eeda5f-2550-438c-9477-8ed28ab50ce6", - "version": 1, + "version": 4, "weekStart": "" } diff --git a/cdviz-grafana/datasources/cdviz-db.yaml b/cdviz-grafana/datasources/cdviz-db.yaml new file mode 100644 index 0000000..7e43761 --- /dev/null +++ b/cdviz-grafana/datasources/cdviz-db.yaml @@ -0,0 +1,22 @@ +apiVersion: 1 + +datasources: + - name: cdviz-db + uid: cdviz-db + type: grafana-postgresql-datasource + access: proxy + url: "$CDVIZ_RO_POSTGRES_HOST:$CDVIZ_RO_POSTGRES_PORT_NUMBER" + user: $CDVIZ_RO_POSTGRES_USER + secureJsonData: + password: $CDVIZ_RO_POSTGRES_PASSWORD + jsonData: + database: $CDVIZ_RO_POSTGRES_DB + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 100 # Grafana v5.4+ + maxIdleConns: 100 # Grafana v5.4+ + maxIdleConnsAuto: true # Grafana v9.5.1+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1500 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false + # - name: TestData + # type: testdata \ No newline at end of file diff --git a/charts/cdviz/Chart.yaml b/charts/cdviz/Chart.yaml index 41bc457..2aaa28f 100644 --- a/charts/cdviz/Chart.yaml +++ b/charts/cdviz/Chart.yaml @@ -7,7 +7,7 @@ appVersion: 0.1.0 # TODO Change this to a version number on release dependencies: - name: postgresql - version: 15.1.4 + version: 15.2.0 # repository: https://charts.bitnami.com/bitnami repository: oci://registry-1.docker.io/bitnamicharts condition: postgresql.enabled diff --git a/charts/cdviz/templates/secrets.yaml b/charts/cdviz/templates/secrets.yaml index 7ab6deb..a0cbc26 100644 --- a/charts/cdviz/templates/secrets.yaml +++ b/charts/cdviz/templates/secrets.yaml @@ -57,9 +57,9 @@ data: POSTGRES_PASSWORD: |- {{ .Values.secrets.values.CDVIZ_RO_POSTGRES_PASSWORD | b64enc }} POSTGRES_HOST: |- - {{ .Values.secrets.values.POSTGRES_HOST | b64enc }} + {{ .Values.secrets.values.CDVIZ_RO_POSTGRES_HOST | default .Values.secrets.values.POSTGRES_HOST | b64enc }} POSTGRES_PORT_NUMBER: |- - {{ .Values.secrets.values.POSTGRES_PORT_NUMBER | b64enc }} + {{ .Values.secrets.values.CDVIZ_RO_POSTGRES_PORT_NUMBER | default .Values.secrets.values.POSTGRES_PORT_NUMBER | b64enc }} POSTGRES_DB: |- {{ .Values.secrets.values.POSTGRES_DB | b64enc }} {{- end }} diff --git a/charts/cdviz/values.yaml b/charts/cdviz/values.yaml index 66d2d20..1f4e894 100644 --- a/charts/cdviz/values.yaml +++ b/charts/cdviz/values.yaml @@ -17,6 +17,8 @@ secrets: # RO (read-only) used by cdviz-grafana to query data from the db CDVIZ_RO_POSTGRES_USER: cdviz-ro CDVIZ_RO_POSTGRES_PASSWORD: postgres-password + # CDVIZ_RO_POSTGRES_HOST: cdviz-postgresql # to override if different from POSTGRES_HOST + # CDVIZ_RO_POSTGRES_PORT_NUMBER: "5432" # to override if different from POSTGRES_HOST cdviz-collector: enabled: true diff --git a/demos/.mise.toml b/demos/.mise.toml new file mode 100644 index 0000000..712c9e6 --- /dev/null +++ b/demos/.mise.toml @@ -0,0 +1,5 @@ +[env] + +[tools] +task = '*' # to have a set of simple commands for repetive task (and CI) +# docker = '*' # to build, to run containers, should be available for some of thoses tools diff --git a/demos/example_01/.gitignore b/demos/example_01/.gitignore new file mode 100644 index 0000000..8032a18 --- /dev/null +++ b/demos/example_01/.gitignore @@ -0,0 +1 @@ +cdevents/*.json \ No newline at end of file diff --git a/demos/example_01/cdevents/.keep b/demos/example_01/cdevents/.keep new file mode 100644 index 0000000..e69de29 diff --git a/demos/example_01/cdviz-collector.toml b/demos/example_01/cdviz-collector.toml new file mode 100644 index 0000000..e232f99 --- /dev/null +++ b/demos/example_01/cdviz-collector.toml @@ -0,0 +1,14 @@ +[sinks.debug] +type = "debug" + +[sinks.cdviz_db] +type = "db" +url = "postgresql://cdviz:postgres-password@127.0.0.1:5432/cdviz" +pool_connections_min = 1 +pool_connections_max = 10 + +[sources.cdevents_folder] +type = "opendal" +kind = "fs" +polling_interval = "10s" +parameters = { root = "./cdevents" } diff --git a/demos/stack-compose/dashboards.yaml b/demos/stack-compose/dashboards.yaml new file mode 100644 index 0000000..2424e58 --- /dev/null +++ b/demos/stack-compose/dashboards.yaml @@ -0,0 +1,22 @@ +apiVersion: 1 + +# mainly to use with demos/docker-compose +providers: +- name: 'cdviz-db-dashboards' + # org id. will default to orgId 1 if not specified + orgId: 1 + # name of the dashboard folder. Required + folder: 'cdviz' + # folder UID. will be automatically generated if not specified + folderUid: '' + # provider type. Required + type: file + # disable dashboard deletion + disableDeletion: false + # enable dashboard editing + editable: true + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 30 + options: + # path to dashboard files on disk. Required + path: /var/provisioning/dashboards \ No newline at end of file diff --git a/demos/stack-compose/docker-compose.yaml b/demos/stack-compose/docker-compose.yaml new file mode 100644 index 0000000..49a1d9d --- /dev/null +++ b/demos/stack-compose/docker-compose.yaml @@ -0,0 +1,84 @@ +version: "3.9" +services: + postgres: + image: docker.io/library/postgres:16 + restart: always + # platform: linux/amd64 + shm_size: 128mb + environment: + POSTGRES_USER: cdviz + POSTGRES_PASSWORD: postgres-password + POSTGRES_DB: cdviz + healthcheck: + test: pg_isready -U cdviz -d cdviz + interval: 10s + timeout: 5s + retries: 5 + # network_mode: host + ports: + - "5432:5432" + # user: "${UID}:${GID}" + volumes: + # In this example, we share the folder `db-data` in our root repository, with the default PostgreSQL data path + # It means that every time the repository is modifying the data inside + # of `/var/lib/postgresql/data/`, automatically the change will appear in `db-data` + # You don't need to create the `db-data` folder. Docker Compose will do it for you + - ./tmp/postgres/data:/var/lib/postgresql/data + # # In this example, we share an init.sql script with the container + # # The init script will be executed when the database is first run + # - ./init.sql:/docker-entrypoint-initdb.d/init.sql + cdviz-db-migrate: + image: docker.io/arigaio/atlas:latest + command: > + migrate apply + --url postgres://cdviz:postgres-password@postgres:5432/cdviz?sslmode=disable + # network_mode: host + depends_on: + postgres: + condition: service_healthy + volumes: + - ../../cdviz-db/migrations:/migrations:ro + adminer: + image: docker.io/library/adminer + restart: always + environment: + - ADMINER_DESIGN=pepa-linha-dark + - ADMINER_DEFAULT_SERVER=postgres + - ADMINER_DEFAULT_DB_DRIVER=pgsql + - ADMINER_DEFAULT_DB_NAME=cdviz + ports: + - 5499:8080 + grafana: + image: docker.io/grafana/grafana-oss:10.4.1 + container_name: grafana + restart: unless-stopped + environment: + # see https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#override-configuration-with-environment-variables + # To override an option, use a predefined pattern GF__. + # - GF_SERVER_ROOT_URL=http://my.grafana.server/ + - GF_INSTALL_PLUGINS=grafana-clock-panel + # - GF_SECURITY_ADMIN_USER=admin + # - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_BASIC_ENABLED=false + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_AUTH_DISABLE_SIGNOUT_MENU=true + # - GF_SECURITY_ALLOW_EMBEDDING=true #Allow IFRAME Calls + - GF_ENABLE_GZIP=true + - CDVIZ_RO_POSTGRES_HOST=postgres + - CDVIZ_RO_POSTGRES_PORT_NUMBER=5432 + - CDVIZ_RO_POSTGRES_USER=cdviz + - CDVIZ_RO_POSTGRES_PASSWORD=postgres-password + - CDVIZ_RO_POSTGRES_DB=cdviz + # network_mode: host + ports: + - "3000:3000" + # user: "${UID}:${GID}" + user: "0" + volumes: + # - ./tmp/grafana/data/:/var/lib/grafana/:rw + - ../../cdviz-grafana/datasources:/etc/grafana/provisioning/datasources:ro + - ../../cdviz-grafana/dashboards:/var/provisioning/dashboards:rw + - ./dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro + # - ./tmp/grafana.ini:/etc/grafana/grafana.ini diff --git a/demos/taskfile.yaml b/demos/taskfile.yaml new file mode 100644 index 0000000..4aba9b1 --- /dev/null +++ b/demos/taskfile.yaml @@ -0,0 +1,46 @@ +version: '3' + +tasks: + + clean: + cmds: + - task: stack:compose:clean + + + stack:compose:up: + dir: stack-compose + cmds: + # - mkdir -p tmp/grafana/data || true + # - mkdir -p tmp/postgres/data || true + - docker compose -f docker-compose.yaml up + + stack:compose:clean: + dir: stack-compose + cmds: + - docker compose -f docker-compose.yaml stop + - docker compose -f docker-compose.yaml rm + - sudo rm -rf tmp + + stack:grafana:view: + cmds: + - open http://127.0.0.1:3000 + + stack:db-admin:view: + cmds: + - open http://127.0.0.1:5499 + + cdviz-collector:build: + internal: true + dir: ../cdviz-collector + cmds: + - cargo build + + example_01:run: + dir: example_01 + deps: [ "cdviz-collector:build" ] + env: + RUST_LOG: "cdviz_collector::sinks=debug,info" # cdviz_collector::sources=debug + OTEL_TRACES_SAMPLER: always_off + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://127.0.0.1:4317" + cmds: + - ../../cdviz-collector/target/debug/cdviz-collector --config ./cdviz-collector.toml \ No newline at end of file diff --git a/doc/images/Drawing 2023-12-27 16.04.47-cdviz-architecture.excalidraw.svg b/doc/images/Drawing 2023-12-27 16.04.47-cdviz-architecture.excalidraw.svg deleted file mode 100644 index 128619d..0000000 --- a/doc/images/Drawing 2023-12-27 16.04.47-cdviz-architecture.excalidraw.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - cdviz-collectorDockercdviz-collectorcdevents (native) providerstestkube, tekton, spinnaker, kpten, harbor, ...S3 Bucketnon-native cd-events providerHTTP(direct ?)SlackGitHubECRK8S eventsArgo EventsGrafana App(dashboards, panels, ...)states, locations ofartifactssvc + bora metricsbora metricsfolder of json files(demo & test)cdviz-demoexamplescdeventsexamplesBUS(NATS, Kafka, SSE, ...)json(cdevents)cdevents-translatorSCM,...GitHub \ No newline at end of file diff --git a/docs/images/Drawing 2023-12-27 16.04.47-cdviz-architecture.excalidraw.svg b/docs/images/Drawing 2023-12-27 16.04.47-cdviz-architecture.excalidraw.svg new file mode 100644 index 0000000..8a366eb --- /dev/null +++ b/docs/images/Drawing 2023-12-27 16.04.47-cdviz-architecture.excalidraw.svg @@ -0,0 +1,1284 @@ + + + + + + + + + + + + + + + + cdviz-collector + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Docker + + + + + + cdviz-collector + + + + + + cdevents (native) providers + + testkube, + tekton, + spinnaker, + kpten, + harbor, + ... + + + + + + + + + + + + + + + + + + + + + + + + + + S3 Bucket + + + non-native cd-events provider + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + HTTP + (direct ?) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Slack + + + + + + + + + + + + + + + + + + GitHub + + + + + + + + + + + + + + + + + + + ECR + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + K8S events + + + + + + Argo Events + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Grafana App + (dashboards, panels, ...) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + states, locations of + artifacts + + + + + + + + + + + + + + + + + + bora metrics + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + folder of json files + (demo & test) + + + + + + + cdviz-demo + examples + + + + + + + + + + + + + + + + + + cdevents + examples + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BUS + (NATS, Kafka, SSE, ...) + + + + + + + + + + + + + + + + + + json(cdevents) + + + + + + cdevents-translator + + + + + + + + + + + + + + + + + + + + + SCM, + ... + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GitHub + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/images/cdevents.svg b/docs/images/cdevents.svg similarity index 100% rename from doc/images/cdevents.svg rename to docs/images/cdevents.svg