From 0b8d2b8083f773c8adbf9fb10034da6941c853d4 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Wed, 9 Oct 2024 20:24:09 -0700 Subject: [PATCH] feat: greptime.env for ev demo on cloud --- ev-open-telemetry/.gitignore | 4 ++- ev-open-telemetry/README.md | 30 ++++++++-------- ev-open-telemetry/docker-compose.yml | 34 +++++++++++++++---- .../ev_observer/ev_observer/__init__.py | 29 ++++++---------- ev-open-telemetry/greptime.env.sample | 7 ++++ ev-open-telemetry/greptime_ds.yml.tpl | 14 ++++++++ 6 files changed, 77 insertions(+), 41 deletions(-) create mode 100644 ev-open-telemetry/greptime.env.sample create mode 100644 ev-open-telemetry/greptime_ds.yml.tpl diff --git a/ev-open-telemetry/.gitignore b/ev-open-telemetry/.gitignore index e19791a..014f9d3 100644 --- a/ev-open-telemetry/.gitignore +++ b/ev-open-telemetry/.gitignore @@ -1,3 +1,5 @@ __pycache__ cache.json -.venv \ No newline at end of file +.venv +greptime.env +greptime.env.bak diff --git a/ev-open-telemetry/README.md b/ev-open-telemetry/README.md index 40eed9d..13b3e87 100644 --- a/ev-open-telemetry/README.md +++ b/ev-open-telemetry/README.md @@ -2,20 +2,20 @@ This project demonstrates capturing Tesla charging and driving metrics with OpenTelemetry and Python in GreptimeDB. - + ## How to run this demo -Ensure you have installed `git`, `docker`, `docker-compose` +Ensure you have installed `git`, `docker` and `docker-compose` 22.4 or newer. You also must have a valid Tesla Login with a registered vehicle. -This project uses [TeslaPy](https://github.com/tdorssers/TeslaPy) +This project uses [TeslaPy](https://github.com/tdorssers/TeslaPy) which leverages the Tesla Owner API to gather the vehicle metrics. To run this demo: **Start the Docker Network** -This command builds the containers and +This command builds the containers and waits for the app container to start up. ```shell @@ -31,23 +31,23 @@ done && docker logs ev-open-telemetry_ev_observer_1 & docker attach ev-open-tele **Authenticate to Tesla** -When the container is running, you will see the output in the logs +When the container is running, you will see the output in the logs `Open this URL to authenticate: https://auth.tesla.com/oauth2/v3/authorize?...` -Follow this URL in your browser window and login -with your Tesla credentials. Upon successful authentication, -you will be redirected to a blank page. -Copy and paste the url from your browser +Follow this URL in your browser window and login +with your Tesla credentials. Upon successful authentication, +you will be redirected to a blank page. +Copy and paste the url from your browser into your terminal, which will use the token to authenticate for you. -After you complete this process once, the `cache.json` file will be able to +After you complete this process once, the `cache.json` file will be able to use the refresh token to keep the authenticated session active. ## How it works This project uses the standard OpenTelemetry SDK to capture metrics -and export them to the OTLP-compatible, GreptimeDB back end. -The configuration of the Metric Provider, Meter, Reader, and Exporter is all done in the [__init__.py](./ev_observer/ev_observer/__init__.py) file of the main package. +and export them to the OTLP-compatible, GreptimeDB back end. +The configuration of the Metric Provider, Meter, Reader, and Exporter is all done in the [__init__.py](./ev_observer/ev_observer/__init__.py) file of the main package. After the meter is created, all instruments created with that meter will be read and exported as configured in the provider. @@ -64,7 +64,7 @@ in the class diagram classDiagram VehicleInstrumentor o-- EVMetricData : contains/updates VehicleInstrumentor o-- AbstractVehicleFetcher : contains - + AbstractVehicleFetcher --> EVMetricData : refreshes EVMetricData --> GreptimeDB : exports metrics to @@ -106,10 +106,10 @@ classDiagram ## Future Development **To add additional metrics to our collection** -1. Create a new class that sub-classes the `MetricCollector` +1. Create a new class that sub-classes the `MetricCollector` 2. Add a property to the `VehicleInstrumentor` that contains a reference to this new collector 3. Update the `AbstractVehicleDataFetcher` implementation to return this new MetricCollector data **To add new collection process with different vehicles** -implement a new `AbstractVehicleDataFetcher` and then configure the `VehicleInstrumentor` to utilize this fetcher \ No newline at end of file +implement a new `AbstractVehicleDataFetcher` and then configure the `VehicleInstrumentor` to utilize this fetcher diff --git a/ev-open-telemetry/docker-compose.yml b/ev-open-telemetry/docker-compose.yml index 5ed7173..bfccc18 100644 --- a/ev-open-telemetry/docker-compose.yml +++ b/ev-open-telemetry/docker-compose.yml @@ -1,6 +1,6 @@ services: greptimedb: - image: docker.io/greptime/greptimedb-dev:dev-20240901-1725202129-e56709b5 + image: docker.io/greptime/greptimedb:v0.9.3 command: standalone start --http-addr=0.0.0.0:4000 --rpc-addr=0.0.0.0:4001 --mysql-addr=0.0.0.0:4002 --postgres-addr 0.0.0.0:4003 ports: - 4000:4000 @@ -24,17 +24,28 @@ services: networks: - demo-network environment: - - GREPTIME_HOST=${GREPTIME_HOST:-greptimedb:4000} - - GREPTIME_DB=${GREPTIME_DB:-public} - SCRAPE_INTERVAL_SEC=${SCRAPE_INTERVAL_SEC:-300} - TESLA_USER_EMAIL=${TESLA_USER_EMAIL} + env_file: + - path: "greptime.env" + required: false stdin_open: true tty: true - depends_on: greptimedb: condition: service_started + envsubst: + image: docker.io/widerplan/envsubst + command: "-i /greptime_ds.yml.tpl -o /config_output/greptime_ds.yml" + volumes: + - ./greptime_ds.yml.tpl:/greptime_ds.yml.tpl + - datasource:/config_output + env_file: + - path: "greptime.env" + required: false + init: true + grafana: build: context: ./grafana @@ -45,11 +56,20 @@ services: - demo-network user: "$UID:$GID" volumes: - - ./grafana_provisioning:/etc/grafana/provisioning + - provisioning:/etc/grafana/provisioning + - datasource:/etc/grafana/provisioning/datasources depends_on: - greptimedb: - condition: service_healthy + envsubst: + condition: service_completed_successfully restart: always networks: demo-network: +volumes: + datasource: + provisioning: + driver: local + driver_opts: + o: bind + type: none + device: ./grafana_provisioning diff --git a/ev-open-telemetry/ev_observer/ev_observer/__init__.py b/ev-open-telemetry/ev_observer/ev_observer/__init__.py index 692ec1a..7b100bf 100644 --- a/ev-open-telemetry/ev_observer/ev_observer/__init__.py +++ b/ev-open-telemetry/ev_observer/ev_observer/__init__.py @@ -1,4 +1,4 @@ -import os +import os, base64 from dotenv import load_dotenv from opentelemetry import metrics from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( @@ -12,31 +12,24 @@ load_dotenv() -required_env_vars = { - "GREPTIME_HOST": "db_host", - "GREPTIME_DB": "db_name", - # "GREPTIME_USER": "db_user", - # "GREPTIME_PASSWORD": "db_password", -} - -db_host, db_name, db_user, db_password = "", "", "", "" -for env_var, var_name in required_env_vars.items(): - value = os.getenv(env_var) - if value is None: - raise Exception(f"Environment variable {env_var} is not set") - globals()[var_name] = value +db_http_scheme = os.getenv("GREPTIME_SCHEME") or "http" +db_host = os.getenv("GREPTIME_HOST") or "greptimedb" +db_port = os.getenv("GREPTIME_PORT") or 4000 +db_name = os.getenv("GREPTIME_DB") or "public" +db_user = os.getenv("GREPTIME_USERNAME") or "" +db_password = os.getenv("GREPTIME_PASSWORD") or "" scrape_interval = int(os.getenv("SCRAPE_INTERVAL_SEC", "60")) is_mock = bool(os.getenv("IS_MOCK", False)) # Uncomment below lines to use basic auth -# auth = f"{db_user}:{db_password}" -# b64_auth = base64.b64encode(auth.encode()).decode("ascii") -endpoint = f"http://{db_host}/v1/otlp/v1/metrics" +auth = f"{db_user}:{db_password}" +b64_auth = base64.b64encode(auth.encode()).decode("ascii") +endpoint = f"{db_http_scheme}://{db_host}:{db_port}/v1/otlp/v1/metrics" exporter = OTLPMetricExporter( endpoint=endpoint, headers={ - # "Authorization": f"Basic {b64_auth}", + "Authorization": f"Basic {b64_auth}", "x-greptime-db-name": db_name, }, timeout=5, diff --git a/ev-open-telemetry/greptime.env.sample b/ev-open-telemetry/greptime.env.sample new file mode 100644 index 0000000..8273323 --- /dev/null +++ b/ev-open-telemetry/greptime.env.sample @@ -0,0 +1,7 @@ +GREPTIME_SCHEME=https +GREPTIME_PORT=443 + +GREPTIME_HOST= +GREPTIME_DB= +GREPTIME_USERNAME= +GREPTIME_PASSWORD= diff --git a/ev-open-telemetry/greptime_ds.yml.tpl b/ev-open-telemetry/greptime_ds.yml.tpl new file mode 100644 index 0000000..f4ded45 --- /dev/null +++ b/ev-open-telemetry/greptime_ds.yml.tpl @@ -0,0 +1,14 @@ +apiVersion: 2 +datasources: + - name: greptimedb + type: prometheus + access: proxy + url: ${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/prometheus + basicAuth: true + basicAuthUser: ${GREPTIME_USERNAME} + jsonData: + httpHeaderName1: X-GREPTIME-DB-NAME + secureJsonData: + basicAuthPassword: ${GREPTIME_PASSWORD} + httpHeaderValue1: ${GREPTIME_DB:=public} + editable: false