From 0d460cf347effd112e919452d088ceb41627ca33 Mon Sep 17 00:00:00 2001 From: hynnot Date: Fri, 31 Jan 2025 14:23:10 +0100 Subject: [PATCH 01/24] First steps --- fastpath/Dockerfile | 15 +++++++++++ fastpath/debian/etc/ooni/fastpath.conf | 3 +++ fastpath/docker-compose.yml | 35 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 fastpath/Dockerfile create mode 100644 fastpath/docker-compose.yml diff --git a/fastpath/Dockerfile b/fastpath/Dockerfile new file mode 100644 index 000000000..7ba1e70c7 --- /dev/null +++ b/fastpath/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.9-slim + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +COPY debian/etc/ooni/fastpath.conf /etc/ooni/fastpath.conf + +EXPOSE 5000 + +CMD ["python", "./run_fastpath"] diff --git a/fastpath/debian/etc/ooni/fastpath.conf b/fastpath/debian/etc/ooni/fastpath.conf index 08639b1e7..982c2ca35 100644 --- a/fastpath/debian/etc/ooni/fastpath.conf +++ b/fastpath/debian/etc/ooni/fastpath.conf @@ -8,3 +8,6 @@ db_uri = postgresql://readonly@localhost/metadb # S3 access credentials s3_access_key = s3_secret_key = + + +clickhouse_url = clickhouse://default:default@clickhouse-server:9000 diff --git a/fastpath/docker-compose.yml b/fastpath/docker-compose.yml new file mode 100644 index 000000000..39c8efb88 --- /dev/null +++ b/fastpath/docker-compose.yml @@ -0,0 +1,35 @@ +services: + fastpath: + build: . + container_name: fastpath-ooni + ports: + - "5000:5000" + environment: + - CLICKHOUSE_HOST=clickhouse-server + - CLICKHOUSE_PORT=8123 + networks: + - fastpath-network + depends_on: + - clickhouse-server + + clickhouse-server: + image: yandex/clickhouse-server:latest + container_name: clickhouse-server + environment: + - CLICKHOUSE_DB=default + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD=default + ports: + - "8123:8123" + - "9000:9000" + volumes: + - clickhouse-data:/var/lib/clickhouse + networks: + - fastpath-network + +networks: + fastpath-network: + driver: bridge + +volumes: + clickhouse-data: \ No newline at end of file From ef14b0acebb60aca934c36c6814d86cd67dc7aef Mon Sep 17 00:00:00 2001 From: hynnot Date: Tue, 4 Mar 2025 11:54:28 +0100 Subject: [PATCH 02/24] Next steps --- fastpath/Dockerfile.clickhouse | 3 + fastpath/{Dockerfile => Dockerfile.fastpath} | 4 + fastpath/clickhouse_init.sql | 205 +++++++++++++++++++ fastpath/docker-compose.yml | 29 ++- fastpath/fastpath/localhttpfeeder.py | 2 + fastpath/fastpath/tests/test_fastpath_api.py | 53 +++++ fastpath/makefile | 7 +- fastpath/requirements.txt | 1 + 8 files changed, 295 insertions(+), 9 deletions(-) create mode 100644 fastpath/Dockerfile.clickhouse rename fastpath/{Dockerfile => Dockerfile.fastpath} (74%) create mode 100644 fastpath/clickhouse_init.sql create mode 100644 fastpath/fastpath/tests/test_fastpath_api.py diff --git a/fastpath/Dockerfile.clickhouse b/fastpath/Dockerfile.clickhouse new file mode 100644 index 000000000..b9b84db16 --- /dev/null +++ b/fastpath/Dockerfile.clickhouse @@ -0,0 +1,3 @@ +FROM yandex/clickhouse-server:latest + +RUN apt-get update && apt-get install -y curl diff --git a/fastpath/Dockerfile b/fastpath/Dockerfile.fastpath similarity index 74% rename from fastpath/Dockerfile rename to fastpath/Dockerfile.fastpath index 7ba1e70c7..3d7398e88 100644 --- a/fastpath/Dockerfile +++ b/fastpath/Dockerfile.fastpath @@ -6,8 +6,12 @@ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt +RUN apt-get update && apt-get install -y build-essential + COPY . . +ENV PYTHONPATH=/app + COPY debian/etc/ooni/fastpath.conf /etc/ooni/fastpath.conf EXPOSE 5000 diff --git a/fastpath/clickhouse_init.sql b/fastpath/clickhouse_init.sql new file mode 100644 index 000000000..da49958ce --- /dev/null +++ b/fastpath/clickhouse_init.sql @@ -0,0 +1,205 @@ +-- Create tables for Clickhouse integ tests + +-- Main tables + +CREATE TABLE IF NOT EXISTS default.fastpath +( + `measurement_uid` String, + `report_id` String, + `input` String, + `probe_cc` String, + `probe_asn` UInt32, + `test_name` String, + `test_start_time` DateTime, + `measurement_start_time` DateTime, + `filename` String, + `scores` String, + `platform` String, + `anomaly` String, + `confirmed` String, + `msm_failure` String, + `domain` String, + `software_name` String, + `software_version` String, + `control_failure` String, + `blocking_general` Float32, + `is_ssl_expected` Int8, + `page_len` Int32, + `page_len_ratio` Float32, + `server_cc` String, + `server_asn` Int8, + `server_as_name` String, + `update_time` DateTime64(3) MATERIALIZED now64(), + `test_version` String, + `test_runtime` Float32, + `architecture` String, + `engine_name` String, + `engine_version` String, + `blocking_type` String, + `test_helper_address` LowCardinality(String), + `test_helper_type` LowCardinality(String), + `ooni_run_link_id` Nullable(UInt64) +) +ENGINE = ReplacingMergeTree +ORDER BY (measurement_start_time, report_id, input) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS default.jsonl +( + `report_id` String, + `input` String, + `s3path` String, + `linenum` Int32, + `measurement_uid` String +) +ENGINE = MergeTree +ORDER BY (report_id, input) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS default.url_priorities ( + `sign` Int8, + `category_code` String, + `cc` String, + `domain` String, + `url` String, + `priority` Int32 +) +ENGINE = CollapsingMergeTree(sign) +ORDER BY (category_code, cc, domain, url, priority) +SETTINGS index_granularity = 1024; + +CREATE TABLE IF NOT EXISTS default.citizenlab +( + `domain` String, + `url` String, + `cc` FixedString(32), + `category_code` String +) +ENGINE = ReplacingMergeTree +ORDER BY (domain, url, cc, category_code) +SETTINGS index_granularity = 4; + +CREATE TABLE IF NOT EXISTS default.citizenlab_flip AS default.citizenlab; + +CREATE TABLE IF NOT EXISTS test_groups ( + `test_name` String, + `test_group` String +) +ENGINE = Join(ANY, LEFT, test_name); + + +-- Auth + +CREATE TABLE IF NOT EXISTS accounts +( + `account_id` FixedString(32), + `role` String +) +ENGINE = EmbeddedRocksDB +PRIMARY KEY account_id; + +CREATE TABLE IF NOT EXISTS session_expunge +( + `account_id` FixedString(32), + `threshold` DateTime DEFAULT now() +) +ENGINE = EmbeddedRocksDB +PRIMARY KEY account_id; + +-- Materialized views + +CREATE MATERIALIZED VIEW IF NOT EXISTS default.counters_test_list +( + `day` DateTime, + `probe_cc` String, + `input` String, + `msmt_cnt` UInt64 +) +ENGINE = SummingMergeTree +PARTITION BY day +ORDER BY (probe_cc, input) +SETTINGS index_granularity = 8192 AS +SELECT + toDate(measurement_start_time) AS day, + probe_cc, + input, + count() AS msmt_cnt +FROM default.fastpath +INNER JOIN default.citizenlab ON fastpath.input = citizenlab.url +WHERE (measurement_start_time < now()) AND (measurement_start_time > (now() - toIntervalDay(8))) AND (test_name = 'web_connectivity') +GROUP BY + day, + probe_cc, + input; + +CREATE MATERIALIZED VIEW IF NOT EXISTS default.counters_asn_test_list +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt32, + `input` String, + `msmt_cnt` UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (probe_cc, probe_asn, input) +SETTINGS index_granularity = 8192 AS +SELECT + toStartOfWeek(measurement_start_time) AS week, + probe_cc, + probe_asn, + input, + count() AS msmt_cnt +FROM default.fastpath +INNER JOIN default.citizenlab ON fastpath.input = citizenlab.url +WHERE (measurement_start_time < now()) AND (measurement_start_time > (now() - toIntervalDay(8))) AND (test_name = 'web_connectivity') +GROUP BY + week, + probe_cc, + probe_asn, + input; + +CREATE TABLE IF NOT EXISTS msmt_feedback +( + `measurement_uid` String, + `account_id` String, + `status` String, + `update_time` DateTime64(3) MATERIALIZED now64() +) +ENGINE = ReplacingMergeTree +ORDER BY (measurement_uid, account_id) +SETTINGS index_granularity = 4; + +CREATE TABLE IF NOT EXISTS default.fingerprints_dns +( + `name` String, + `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6), + `other_names` String, + `location_found` String, + `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4), + `pattern` String, + `confidence_no_fp` UInt8, + `expected_countries` String, + `source` String, + `exp_url` String, + `notes` String +) +ENGINE = EmbeddedRocksDB +PRIMARY KEY name; + +CREATE TABLE IF NOT EXISTS default.fingerprints_http +( + `name` String, + `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6, 'injb' = 7, 'prov' = 8), + `other_names` String, + `location_found` String, + `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4), + `pattern` String, + `confidence_no_fp` UInt8, + `expected_countries` String, + `source` String, + `exp_url` String, + `notes` String +) +ENGINE = EmbeddedRocksDB +PRIMARY KEY name; + diff --git a/fastpath/docker-compose.yml b/fastpath/docker-compose.yml index 39c8efb88..f7a869750 100644 --- a/fastpath/docker-compose.yml +++ b/fastpath/docker-compose.yml @@ -1,35 +1,52 @@ services: fastpath: - build: . + build: + context: . + dockerfile: Dockerfile.fastpath container_name: fastpath-ooni ports: - "5000:5000" + - "8472:8472" environment: - CLICKHOUSE_HOST=clickhouse-server - CLICKHOUSE_PORT=8123 + volumes: + - .:/app + working_dir: /app networks: - fastpath-network depends_on: - - clickhouse-server + clickhouse-server: + condition: service_healthy + restart: true clickhouse-server: - image: yandex/clickhouse-server:latest + build: + context: . + dockerfile: Dockerfile.clickhouse container_name: clickhouse-server environment: - CLICKHOUSE_DB=default - CLICKHOUSE_USER=default - CLICKHOUSE_PASSWORD=default ports: - - "8123:8123" - "9000:9000" + - "8123:8123" + - "9009:9009" volumes: - - clickhouse-data:/var/lib/clickhouse + - ./clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql networks: - fastpath-network + healthcheck: + test: ["CMD", "curl", "-f", "-u", "default:default", "http://localhost:8123/?query=SELECT%201"] + interval: 30s + retries: 3 + start_period: 60s + timeout: 10s networks: fastpath-network: driver: bridge volumes: - clickhouse-data: \ No newline at end of file + clickhouse-data: diff --git a/fastpath/fastpath/localhttpfeeder.py b/fastpath/fastpath/localhttpfeeder.py index 10b4d19b8..eb36b6227 100644 --- a/fastpath/fastpath/localhttpfeeder.py +++ b/fastpath/fastpath/localhttpfeeder.py @@ -21,6 +21,8 @@ def load_config(self): assert key in self.cfg.settings self.cfg.set(key, value) + self.cfg.set('reload', True) + def load(self): return self.application diff --git a/fastpath/fastpath/tests/test_fastpath_api.py b/fastpath/fastpath/tests/test_fastpath_api.py new file mode 100644 index 000000000..c3d272b79 --- /dev/null +++ b/fastpath/fastpath/tests/test_fastpath_api.py @@ -0,0 +1,53 @@ +import json +from urllib.error import HTTPError +from urllib.request import urlopen + +import pytest + + +# def test_fastpath_error_data_is_empty(): +# url = f"http://127.0.0.1:8472/" +# data = "" + +# with pytest.raises(TypeError, match="POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str."): +# urlopen(url, data) + +# def test_fastpath_error_measurement_uid_is_empty(): +# measurement_uid = "" +# url = f"http://127.0.0.1:8472/{measurement_uid}" +# data = "some_data".encode('utf-8') + +# with pytest.raises(HTTPError, match="Internal Server Error"): +# urlopen(url, data) + +# def test_fastpath_error_measurement_uid_does_not_start_with_2(): +# measurement_uid = "10210208220710.181572_MA_ndt_7888edc7748936bf" +# url = f"http://127.0.0.1:8472/{measurement_uid}" +# data = "some_data".encode('utf-8') + +# with pytest.raises(HTTPError, match="Internal Server Error"): +# urlopen(url, data) + +# def test_fastpath_response_ok(): +# measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" +# url = f"http://127.0.0.1:8472/{measurement_uid}" +# data = {} + +# response = urlopen(url, json.dumps(data).encode('utf-8')) + +# assert response.getcode() == 200 +# assert response.read() == b"" + +def test_fastpath_TESTS(): + measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" + url = f"http://127.0.0.1:8472/{measurement_uid}" + data = { + 'report_id': 'report_id', + # 'input': 'input', + 'probe_cc': 'ZZ' + } + + response = urlopen(url, json.dumps(data).encode('utf-8')) + + assert response.getcode() == 200 + assert response.read() == b"" \ No newline at end of file diff --git a/fastpath/makefile b/fastpath/makefile index 2cfa6ad47..5bd09e720 100644 --- a/fastpath/makefile +++ b/fastpath/makefile @@ -1,10 +1,11 @@ local_quickdeploy: - sudo cp fastpath/*.py /usr/lib/python3.7/dist-packages/fastpath/ - sudo systemctl restart fastpath + cp fastpath/*.py /usr/local/lib/python3.9/ + systemctl restart fastpath local_functests: - PYTHONPATH=. pytest-3 -s --log-cli-level info $(args) + # PYTHONPATH=. pytest-3 -s --log-cli-level info $(args) + pytest -s --log-cli-level info $(args) local_functests_coverage: PYTHONPATH=. pytest-3 -s --cov=fastpath diff --git a/fastpath/requirements.txt b/fastpath/requirements.txt index 891b64ce8..ad584de09 100644 --- a/fastpath/requirements.txt +++ b/fastpath/requirements.txt @@ -8,3 +8,4 @@ gunicorn psycopg2-binary # systemd <- This is an optional requirement on linux clickhouse-driver +pytest From 9342b85d6e0a91b1482390515495a97f8d91f3ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 29 Apr 2025 16:32:25 +0200 Subject: [PATCH 03/24] Use right source for clickhouse image --- fastpath/Dockerfile.clickhouse | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastpath/Dockerfile.clickhouse b/fastpath/Dockerfile.clickhouse index b9b84db16..6e9c3b727 100644 --- a/fastpath/Dockerfile.clickhouse +++ b/fastpath/Dockerfile.clickhouse @@ -1,3 +1,3 @@ -FROM yandex/clickhouse-server:latest +FROM clickhouse/clickhouse-server:latest RUN apt-get update && apt-get install -y curl From 8863b732d993eb776eaed8b381f93531bd75877f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 10:41:16 +0200 Subject: [PATCH 04/24] Add requests library used in tests --- fastpath/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/fastpath/requirements.txt b/fastpath/requirements.txt index ad584de09..0d2998b8e 100644 --- a/fastpath/requirements.txt +++ b/fastpath/requirements.txt @@ -9,3 +9,4 @@ psycopg2-binary # systemd <- This is an optional requirement on linux clickhouse-driver pytest +requests From 62a06e44c24f3dd4fbd30f5ad8688a52e66ad64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 10:46:19 +0200 Subject: [PATCH 05/24] Add docker related operations --- fastpath/makefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fastpath/makefile b/fastpath/makefile index 5bd09e720..aa6f0ca40 100644 --- a/fastpath/makefile +++ b/fastpath/makefile @@ -59,3 +59,19 @@ beta_functests: beta_monitor_metrics: ssh $(shell cat .betahost) \ "tcpdump -npqi lo udp port 8125 -A -l | grep --line-buffered fastpat | sed 's/.*fastpath\.//'" + +docker: + docker compose --profile default up --build -d + +docker-test: + docker compose --profile test up --build -d + +docker-all: + docker compose --profile all up --build -d + +docker-down: + docker compose --profile all down + +# Use this to login into the fastpath service container, useful for testing +docker-login: + docker compose exec fastpath bash \ No newline at end of file From 59572bd979ffed3741e1b3fbf05304289166882f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 10:48:26 +0200 Subject: [PATCH 06/24] Added profiles to select which services to run --- fastpath/docker-compose.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/fastpath/docker-compose.yml b/fastpath/docker-compose.yml index f7a869750..d273ec314 100644 --- a/fastpath/docker-compose.yml +++ b/fastpath/docker-compose.yml @@ -3,7 +3,7 @@ services: build: context: . dockerfile: Dockerfile.fastpath - container_name: fastpath-ooni + container_name: ooni-fastpath ports: - "5000:5000" - "8472:8472" @@ -13,12 +13,14 @@ services: volumes: - .:/app working_dir: /app - networks: - - fastpath-network depends_on: clickhouse-server: condition: service_healthy restart: true + profiles: + - default + - all + - test clickhouse-server: build: @@ -35,18 +37,15 @@ services: - "9009:9009" volumes: - ./clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql - networks: - - fastpath-network healthcheck: test: ["CMD", "curl", "-f", "-u", "default:default", "http://localhost:8123/?query=SELECT%201"] interval: 30s retries: 3 start_period: 60s timeout: 10s - -networks: - fastpath-network: - driver: bridge + profiles: + - all + - test volumes: clickhouse-data: From 78a9dad90df66148a3e5261490861810708e3aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 10:49:30 +0200 Subject: [PATCH 07/24] Add context comment --- fastpath/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/fastpath/docker-compose.yml b/fastpath/docker-compose.yml index d273ec314..c15e07e4c 100644 --- a/fastpath/docker-compose.yml +++ b/fastpath/docker-compose.yml @@ -22,6 +22,7 @@ services: - all - test + # This service is used only for testing, in prod we use the actual clickhouse db clickhouse-server: build: context: . From a592191dd4898c197512dc4bfd6376543dd28551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 11:03:55 +0200 Subject: [PATCH 08/24] Remove dependency on clickhouse to allow fastpath to run standalone --- fastpath/docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fastpath/docker-compose.yml b/fastpath/docker-compose.yml index c15e07e4c..2d245f224 100644 --- a/fastpath/docker-compose.yml +++ b/fastpath/docker-compose.yml @@ -13,10 +13,6 @@ services: volumes: - .:/app working_dir: /app - depends_on: - clickhouse-server: - condition: service_healthy - restart: true profiles: - default - all From dd59865b16edfb64a088847e7d80da5a2c3df116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 11:06:56 +0200 Subject: [PATCH 09/24] binding to 0.0.0.0 to accept traffic from outside docker --- fastpath/fastpath/localhttpfeeder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastpath/fastpath/localhttpfeeder.py b/fastpath/fastpath/localhttpfeeder.py index eb36b6227..69f945230 100644 --- a/fastpath/fastpath/localhttpfeeder.py +++ b/fastpath/fastpath/localhttpfeeder.py @@ -42,5 +42,5 @@ def handler_app(environ, start_response): start_response("200 OK", []) return [b""] - options = {"bind": f"127.0.0.1:{API_PORT}"} + options = {"bind": f"0.0.0.0:{API_PORT}"} MsmtFeeder(handler_app, options).run() From 2d0b94b48cba60a4638fed7427e53735eb250593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 11:31:36 +0200 Subject: [PATCH 10/24] Using simpler dockerfile configuration --- fastpath/{Dockerfile.fastpath => Dockerfile} | 0 fastpath/Dockerfile.clickhouse | 3 --- 2 files changed, 3 deletions(-) rename fastpath/{Dockerfile.fastpath => Dockerfile} (100%) delete mode 100644 fastpath/Dockerfile.clickhouse diff --git a/fastpath/Dockerfile.fastpath b/fastpath/Dockerfile similarity index 100% rename from fastpath/Dockerfile.fastpath rename to fastpath/Dockerfile diff --git a/fastpath/Dockerfile.clickhouse b/fastpath/Dockerfile.clickhouse deleted file mode 100644 index 6e9c3b727..000000000 --- a/fastpath/Dockerfile.clickhouse +++ /dev/null @@ -1,3 +0,0 @@ -FROM clickhouse/clickhouse-server:latest - -RUN apt-get update && apt-get install -y curl From d1827d124a0b4d16af83319383ec96a1ec8ddb50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 11:31:50 +0200 Subject: [PATCH 11/24] Using simpler dockerfile configuration --- fastpath/docker-compose.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fastpath/docker-compose.yml b/fastpath/docker-compose.yml index 2d245f224..e46200d94 100644 --- a/fastpath/docker-compose.yml +++ b/fastpath/docker-compose.yml @@ -2,7 +2,7 @@ services: fastpath: build: context: . - dockerfile: Dockerfile.fastpath + dockerfile: Dockerfile container_name: ooni-fastpath ports: - "5000:5000" @@ -20,9 +20,7 @@ services: # This service is used only for testing, in prod we use the actual clickhouse db clickhouse-server: - build: - context: . - dockerfile: Dockerfile.clickhouse + image: clickhouse/clickhouse-server:latest container_name: clickhouse-server environment: - CLICKHOUSE_DB=default From cc4be7eb3534ddb1e8c67b698512b4e4b9b10a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 12:54:04 +0200 Subject: [PATCH 12/24] Fixing dependency betwen fastpath and clickhouse --- fastpath/docker-compose.yml | 6 +----- fastpath/makefile | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/fastpath/docker-compose.yml b/fastpath/docker-compose.yml index e46200d94..b2a5b70b4 100644 --- a/fastpath/docker-compose.yml +++ b/fastpath/docker-compose.yml @@ -7,16 +7,12 @@ services: ports: - "5000:5000" - "8472:8472" - environment: - - CLICKHOUSE_HOST=clickhouse-server - - CLICKHOUSE_PORT=8123 volumes: - .:/app working_dir: /app profiles: - default - all - - test # This service is used only for testing, in prod we use the actual clickhouse db clickhouse-server: @@ -40,7 +36,7 @@ services: timeout: 10s profiles: - all - - test + - clickhouse volumes: clickhouse-data: diff --git a/fastpath/makefile b/fastpath/makefile index aa6f0ca40..f0660d9ca 100644 --- a/fastpath/makefile +++ b/fastpath/makefile @@ -63,15 +63,26 @@ beta_monitor_metrics: docker: docker compose --profile default up --build -d -docker-test: - docker compose --profile test up --build -d +# Runs docker in foreground, useful for checking errors in the image before it runs +docker-fg: + docker compose --profile default up --build -docker-all: - docker compose --profile all up --build -d +# Runs both fastpath and the testing clickhous. +# Mind the fastpath configuration in fastpath.conf +docker-all: docker-clickhouse + docker compose --profile default up --build -d docker-down: docker compose --profile all down +# If you need to test the fastpath locally you can use this rule to spawn the clickhouse database +# locally and then use `make docker` or `make docker-fg` to start the fastpath container +docker-clickhouse: + docker compose --profile clickhouse up -d + # Use this to login into the fastpath service container, useful for testing docker-login: - docker compose exec fastpath bash \ No newline at end of file + docker compose exec fastpath bash + +docker-logs: + docker compose logs fastapth -f $(args) From f696e9ea73c8816541278aa718ad41ead60a3511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Thu, 1 May 2025 13:05:38 +0200 Subject: [PATCH 13/24] Improving usability of makefile --- fastpath/makefile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/fastpath/makefile b/fastpath/makefile index f0660d9ca..5c06db5ba 100644 --- a/fastpath/makefile +++ b/fastpath/makefile @@ -70,13 +70,21 @@ docker-fg: # Runs both fastpath and the testing clickhous. # Mind the fastpath configuration in fastpath.conf docker-all: docker-clickhouse - docker compose --profile default up --build -d + echo "Waiting for clickhouse..." + sleep 4 + docker compose --profile default up --build -d +# Turns off every service docker-down: docker compose --profile all down # If you need to test the fastpath locally you can use this rule to spawn the clickhouse database -# locally and then use `make docker` or `make docker-fg` to start the fastpath container +# locally and then use `make docker` or `make docker-fg` to start the fastpath container. Ex: +# ``` +# make docker-clickhouse +# make docker +# ``` +# You can also use `make docker-all` for that purpose docker-clickhouse: docker compose --profile clickhouse up -d @@ -84,5 +92,11 @@ docker-clickhouse: docker-login: docker compose exec fastpath bash +# Get logs from the fastpath docker service docker-logs: - docker compose logs fastapth -f $(args) + docker compose logs fastpath -f + +# Get logs for a specified service. Example: +# `make docker-logs-for args="clickhouse-server"` +docker-logs-for: + docker compose logs $(args) -f From d9c647e4a1a28a0155c097cdfe057384e38f275a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 2 May 2025 10:17:49 +0200 Subject: [PATCH 14/24] Creating multi stage docker file --- fastpath/Dockerfile | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/fastpath/Dockerfile b/fastpath/Dockerfile index 3d7398e88..f61b25972 100644 --- a/fastpath/Dockerfile +++ b/fastpath/Dockerfile @@ -1,18 +1,29 @@ -FROM python:3.9-slim +# Stage 1: Building +FROM python:slim AS builder + +RUN apt update && apt install -y --no-install-recommends \ + build-essential \ + gcc \ + && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --prefix=/install --no-cache-dir -r requirements.txt + +# Stage 2: Running +FROM python:slim -RUN apt-get update && apt-get install -y build-essential +RUN apt update && rm -rf /var/lib/apt/lists/* +WORKDIR /app + +COPY --from=builder /install /usr/local COPY . . ENV PYTHONPATH=/app - -COPY debian/etc/ooni/fastpath.conf /etc/ooni/fastpath.conf +COPY fastpath.conf /etc/ooni/fastpath.conf EXPOSE 5000 From 5e78cc466bc77d7300caecd96b066fba4bae977f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 2 May 2025 10:58:33 +0200 Subject: [PATCH 15/24] Remove deprecated pkg_resources dependency --- fastpath/fastpath/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fastpath/fastpath/core.py b/fastpath/fastpath/core.py index 9593324aa..8934de6e2 100644 --- a/fastpath/fastpath/core.py +++ b/fastpath/fastpath/core.py @@ -25,7 +25,7 @@ import time import yaml -from pkg_resources import parse_version +from packaging.version import Version import ujson # debdeps: python3-ujson try: @@ -1367,14 +1367,14 @@ def score_signal(msm: dict) -> dict: scores["accuracy"] = 0.0 return scores - if parse_version(tv) <= parse_version("0.2.3") and start_time >= datetime( + if Version(tv) <= Version("0.2.3") and start_time >= datetime( 2023, 11, 7 ): # https://github.com/ooni/probe/issues/2627 scores["accuracy"] = 0.0 return scores - if parse_version(tv) < parse_version("0.2.2") and start_time >= datetime( + if Version(tv) < Version("0.2.2") and start_time >= datetime( 2022, 10, 19 ): scores["accuracy"] = 0.0 @@ -1384,7 +1384,7 @@ def score_signal(msm: dict) -> dict: # engine_version < 3.17.2 and measurement_start_time > 2023-05-02 annot = g_or(msm, "annotations", {}) ev = g_or(annot, "engine_version", "0.0.0") - if parse_version(ev) < parse_version("3.17.2") and start_time >= datetime( + if Version(ev) < Version("3.17.2") and start_time >= datetime( 2023, 5, 2 ): scores["accuracy"] = 0.0 From 41f7176a040c72137144cda6094c9b8cac3b084c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 2 May 2025 11:02:04 +0200 Subject: [PATCH 16/24] Skip broken tests --- .../fastpath/tests/test_functional_normalize.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fastpath/fastpath/tests/test_functional_normalize.py b/fastpath/fastpath/tests/test_functional_normalize.py index c77f7bf07..f76798b45 100644 --- a/fastpath/fastpath/tests/test_functional_normalize.py +++ b/fastpath/fastpath/tests/test_functional_normalize.py @@ -217,15 +217,23 @@ def test_normalize_json(cans): assert hash(entry) == expected[n] +@pytest.mark.skip("YAML ingestion deprecated") def test_generate_report_id_empty(): header = {} + # This generate_report_id function is bugged bc it uses naive datetimes, so + # it will generate a different id depending on the timezone configuration of + # the machine running the code report_id = norm.generate_report_id(header) exp = "19700101T010000Z_KWnRnnxAmNrJfoqrTxAKhVDgGkiuSYfGDSecYaayqhcqlfOXCX" assert report_id == exp +@pytest.mark.skip("YAML ingestion deprecated") def test_generate_report_id(): header = dict(probe_cc="UK", test_name="web_connectivity") + # This generate_report_id function is bugged bc it uses naive datetimes, so + # it will generate a different id depending on the timezone configuration of + # the machine running the code report_id = norm.generate_report_id(header) exp = "19700101T010000Z_LLWQMcPHNefGtRNzxcgKlXlSjKmRuyyKLycBDGwNiNEbMztVzb" assert report_id == exp @@ -275,12 +283,20 @@ def test_normalize_yaml_dns_consistency_2017(cans): rfn = canfn.split("/", 1)[1][:-4] # remove testdata/ and .lz4 # s3://ooni-data/autoclaved/jsonl.tar.lz4/2017-12-21/20171220T153044Z-BE-AS5432-dns_consistency-mnKRlHuqk8Eo6XMJt5ZkVQrgReaEXPEWaO9NafgXxSVIhAswTXT7QJc6zhsuttpK-0.1.0-probe.yaml.lz4 # lz4cat | head -n1 | jq -S . > fastpath/tests/data/yaml17_0.json + from pprint import pprint with lz4frame.open(can) as f: for n, entry in enumerate(norm.iter_yaml_msmt_normalized(f, day, rfn)): ujson.dumps(entry) # ensure it's serializable if n == 0: with open("fastpath/tests/data/yaml17_0.json") as f: exp = ujson.load(f) + + print("=== ENTRY ===") + pprint(entry) + + print("=== EXP ===") + pprint(exp) + assert entry == exp elif n > 20: break From 522a851210233549ea9e55958fff5f25bf09b38f Mon Sep 17 00:00:00 2001 From: Luis Diaz <41093870+LDiazN@users.noreply.github.com> Date: Fri, 2 May 2025 11:48:18 +0200 Subject: [PATCH 17/24] Upgrade deprecated upload-artifact action in test_fastpath.yml --- .github/workflows/test_fastpath.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_fastpath.yml b/.github/workflows/test_fastpath.yml index 8f32b67e4..dc0d9f33e 100644 --- a/.github/workflows/test_fastpath.yml +++ b/.github/workflows/test_fastpath.yml @@ -29,7 +29,7 @@ jobs: fastpath/tests/test_unit.py - name: Archive code coverage HTML pages - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: coverage path: fastpath/htmlcov From 2d19f5155c16a4c6a232c246d9a8e08d6418c566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 2 May 2025 11:50:09 +0200 Subject: [PATCH 18/24] Move fastpath.conf to the top dir --- fastpath/fastpath.conf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 fastpath/fastpath.conf diff --git a/fastpath/fastpath.conf b/fastpath/fastpath.conf new file mode 100644 index 000000000..24dc93590 --- /dev/null +++ b/fastpath/fastpath.conf @@ -0,0 +1,15 @@ +# This is the default configration file used by Docker. Replace it or modify it to +# to set up docker +[DEFAULT] +# Collector hostnames, comma separated +collectors = localhost + +# Database connection URI +db_uri = postgresql://readonly@localhost/metadb + +# S3 access credentials +s3_access_key = +s3_secret_key = + + +clickhouse_url = clickhouse://default:default@clickhouse-server:9000 From e65dae0aaa6e34e90d8d5aca29f58a738b9d4772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 2 May 2025 12:26:19 +0200 Subject: [PATCH 19/24] remove useless print --- fastpath/fastpath/tests/test_functional_normalize.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fastpath/fastpath/tests/test_functional_normalize.py b/fastpath/fastpath/tests/test_functional_normalize.py index f76798b45..c21c3d7af 100644 --- a/fastpath/fastpath/tests/test_functional_normalize.py +++ b/fastpath/fastpath/tests/test_functional_normalize.py @@ -283,20 +283,12 @@ def test_normalize_yaml_dns_consistency_2017(cans): rfn = canfn.split("/", 1)[1][:-4] # remove testdata/ and .lz4 # s3://ooni-data/autoclaved/jsonl.tar.lz4/2017-12-21/20171220T153044Z-BE-AS5432-dns_consistency-mnKRlHuqk8Eo6XMJt5ZkVQrgReaEXPEWaO9NafgXxSVIhAswTXT7QJc6zhsuttpK-0.1.0-probe.yaml.lz4 # lz4cat | head -n1 | jq -S . > fastpath/tests/data/yaml17_0.json - from pprint import pprint with lz4frame.open(can) as f: for n, entry in enumerate(norm.iter_yaml_msmt_normalized(f, day, rfn)): ujson.dumps(entry) # ensure it's serializable if n == 0: with open("fastpath/tests/data/yaml17_0.json") as f: exp = ujson.load(f) - - print("=== ENTRY ===") - pprint(entry) - - print("=== EXP ===") - pprint(exp) - assert entry == exp elif n > 20: break From 9f9c28639c3f1ed470a6887d612289341ce1b79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Fri, 2 May 2025 16:41:50 +0200 Subject: [PATCH 20/24] Working on adding integration tests for the docker deployment --- fastpath/fastpath/tests/docker-compose.yml | 35 ++++++++++++ fastpath/fastpath/tests/integ/conftest.py | 40 ++++++++++++++ .../fastpath/tests/integ/test_fastpath_api.py | 55 +++++++++++++++++++ fastpath/requirements.txt | 1 + 4 files changed, 131 insertions(+) create mode 100644 fastpath/fastpath/tests/docker-compose.yml create mode 100644 fastpath/fastpath/tests/integ/conftest.py create mode 100644 fastpath/fastpath/tests/integ/test_fastpath_api.py diff --git a/fastpath/fastpath/tests/docker-compose.yml b/fastpath/fastpath/tests/docker-compose.yml new file mode 100644 index 000000000..c5072d18d --- /dev/null +++ b/fastpath/fastpath/tests/docker-compose.yml @@ -0,0 +1,35 @@ +services: + fastpath: + build: + context: ../../ + dockerfile: Dockerfile + ports: + - "5000" + - "8472" + working_dir: /app + depends_on: + test-clickhouse-server: + condition: service_healthy + + # This service is used only for testing, in prod we use the actual clickhouse db + test-clickhouse-server: + image: clickhouse/clickhouse-server:latest + environment: + - CLICKHOUSE_DB=default + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD=default + ports: + - "9000" + - "8123" + - "9009" + volumes: + - ../../clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql + healthcheck: + test: ["CMD", "curl", "-f", "-u", "default:default", "http://localhost:8123/?query=SELECT%201"] + interval: 30s + retries: 3 + start_period: 60s + timeout: 10s + +volumes: + clickhouse-data: diff --git a/fastpath/fastpath/tests/integ/conftest.py b/fastpath/fastpath/tests/integ/conftest.py new file mode 100644 index 000000000..4a86d54ea --- /dev/null +++ b/fastpath/fastpath/tests/integ/conftest.py @@ -0,0 +1,40 @@ +import pytest +import requests +from clickhouse_driver.client import Client as ClickhouseClient + + +@pytest.fixture(scope="session") +def clickhouse_server(docker_ip, docker_services): + port = docker_services.port_for("clickhouse", 9000) + url = "clickhouse://test:test@{}:{}".format(docker_ip, port) + docker_services.wait_until_responsive( + timeout=30.0, pause=0.1, check=lambda: is_clickhouse_running(url) + ) + yield url + +@pytest.fixture(scope="session") +def fastpath_service(docker_ip, docker_services): + port = docker_services.port_for("fastpath", 8472) + url = f"http://{docker_ip}:{port}" + docker_services.wait_until_responsive( + timeout=20, pause=0.1, check=lambda: is_fastpath_running(url) + ) + + yield url + +def is_fastpath_running(url : str) -> bool: + print("checking if fastpath is running...") + try: + req = requests.get(url) + assert req.status_code == 200 + return True + except Exception: + return False + +def is_clickhouse_running(url): + try: + with ClickhouseClient.from_url(url) as client: + client.execute("SELECT 1") + return True + except Exception: + return False diff --git a/fastpath/fastpath/tests/integ/test_fastpath_api.py b/fastpath/fastpath/tests/integ/test_fastpath_api.py new file mode 100644 index 000000000..2e7827faa --- /dev/null +++ b/fastpath/fastpath/tests/integ/test_fastpath_api.py @@ -0,0 +1,55 @@ +import json +from urllib.error import HTTPError +from urllib.request import urlopen + +import pytest + + +# def test_fastpath_error_data_is_empty(): +# url = f"http://127.0.0.1:8472/" +# data = "" + +# with pytest.raises(TypeError, match="POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str."): +# urlopen(url, data) + +# def test_fastpath_error_measurement_uid_is_empty(): +# measurement_uid = "" +# url = f"http://127.0.0.1:8472/{measurement_uid}" +# data = "some_data".encode('utf-8') + +# with pytest.raises(HTTPError, match="Internal Server Error"): +# urlopen(url, data) + +# def test_fastpath_error_measurement_uid_does_not_start_with_2(): +# measurement_uid = "10210208220710.181572_MA_ndt_7888edc7748936bf" +# url = f"http://127.0.0.1:8472/{measurement_uid}" +# data = "some_data".encode('utf-8') + +# with pytest.raises(HTTPError, match="Internal Server Error"): +# urlopen(url, data) + +# def test_fastpath_response_ok(): +# measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" +# url = f"http://127.0.0.1:8472/{measurement_uid}" +# data = {} + +# response = urlopen(url, json.dumps(data).encode('utf-8')) + +# assert response.getcode() == 200 +# assert response.read() == b"" + +def test_fastpath_TESTS(fastpath_service): + raise ValueError("Crashing on purpose") + print(fastpath_service) + measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" + url = f"http://127.0.0.1:8472/{measurement_uid}" + data = { + 'report_id': 'report_id', + # 'input': 'input', + 'probe_cc': 'ZZ' + } + + response = urlopen(url, json.dumps(data).encode('utf-8')) + + assert response.getcode() == 200 + assert response.read() == b"" \ No newline at end of file diff --git a/fastpath/requirements.txt b/fastpath/requirements.txt index 0d2998b8e..9a00ab30b 100644 --- a/fastpath/requirements.txt +++ b/fastpath/requirements.txt @@ -10,3 +10,4 @@ psycopg2-binary clickhouse-driver pytest requests +pytest-docker \ No newline at end of file From 894d3e0786f64af3de5abcf42bd3f82960a7aabb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 5 May 2025 10:48:35 +0200 Subject: [PATCH 21/24] Added integration tests with docker --- fastpath/Dockerfile | 2 +- fastpath/fastpath/tests/docker-compose.yml | 11 ++-- fastpath/fastpath/tests/integ/conftest.py | 17 +++--- .../fastpath/tests/integ/test_fastpath_api.py | 4 +- fastpath/fastpath/tests/test_fastpath_api.py | 53 ------------------- 5 files changed, 14 insertions(+), 73 deletions(-) delete mode 100644 fastpath/fastpath/tests/test_fastpath_api.py diff --git a/fastpath/Dockerfile b/fastpath/Dockerfile index f61b25972..c78b677ef 100644 --- a/fastpath/Dockerfile +++ b/fastpath/Dockerfile @@ -27,4 +27,4 @@ COPY fastpath.conf /etc/ooni/fastpath.conf EXPOSE 5000 -CMD ["python", "./run_fastpath"] +CMD ["python", "/app/run_fastpath"] diff --git a/fastpath/fastpath/tests/docker-compose.yml b/fastpath/fastpath/tests/docker-compose.yml index c5072d18d..4e0c98e7b 100644 --- a/fastpath/fastpath/tests/docker-compose.yml +++ b/fastpath/fastpath/tests/docker-compose.yml @@ -8,14 +8,12 @@ services: - "8472" working_dir: /app depends_on: - test-clickhouse-server: + clickhouse-server: condition: service_healthy - # This service is used only for testing, in prod we use the actual clickhouse db - test-clickhouse-server: + clickhouse-server: image: clickhouse/clickhouse-server:latest environment: - - CLICKHOUSE_DB=default - CLICKHOUSE_USER=default - CLICKHOUSE_PASSWORD=default ports: @@ -25,11 +23,8 @@ services: volumes: - ../../clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql healthcheck: - test: ["CMD", "curl", "-f", "-u", "default:default", "http://localhost:8123/?query=SELECT%201"] + test: ["CMD", "clickhouse-client", "--query", "select 1;"] interval: 30s retries: 3 start_period: 60s timeout: 10s - -volumes: - clickhouse-data: diff --git a/fastpath/fastpath/tests/integ/conftest.py b/fastpath/fastpath/tests/integ/conftest.py index 4a86d54ea..0961c0891 100644 --- a/fastpath/fastpath/tests/integ/conftest.py +++ b/fastpath/fastpath/tests/integ/conftest.py @@ -2,22 +2,24 @@ import requests from clickhouse_driver.client import Client as ClickhouseClient +# Time to wait for docker services +TIMEOUT = 10.0 @pytest.fixture(scope="session") -def clickhouse_server(docker_ip, docker_services): - port = docker_services.port_for("clickhouse", 9000) - url = "clickhouse://test:test@{}:{}".format(docker_ip, port) +def clickhouse_service(docker_ip, docker_services): + port = docker_services.port_for("clickhouse-server", 9000) + url = "clickhouse://default:default@{}:{}".format(docker_ip, port) docker_services.wait_until_responsive( - timeout=30.0, pause=0.1, check=lambda: is_clickhouse_running(url) + timeout=TIMEOUT, pause=0.1, check=lambda: is_clickhouse_running(url) ) yield url @pytest.fixture(scope="session") -def fastpath_service(docker_ip, docker_services): +def fastpath_service(docker_ip, docker_services, clickhouse_service): port = docker_services.port_for("fastpath", 8472) url = f"http://{docker_ip}:{port}" docker_services.wait_until_responsive( - timeout=20, pause=0.1, check=lambda: is_fastpath_running(url) + timeout=TIMEOUT, pause=0.1, check=lambda: is_fastpath_running(url) ) yield url @@ -26,8 +28,7 @@ def is_fastpath_running(url : str) -> bool: print("checking if fastpath is running...") try: req = requests.get(url) - assert req.status_code == 200 - return True + return req.status_code == 200 except Exception: return False diff --git a/fastpath/fastpath/tests/integ/test_fastpath_api.py b/fastpath/fastpath/tests/integ/test_fastpath_api.py index 2e7827faa..0c5c597cf 100644 --- a/fastpath/fastpath/tests/integ/test_fastpath_api.py +++ b/fastpath/fastpath/tests/integ/test_fastpath_api.py @@ -39,10 +39,8 @@ # assert response.read() == b"" def test_fastpath_TESTS(fastpath_service): - raise ValueError("Crashing on purpose") - print(fastpath_service) measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" - url = f"http://127.0.0.1:8472/{measurement_uid}" + url = f"{fastpath_service}/{measurement_uid}" data = { 'report_id': 'report_id', # 'input': 'input', diff --git a/fastpath/fastpath/tests/test_fastpath_api.py b/fastpath/fastpath/tests/test_fastpath_api.py deleted file mode 100644 index c3d272b79..000000000 --- a/fastpath/fastpath/tests/test_fastpath_api.py +++ /dev/null @@ -1,53 +0,0 @@ -import json -from urllib.error import HTTPError -from urllib.request import urlopen - -import pytest - - -# def test_fastpath_error_data_is_empty(): -# url = f"http://127.0.0.1:8472/" -# data = "" - -# with pytest.raises(TypeError, match="POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str."): -# urlopen(url, data) - -# def test_fastpath_error_measurement_uid_is_empty(): -# measurement_uid = "" -# url = f"http://127.0.0.1:8472/{measurement_uid}" -# data = "some_data".encode('utf-8') - -# with pytest.raises(HTTPError, match="Internal Server Error"): -# urlopen(url, data) - -# def test_fastpath_error_measurement_uid_does_not_start_with_2(): -# measurement_uid = "10210208220710.181572_MA_ndt_7888edc7748936bf" -# url = f"http://127.0.0.1:8472/{measurement_uid}" -# data = "some_data".encode('utf-8') - -# with pytest.raises(HTTPError, match="Internal Server Error"): -# urlopen(url, data) - -# def test_fastpath_response_ok(): -# measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" -# url = f"http://127.0.0.1:8472/{measurement_uid}" -# data = {} - -# response = urlopen(url, json.dumps(data).encode('utf-8')) - -# assert response.getcode() == 200 -# assert response.read() == b"" - -def test_fastpath_TESTS(): - measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" - url = f"http://127.0.0.1:8472/{measurement_uid}" - data = { - 'report_id': 'report_id', - # 'input': 'input', - 'probe_cc': 'ZZ' - } - - response = urlopen(url, json.dumps(data).encode('utf-8')) - - assert response.getcode() == 200 - assert response.read() == b"" \ No newline at end of file From 6457384f6aa1f1aca94eec82929e45fa89c85981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 5 May 2025 11:02:00 +0200 Subject: [PATCH 22/24] Added requests to simplify testing --- fastpath/fastpath/tests/integ/test_fastpath_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fastpath/fastpath/tests/integ/test_fastpath_api.py b/fastpath/fastpath/tests/integ/test_fastpath_api.py index 0c5c597cf..b01b7b221 100644 --- a/fastpath/fastpath/tests/integ/test_fastpath_api.py +++ b/fastpath/fastpath/tests/integ/test_fastpath_api.py @@ -1,6 +1,7 @@ import json from urllib.error import HTTPError from urllib.request import urlopen +import requests as r import pytest @@ -38,7 +39,7 @@ # assert response.getcode() == 200 # assert response.read() == b"" -def test_fastpath_TESTS(fastpath_service): +def test_fastpath_basic(fastpath_service): measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" url = f"{fastpath_service}/{measurement_uid}" data = { @@ -47,7 +48,7 @@ def test_fastpath_TESTS(fastpath_service): 'probe_cc': 'ZZ' } - response = urlopen(url, json.dumps(data).encode('utf-8')) + response = r.get(url, data=data) - assert response.getcode() == 200 - assert response.read() == b"" \ No newline at end of file + assert response.status_code == 200 + assert response.content == b"" \ No newline at end of file From 397908ed8c7f0addcd3f29923d05bf0dc0cf23b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Mon, 5 May 2025 11:47:10 +0200 Subject: [PATCH 23/24] Add more integration tests --- .../fastpath/tests/integ/test_fastpath_api.py | 52 +++++++------------ 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/fastpath/fastpath/tests/integ/test_fastpath_api.py b/fastpath/fastpath/tests/integ/test_fastpath_api.py index b01b7b221..1c2eb410c 100644 --- a/fastpath/fastpath/tests/integ/test_fastpath_api.py +++ b/fastpath/fastpath/tests/integ/test_fastpath_api.py @@ -1,43 +1,31 @@ -import json -from urllib.error import HTTPError -from urllib.request import urlopen import requests as r -import pytest +def test_fastpath_error_measurement_uid_is_empty(fastpath_service): + measurement_uid = "" + url = f"{fastpath_service}/{measurement_uid}" + resp = r.post(url, data={}) + assert resp.status_code == 500 + assert "Internal Server Error" in resp.content.decode() -# def test_fastpath_error_data_is_empty(): -# url = f"http://127.0.0.1:8472/" -# data = "" - -# with pytest.raises(TypeError, match="POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str."): -# urlopen(url, data) - -# def test_fastpath_error_measurement_uid_is_empty(): -# measurement_uid = "" -# url = f"http://127.0.0.1:8472/{measurement_uid}" -# data = "some_data".encode('utf-8') - -# with pytest.raises(HTTPError, match="Internal Server Error"): -# urlopen(url, data) +def test_fastpath_error_measurement_uid_does_not_start_with_2(fastpath_service): + measurement_uid = "10210208220710.181572_MA_ndt_7888edc7748936bf" + url = f"{fastpath_service}/{measurement_uid}" + resp = r.post(url, data = b"") -# def test_fastpath_error_measurement_uid_does_not_start_with_2(): -# measurement_uid = "10210208220710.181572_MA_ndt_7888edc7748936bf" -# url = f"http://127.0.0.1:8472/{measurement_uid}" -# data = "some_data".encode('utf-8') + assert resp.status_code == 500 + assert "Internal Server Error" in resp.content.decode() -# with pytest.raises(HTTPError, match="Internal Server Error"): -# urlopen(url, data) -# def test_fastpath_response_ok(): -# measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" -# url = f"http://127.0.0.1:8472/{measurement_uid}" -# data = {} +def test_fastpath_empty_response_ok(fastpath_service): + measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" + url = f"{fastpath_service}/{measurement_uid}" + data = {} -# response = urlopen(url, json.dumps(data).encode('utf-8')) + response = r.post(url, data=data) -# assert response.getcode() == 200 -# assert response.read() == b"" + assert response.status_code == 200 + assert response.content == b"" def test_fastpath_basic(fastpath_service): measurement_uid = "20210208220710.181572_MA_ndt_7888edc7748936bf" @@ -48,7 +36,7 @@ def test_fastpath_basic(fastpath_service): 'probe_cc': 'ZZ' } - response = r.get(url, data=data) + response = r.post(url, data=data) assert response.status_code == 200 assert response.content == b"" \ No newline at end of file From c1a1c2d117348846eb8d8ead53051963bd1623df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 7 May 2025 12:46:12 +0200 Subject: [PATCH 24/24] Add better healthcheck to docker compose --- fastpath/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastpath/docker-compose.yml b/fastpath/docker-compose.yml index b2a5b70b4..feb59118e 100644 --- a/fastpath/docker-compose.yml +++ b/fastpath/docker-compose.yml @@ -29,7 +29,7 @@ services: volumes: - ./clickhouse_init.sql:/docker-entrypoint-initdb.d/init.sql healthcheck: - test: ["CMD", "curl", "-f", "-u", "default:default", "http://localhost:8123/?query=SELECT%201"] + test: ["CMD", "clickhouse-client", "--query", "select 1;"] interval: 30s retries: 3 start_period: 60s