From 273d039d97b4407d42f703190af6d626990ab681 Mon Sep 17 00:00:00 2001 From: Misha Tomilov Date: Fri, 3 Jan 2025 15:08:16 +0100 Subject: [PATCH] Remove sync blocklist --- Makefile | 1 - README.md | 1 - checkmate/app.py | 4 +- checkmate/celery_async/tasks.py | 27 +----------- tests/unit/checkmate/app_test.py | 1 - .../unit/checkmate/celery_async/tasks_test.py | 44 +------------------ tox.ini | 2 - 7 files changed, 3 insertions(+), 77 deletions(-) diff --git a/Makefile b/Makefile index 6651a0ab..0eeafea0 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,6 @@ run-docker: -e "NEW_RELIC_LICENSE_KEY=$(NEW_RELIC_LICENSE_KEY)" \ -e "NEW_RELIC_ENVIRONMENT=dev" \ -e "NEW_RELIC_APP_NAME=checkmate (dev)" \ - -e "CHECKMATE_BLOCKLIST_URL=https://hypothesis-via.s3-us-west-1.amazonaws.com/via-blocklist.txt" \ -p 9099:9099 \ --name checkmate hypothesis/checkmate:$(DOCKER_TAG) diff --git a/README.md b/README.md index d6677220..6320b1b5 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,6 @@ Environment variables: | Name | Effect | Example | |------|--------|---------| -| `CHECKMATE_BLOCKLIST_URL` | Where to download the blocklist online | `https://some-aws-s3.bucket/file.txt` | | `CHECKMATE_SECRET` | Secret used for signing URLs | `AB823F97FF2E330C1A20` | `PUBLIC_SCHEME` | Scheme used on the public accessible checkmate instance | `https` | `PUBLIC_HOST` | Host of the public accessible checkmate instance | `some-domain.com` diff --git a/checkmate/app.py b/checkmate/app.py index 9b5a4b66..0ca0446e 100644 --- a/checkmate/app.py +++ b/checkmate/app.py @@ -62,9 +62,7 @@ def add_api_keys_from_env(self, param_name): return keys def _configure_checkmate(self, config): - if self.celery_worker: - self.add_setting_from_env("checkmate_blocklist_url") - else: + if not self.celery_worker: # The celery workers don't need to know about this stuff config.include("pyramid_services") diff --git a/checkmate/celery_async/tasks.py b/checkmate/celery_async/tasks.py index 28ef9e67..d75f61ff 100644 --- a/checkmate/celery_async/tasks.py +++ b/checkmate/celery_async/tasks.py @@ -1,39 +1,14 @@ """Async tasks.""" from celery.utils.log import get_task_logger -from requests import RequestException from checkmate.celery_async.celery import app -from checkmate.checker.url import CustomRules, URLHaus +from checkmate.checker.url import URLHaus from checkmate.exceptions import StageRetryableException LOG = get_task_logger(__name__) -@app.task -def sync_blocklist(): - """Download the online version of the blocklist.""" - - # pylint: disable=no-member - # PyLint doesn't know about the `request_context` method that we add - with app.request_context() as request: - url = request.registry.settings["checkmate_blocklist_url"] - if not url: - LOG.warning("Not updating blocklist as no URL is present") - return - - LOG.info("Updating blocklist from '%s'", url) - - with request.tm: - try: - raw_rules = CustomRules(request.db).load_simple_rule_url(url) - except RequestException as err: - LOG.exception("Could not update blocklist: %s", err) - return - - LOG.info("Updated %s custom rules", len(raw_rules)) - - def pipeline_task(wrapped_function): """Mark a function as a standard checker pipeline task. diff --git a/tests/unit/checkmate/app_test.py b/tests/unit/checkmate/app_test.py index 901ba3a3..fe25c412 100644 --- a/tests/unit/checkmate/app_test.py +++ b/tests/unit/checkmate/app_test.py @@ -7,7 +7,6 @@ CELERY_SETTINGS = { "database_url": "celery_db", - "checkmate_blocklist_url": "some_url", } REQUIRED_APP_SETTINGS = { diff --git a/tests/unit/checkmate/celery_async/tasks_test.py b/tests/unit/checkmate/celery_async/tasks_test.py index 2ad23aa4..d6bc55c4 100644 --- a/tests/unit/checkmate/celery_async/tasks_test.py +++ b/tests/unit/checkmate/celery_async/tasks_test.py @@ -1,44 +1,9 @@ from contextlib import contextmanager -from unittest.mock import sentinel import pytest -from requests import RequestException from checkmate.app import CheckmateConfigurator -from checkmate.celery_async.tasks import ( - initialize_urlhaus, - sync_blocklist, - sync_urlhaus, -) - - -@pytest.mark.usefixtures("CustomRules") -class TestSyncBlocklist: - def test_it_works_with_url(self, CustomRules, pyramid_request): - pyramid_request.registry.settings["checkmate_blocklist_url"] = sentinel.url - - sync_blocklist() - - CustomRules.assert_called_once_with(pyramid_request.db) - CustomRules.return_value.load_simple_rule_url.assert_called_once_with( - sentinel.url - ) - - def test_it_does_nothing_without_url(self, CustomRules, pyramid_request): - pyramid_request.registry.settings["checkmate_blocklist_url"] = None - - sync_blocklist() - - CustomRules.assert_not_called() - - def test_it_handles_request_exceptions(self, CustomRules, pyramid_request): - pyramid_request.registry.settings["checkmate_blocklist_url"] = sentinel.url - - CustomRules.return_value.load_simple_rule_url.side_effect = RequestException - - sync_blocklist() - - # Nothing really happens, we just carry on +from checkmate.celery_async.tasks import initialize_urlhaus, sync_urlhaus @pytest.mark.usefixtures("URLHaus") @@ -61,18 +26,11 @@ def test_it(self, pyramid_request, URLHaus): @pytest.fixture def pyramid_config(pyramid_config): - pyramid_config.registry.settings["checkmate_blocklist_url"] = sentinel.default - CheckmateConfigurator(pyramid_config, celery_worker=True) return pyramid_config -@pytest.fixture() -def CustomRules(patch): - return patch("checkmate.celery_async.tasks.CustomRules") - - @pytest.fixture() def URLHaus(patch): return patch("checkmate.celery_async.tasks.URLHaus") diff --git a/tox.ini b/tox.ini index e0a3ab7a..51823c57 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,6 @@ setenv = dev: SENTRY_ENVIRONMENT = {env:SENTRY_ENVIRONMENT:dev} dev: CELERY_BROKER_URL={env:CELERY_BROKER_URL:amqp://guest:guest@localhost:5673//} dev: DATABASE_URL={env:DATABASE_URL:postgresql://postgres@localhost:5434/postgres} - dev: CHECKMATE_BLOCKLIST_URL = {env:CHECKMATE_BLOCKLIST_URL:http://dummy.example.com/blocklist} dev: PUBLIC_SCHEME = {env:PUBLIC_SCHEME:http} dev: PUBLIC_HOST = {env:PUBLIC_HOST:localhost} dev: PUBLIC_PORT = {env:PUBLIC_PORT:9099} @@ -39,7 +38,6 @@ passenv = dev: CHROME_EXTENSION_ID dev: SENTRY_DSN dev: NEW_RELIC_LICENSE_KEY - dev: CHECKMATE_BLOCKLIST_URL dev: GOOGLE_CLIENT_ID dev: GOOGLE_CLIENT_SECRET deps =