Remove sync blocklist

Makefile

@@ -125,7 +125,6 @@ run-docker:
 	    -e "NEW_RELIC_LICENSE_KEY=$(NEW_RELIC_LICENSE_KEY)" \
 	    -e "NEW_RELIC_ENVIRONMENT=dev" \
 	    -e "NEW_RELIC_APP_NAME=checkmate (dev)" \
-	    -e "CHECKMATE_BLOCKLIST_URL=https://hypothesis-via.s3-us-west-1.amazonaws.com/via-blocklist.txt" \
 	    -p 9099:9099 \
 	    --name checkmate hypothesis/checkmate:$(DOCKER_TAG)
 

README.md

@@ -127,7 +127,6 @@ Environment variables:
 
 | Name | Effect | Example |
 |------|--------|---------|
-| `CHECKMATE_BLOCKLIST_URL`   | Where to download the blocklist online | `https://some-aws-s3.bucket/file.txt` |
 | `CHECKMATE_SECRET` | Secret used for signing URLs | `AB823F97FF2E330C1A20`
 | `PUBLIC_SCHEME` | Scheme used on the public accessible checkmate instance | `https`
 | `PUBLIC_HOST` | Host of the public accessible checkmate instance | `some-domain.com`

checkmate/app.py

@@ -62,9 +62,7 @@ def add_api_keys_from_env(self, param_name):
         return keys
 
     def _configure_checkmate(self, config):
-        if self.celery_worker:
-            self.add_setting_from_env("checkmate_blocklist_url")
-        else:
+        if not self.celery_worker:
             # The celery workers don't need to know about this stuff
 
             config.include("pyramid_services")

checkmate/celery_async/tasks.py

@@ -1,39 +1,14 @@
 """Async tasks."""
 
 from celery.utils.log import get_task_logger
-from requests import RequestException
 
 from checkmate.celery_async.celery import app
-from checkmate.checker.url import CustomRules, URLHaus
+from checkmate.checker.url import URLHaus
 from checkmate.exceptions import StageRetryableException
 
 LOG = get_task_logger(__name__)
 
 
-@app.task
-def sync_blocklist():
-    """Download the online version of the blocklist."""
-
-    # pylint: disable=no-member
-    # PyLint doesn't know about the `request_context` method that we add
-    with app.request_context() as request:
-        url = request.registry.settings["checkmate_blocklist_url"]
-        if not url:
-            LOG.warning("Not updating blocklist as no URL is present")
-            return
-
-        LOG.info("Updating blocklist from '%s'", url)
-
-        with request.tm:
-            try:
-                raw_rules = CustomRules(request.db).load_simple_rule_url(url)
-            except RequestException as err:
-                LOG.exception("Could not update blocklist: %s", err)
-                return
-
-        LOG.info("Updated %s custom rules", len(raw_rules))
-
-
 def pipeline_task(wrapped_function):
     """Mark a function as a standard checker pipeline task.
 

tests/unit/checkmate/app_test.py

@@ -7,7 +7,6 @@
 
 CELERY_SETTINGS = {
     "database_url": "celery_db",
-    "checkmate_blocklist_url": "some_url",
 }
 
 REQUIRED_APP_SETTINGS = {

tests/unit/checkmate/celery_async/tasks_test.py

@@ -1,44 +1,9 @@
 from contextlib import contextmanager
-from unittest.mock import sentinel
 
 import pytest
-from requests import RequestException
 
 from checkmate.app import CheckmateConfigurator
-from checkmate.celery_async.tasks import (
-    initialize_urlhaus,
-    sync_blocklist,
-    sync_urlhaus,
-)
-
-
-@pytest.mark.usefixtures("CustomRules")
-class TestSyncBlocklist:
-    def test_it_works_with_url(self, CustomRules, pyramid_request):
-        pyramid_request.registry.settings["checkmate_blocklist_url"] = sentinel.url
-
-        sync_blocklist()
-
-        CustomRules.assert_called_once_with(pyramid_request.db)
-        CustomRules.return_value.load_simple_rule_url.assert_called_once_with(
-            sentinel.url
-        )
-
-    def test_it_does_nothing_without_url(self, CustomRules, pyramid_request):
-        pyramid_request.registry.settings["checkmate_blocklist_url"] = None
-
-        sync_blocklist()
-
-        CustomRules.assert_not_called()
-
-    def test_it_handles_request_exceptions(self, CustomRules, pyramid_request):
-        pyramid_request.registry.settings["checkmate_blocklist_url"] = sentinel.url
-
-        CustomRules.return_value.load_simple_rule_url.side_effect = RequestException
-
-        sync_blocklist()
-
-        # Nothing really happens, we just carry on
+from checkmate.celery_async.tasks import initialize_urlhaus, sync_urlhaus
 
 
 @pytest.mark.usefixtures("URLHaus")
@@ -61,18 +26,11 @@ def test_it(self, pyramid_request, URLHaus):
 
 @pytest.fixture
 def pyramid_config(pyramid_config):
-    pyramid_config.registry.settings["checkmate_blocklist_url"] = sentinel.default
-
     CheckmateConfigurator(pyramid_config, celery_worker=True)
 
     return pyramid_config
 
 
-@pytest.fixture()
-def CustomRules(patch):
-    return patch("checkmate.celery_async.tasks.CustomRules")
-
-
 @pytest.fixture()
 def URLHaus(patch):
     return patch("checkmate.celery_async.tasks.URLHaus")

tox.ini

@@ -23,7 +23,6 @@ setenv =
     dev: SENTRY_ENVIRONMENT = {env:SENTRY_ENVIRONMENT:dev}
     dev: CELERY_BROKER_URL={env:CELERY_BROKER_URL:amqp://guest:guest@localhost:5673//}
     dev: DATABASE_URL={env:DATABASE_URL:postgresql://postgres@localhost:5434/postgres}
-    dev: CHECKMATE_BLOCKLIST_URL = {env:CHECKMATE_BLOCKLIST_URL:http://dummy.example.com/blocklist}
     dev: PUBLIC_SCHEME = {env:PUBLIC_SCHEME:http}
     dev: PUBLIC_HOST = {env:PUBLIC_HOST:localhost}
     dev: PUBLIC_PORT = {env:PUBLIC_PORT:9099}
@@ -39,7 +38,6 @@ passenv =
     dev: CHROME_EXTENSION_ID
     dev: SENTRY_DSN
     dev: NEW_RELIC_LICENSE_KEY
-    dev: CHECKMATE_BLOCKLIST_URL
     dev: GOOGLE_CLIENT_ID
     dev: GOOGLE_CLIENT_SECRET
 deps =