Skip to content

Commit

Permalink
Apply suggestions from code review
Browse files Browse the repository at this point in the history
Co-authored-by: William Durand <[email protected]>
  • Loading branch information
KevinMind and willdurand committed Nov 25, 2024
1 parent 096c8e5 commit 2488f60
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 23 deletions.
20 changes: 8 additions & 12 deletions src/olympia/blocklist/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ def upload_filter(generation_time, filter_list=None, create_stash=False):
bucket, REMOTE_SETTINGS_COLLECTION_MLBF, sign_off_needed=False
)
mlbf = MLBF.load_from_storage(generation_time, error_on_missing=True)
# Download old records before uploading new ones
# this ensures we do not delete any records we just uplaoded
# Download old records before uploading new ones.
# This ensures we do not delete any records we just uploaded.
old_records = server.records()
attachment_types_to_delete = []

Expand Down Expand Up @@ -150,7 +150,7 @@ def upload_filter(generation_time, filter_list=None, create_stash=False):
# so we can delete stashes older than this new filter
base_filter_ids[block_type] = generation_time

# It is possible to upload a stash and a filter in the same task
# It is possible to upload a stash and a filter in the same task.
if create_stash:
with mlbf.storage.open(mlbf.stash_path, 'r') as stash_file:
stash_data = json.load(stash_file)
Expand All @@ -165,9 +165,7 @@ def upload_filter(generation_time, filter_list=None, create_stash=False):

# Get the oldest base filter id so we can delete only stashes
# that are definitely not needed anymore
oldest_base_filter_id = (
min(base_filter_ids.values()) if base_filter_ids else None
)
oldest_base_filter_id = min(base_filter_ids.values()) if base_filter_ids else None

for record in old_records:
# Delete attachment records that match the
Expand All @@ -176,7 +174,6 @@ def upload_filter(generation_time, filter_list=None, create_stash=False):
# per block_type
if 'attachment' in record:
attachment_type = record['attachment_type']

if attachment_type in attachment_types_to_delete:
server.delete_record(record['id'])

Expand All @@ -185,18 +182,17 @@ def upload_filter(generation_time, filter_list=None, create_stash=False):
# cannot apply to any existing filter since we uploaded
elif 'stash' in record and oldest_base_filter_id is not None:
record_time = record['stash_time']

if record_time < oldest_base_filter_id:
server.delete_record(record['id'])

# Commit the changes to remote settings for review.
# only after any changes to records (attachments and stashes)
# and including deletions can we commit the session
# and update the config with the new timestamps
# Only after any changes to records (attachments and stashes)
# and including deletions can we commit the session and update
# the config with the new timestamps.
server.complete_session()
set_config(MLBF_TIME_CONFIG_KEY, generation_time, json_value=True)

# Update the base_filter_id for uploaded filters
# Update the base_filter_id for uploaded filters.
for block_type in filters_to_upload:
# We currently write to the old singular config key for hard blocks
# to preserve backward compatibility.
Expand Down
22 changes: 11 additions & 11 deletions src/olympia/blocklist/tests/test_tasks.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import json
import os
from datetime import datetime, timedelta
from typing import Dict, List
from unittest import TestCase, mock

from django.conf import settings
Expand Down Expand Up @@ -267,7 +266,9 @@ def test_skip_cleanup_when_no_filters_or_config_keys(self):

upload_filter(self.generation_time, filter_list=[])

assert get_config(MLBF_BASE_ID_CONFIG_KEY(BlockType.BLOCKED, compat=True)) == None
assert (
get_config(MLBF_BASE_ID_CONFIG_KEY(BlockType.BLOCKED, compat=True)) is None
)
assert not self.mocks['delete_record'].called

set_config(
Expand Down Expand Up @@ -335,10 +336,12 @@ def test_cleanup_records_older_than_oldest_base_filter_id(self):
json_value=True,
)
upload_filter(self.generation_time, filter_list=[])
assert self.mocks['delete_record'].call_args_list == [mock.call(0), mock.call(1)]
assert self.mocks['delete_record'].call_args_list == [
mock.call(0),
mock.call(1),
]
self.mocks['delete_record'].reset_mock()


# Delete all records older than the new active filter
# which is t3
upload_filter(t3, filter_list=[BlockType.BLOCKED.name])
Expand Down Expand Up @@ -384,7 +387,8 @@ def test_cleanup_old_stash_records_when_uploading_new_filter(self):
mlbf = MLBF.generate_from_db(t2)
mlbf.generate_and_write_filter(BlockType.BLOCKED)

# Remote settings returns the old filter and a stash created before the new filter
# Remote settings returns the old filter
# and a stash created before the new filter
# this stash should also be deleted
self.mocks['records'].return_value = [
self._attachment(0, 'bloomfilter-base', t0),
Expand Down Expand Up @@ -419,9 +423,7 @@ def test_ignore_soft_blocked_if_switch_is_disabled(self):
self._stash(1, t0),
]

upload_filter(
self.generation_time, filter_list=[BlockType.SOFT_BLOCKED.name]
)
upload_filter(self.generation_time, filter_list=[BlockType.SOFT_BLOCKED.name])

assert not self.mocks['delete_record'].called

Expand Down Expand Up @@ -469,9 +471,7 @@ def test_create_stashed_filter(self):

def test_raises_when_no_filter_exists(self):
with self.assertRaises(FileNotFoundError):
upload_filter(
self.generation_time, filter_list=[BlockType.BLOCKED.name]
)
upload_filter(self.generation_time, filter_list=[BlockType.BLOCKED.name])

def test_raises_when_no_stash_exists(self):
with self.assertRaises(FileNotFoundError):
Expand Down

0 comments on commit 2488f60

Please sign in to comment.