Skip to content

Commit

Permalink
Add compatibility for legacy bloom filter file naming
Browse files Browse the repository at this point in the history
  • Loading branch information
KevinMind committed Nov 25, 2024
1 parent 29949e2 commit 8c7f026
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 5 deletions.
3 changes: 2 additions & 1 deletion src/olympia/blocklist/cron.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def _upload_mlbf_to_remote_settings(*, force_base=False):
or mlbf.should_upload_filter(block_type, base_filter)
):
base_filters_to_update.append(block_type)
# Only update the stash if we should AND if we aren't already
# Only update the stash if we should AND if we aren't already
# re-uploading the filter for this block type.
elif mlbf.should_upload_stash(block_type, previous_filter or base_filter):
create_stash = True
Expand All @@ -118,6 +118,7 @@ def _upload_mlbf_to_remote_settings(*, force_base=False):

# Until we are ready to enable soft blocking, it should not be possible
# to create a stash and a filter at the same iteration
# This should never happen, but we'll raise an exception just in case.
if create_stash and len(base_filters_to_update) > 0:
raise Exception(
'Cannot upload stash and filter without implementing soft blocking'
Expand Down
17 changes: 13 additions & 4 deletions src/olympia/blocklist/mlbf.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,9 +209,10 @@ def hash_filter_inputs(cls, input_list):
for (guid, version) in input_list
]

def filter_path(self, block_type: BlockType):
# TODO: explain / test
if block_type == BlockType.BLOCKED:
def filter_path(self, block_type: BlockType, compat: bool = False):
# Override the return value of the BLOCKED filter
# to for backwards compatibility with the old file name
if block_type == BlockType.BLOCKED and compat:
return self.storage.path('filter')
return self.storage.path(f'filter-{BaseMLBFLoader.data_type_key(block_type)}')

Expand All @@ -233,7 +234,15 @@ def generate_and_write_filter(self, block_type: BlockType = BlockType.BLOCKED):
not_blocked=self.data.not_blocked_items,
)

# write bloomfilter
# write bloomfilter to old and new file names
mlbf_path = self.filter_path(block_type, compat=True)
with self.storage.open(mlbf_path, 'wb') as filter_file:
log.info(f'Writing to file {mlbf_path}')
bloomfilter.tofile(filter_file)
stats['mlbf_filesize'] = os.stat(mlbf_path).st_size

# also write to the new file name. After the switch is complete,
# this file will be used and the old file will be deleted.
mlbf_path = self.filter_path(block_type)
with self.storage.open(mlbf_path, 'wb') as filter_file:
log.info(f'Writing to file {mlbf_path}')
Expand Down
12 changes: 12 additions & 0 deletions src/olympia/blocklist/tests/test_mlbf.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,18 @@ def test_hash_filter_inputs_returns_set_of_hashed_strings(self):


class TestMLBF(_MLBFBase):
def test_filter_path(self):
mlbf = MLBF.generate_from_db('test')
assert mlbf.filter_path(BlockType.BLOCKED, compat=True).endswith('filter')
assert mlbf.filter_path(BlockType.BLOCKED).endswith('filter-blocked')
assert mlbf.filter_path(BlockType.SOFT_BLOCKED).endswith('filter-soft_blocked')

def test_save_filter_writes_to_both_file_names(self):
mlbf = MLBF.generate_from_db('test')
mlbf.generate_and_write_filter(BlockType.BLOCKED)
assert mlbf.storage.exists('filter')
assert mlbf.storage.exists('filter-blocked')

def test_get_data_from_db(self):
self._blocked_addon()
mlbf = MLBF.generate_from_db('test')
Expand Down

0 comments on commit 8c7f026

Please sign in to comment.