diff --git a/ultravox/model/ultravox_model.py b/ultravox/model/ultravox_model.py
index 1b0a575..ccea3e0 100644
--- a/ultravox/model/ultravox_model.py
+++ b/ultravox/model/ultravox_model.py
@@ -291,7 +291,7 @@ def _create_audio_tower(
                     config.audio_latency_block_size, dtype=config.torch_dtype
                 )
             else:
-                assert config.audio_latency_block_size not in (
+                assert config.audio_latency_block_size in (
                     None,
                     0,
                 ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
@@ -305,7 +305,7 @@ def _create_audio_tower(
                     config.audio_latency_block_size, dtype=config.torch_dtype
                 )
             else:
-                assert config.audio_latency_block_size not in (
+                assert config.audio_latency_block_size in (
                     None,
                     0,
                 ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"