chore: run regression tests with epoll (#4426)

* chore: run regression tests with epoll

Signed-off-by: kostas <[email protected]>

.github/actions/regression-tests/action.yml

@@ -28,6 +28,9 @@ inputs:
   s3-bucket:
     required: true
     type: string
+  epoll:
+    required: false
+    type: string
 
 runs:
   using: "composite"
@@ -46,7 +49,15 @@ runs:
         export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/${{inputs.build-folder-name}}/${{inputs.dfly-executable}}"
         export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 # to crash on errors
 
-        timeout 50m pytest -m "${{inputs.filter}}" --durations=10 --timeout=300 --color=yes --json-report --json-report-file=report.json dragonfly --log-cli-level=INFO || code=$?
+        if [[ "${{inputs.epoll}}" == 'true' ]]; then
+          export FILTER="${{inputs.filter}} and not exclude_epoll"
+          # Run only replication tests with epoll
+          timeout 50m pytest -m "$FILTER" --durations=10 --timeout=300 --color=yes --json-report --json-report-file=report.json dragonfly --df force_epoll=true --log-cli-level=INFO || code=$?
+        else
+          export FILTER="${{inputs.filter}}"
+          # Run only replication tests with epoll
+          timeout 50m pytest -m "$FILTER" --durations=10 --timeout=300 --color=yes --json-report --json-report-file=report.json dragonfly --log-cli-level=INFO || code=$?
+        fi
 
         # timeout returns 124 if we exceeded the timeout duration
         if [[ $code -eq 124 ]]; then

.github/workflows/regression-tests.yml

@@ -11,8 +11,13 @@ jobs:
       matrix:
         # Test of these containers
         container: ["ubuntu-dev:20"]
+        proactor: [Uring, Epoll]
         build-type: [Debug, Release]
         runner: [ubuntu-latest, [self-hosted, linux, ARM64]]
+        exclude:
+          - proactor: Epoll
+            build-type: Debug
+
     runs-on: ${{ matrix.runner }}
 
     container:
@@ -31,6 +36,7 @@ jobs:
           cat /proc/cpuinfo
           ulimit -a
           env
+
       - name: Configure & Build
         run: |
           # -no-pie to disable address randomization so we could symbolize stacktraces
@@ -52,6 +58,9 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY }}
           aws-secret-access-key: ${{ secrets.AWS_S3_ACCESS_SECRET }}
           s3-bucket: ${{ secrets.S3_REGTEST_BUCKET }}
+          # Chain ternary oprator of the form (which can be nested)
+          # (expression == condition && <true expression> || <false expression>)
+          epoll: ${{ matrix.proactor == 'Epoll' && 'true' || '' }}
 
       - name: Upload logs on failure
         if: failure()

tests/dragonfly/cluster_test.py

@@ -1458,6 +1458,7 @@ async def test_migration_with_key_ttl(df_factory):
     assert await nodes[1].client.execute_command("stick k_sticky") == 0
 
 
+@pytest.mark.exclude_epoll
 @dfly_args({"proactor_threads": 4, "cluster_mode": "yes", "migration_finalization_timeout_ms": 5})
 async def test_network_disconnect_during_migration(df_factory):
     instances = [
@@ -1948,6 +1949,7 @@ async def start_save():
     assert await seeder.compare(capture_before_migration, nodes[1].instance.port)
 
 
+@pytest.mark.exclude_epoll
 @dfly_args({"proactor_threads": 4, "cluster_mode": "yes"})
 @pytest.mark.asyncio
 async def test_cluster_migration_cancel(df_factory: DflyInstanceFactory):
@@ -2013,6 +2015,7 @@ async def node1size0():
 @dfly_args({"proactor_threads": 2, "cluster_mode": "yes"})
 @pytest.mark.asyncio
 @pytest.mark.opt_only
+@pytest.mark.exclude_epoll
 async def test_cluster_migration_huge_container(df_factory: DflyInstanceFactory):
     instances = [
         df_factory.create(port=next(next_port), admin_port=next(next_port)) for i in range(2)
@@ -2562,6 +2565,7 @@ async def test_cluster_memory_consumption_migration(df_factory: DflyInstanceFact
     await check_for_no_state_status([node.admin_client for node in nodes])
 
 
+@pytest.mark.exclude_epoll
 @pytest.mark.asyncio
 @dfly_args({"proactor_threads": 4, "cluster_mode": "yes"})
 async def test_migration_timeout_on_sync(df_factory: DflyInstanceFactory, df_seeder_factory):

tests/dragonfly/connection_test.py

@@ -528,6 +528,7 @@ async def test_keyspace_events_config_set(async_client: aioredis.Redis):
         pass
 
 
+@pytest.mark.exclude_epoll
 async def test_reply_count(async_client: aioredis.Redis):
     """Make sure reply aggregations reduce reply counts for common cases"""
 

tests/dragonfly/replication_test.py

@@ -28,6 +28,7 @@
 M_OPT = [pytest.mark.opt_only]
 M_SLOW = [pytest.mark.slow]
 M_STRESS = [pytest.mark.slow, pytest.mark.opt_only]
+M_NOT_EPOLL = [pytest.mark.exclude_epoll]
 
 
 async def wait_for_replicas_state(*clients, state="online", node_role="slave", timeout=0.05):
@@ -1490,6 +1491,7 @@ async def test_tls_replication(
     await proxy.close(proxy_task)
 
 
+@pytest.mark.exclude_epoll
 async def test_ipv6_replication(df_factory: DflyInstanceFactory):
     """Test that IPV6 addresses work for replication, ::1 is 127.0.0.1 localhost"""
     master = df_factory.create(proactor_threads=1, bind="::1", port=1111)
@@ -2664,6 +2666,7 @@ async def test_replication_timeout_on_full_sync_heartbeat_expiry(
     await assert_replica_reconnections(replica, 0)
 
 
+@pytest.mark.exclude_epoll
 @pytest.mark.parametrize(
     "element_size, elements_number",
     [(16, 30000), (30000, 16)],

tests/dragonfly/snapshot_test.py

@@ -482,6 +482,7 @@ async def test_bgsave_and_save(async_client: aioredis.Redis):
     await async_client.execute_command("SAVE")
 
 
+@pytest.mark.exclude_epoll
 @dfly_args(
     {
         **BASIC_ARGS,

tests/dragonfly/tiering_test.py

@@ -45,6 +45,7 @@ async def test_basic_memory_usage(async_client: aioredis.Redis):
     )  # the grown table itself takes up lots of space
 
 
+@pytest.mark.exclude_epoll
 @pytest.mark.opt_only
 @dfly_args(
     {

tests/pytest.ini

@@ -9,3 +9,4 @@ addopts = -ra --emoji
 markers =
   slow: marks tests as slow (deselect with '-m "not slow"')
   opt_only: marks tests that are only reasonable to run against an opt-built Dragonfly
+  exclude_epoll: marks tests that should not run on epoll socket