Split config, add QUIC pcaps to benchmarks, doc bits

ci-based/config-tests.yml

@@ -0,0 +1,106 @@
+---
+ZEEK_TESTS:
+  - id: pcap-ixia-ent-data-center-2-30sec-500mbps
+    pcap_file: ixia_RamEntDataCenter2_30sec_500Mbps.pcap
+
+  - id: pcap-zeek-testing-ipv6
+    pcap_file: zeek-testing-ipv6.trace
+
+  - id: pcap-zeek-testing-2009-m57-day11-18
+    pcap_file: 2009-M57-day11-18.trace
+
+  - id: pcap-50k-tcp-conns
+    pcap_file: 50k-tcp-conns.pcap
+
+  - id: pcap-much-alexa-https-top-100
+    pcap_file: much-alexa-https-top-100.pcap
+
+  - id: pcap-much-alexa-dns-top-100
+    pcap_file: much-alexa-dns-top-100.pcap
+
+  - id: pcap-http-many-smaller
+    pcap_file: http_many_smaller.pcap
+
+  - id: pcap-500k-syns
+    pcap_file: 500k-syns.pcap
+
+  - id: micro-misc-zeek-version
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/misc/zeek-version.zeek
+
+  # Same as above, but not bare and load test-all-policy, too.
+  - id: micro-misc-zeek-version-all-policy
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D test-all-policy microbenchmarks/misc/zeek-version.zeek
+
+  - id: micro-vector-ops-simple-value
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/vector-ops/simple-value.zeek
+
+  - id: micro-vector-ops-complex-value
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/vector-ops/complex-value.zeek
+
+  - id: micro-table-ops-simple-key-value
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/table-ops/simple-key-value.zeek
+
+  - id: micro-table-ops-complex-key-value
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/table-ops/complex-key-value.zeek
+
+  - id: micro-table-ops-complex-key-value-2
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/table-ops/complex-key-value-2.zeek
+
+  - id: micro-table-ops-copy
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/table-ops/copy.zeek
+
+  - id: micro-table-ops-small-table-simple-key
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/table-ops/small-table-simple-key.zeek
+
+  - id: micro-table-ops-small-table-complex-key
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/table-ops/small-table-complex-key.zeek
+
+  - id: micro-logging-tsv-one-stream
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/logging/one-stream.zeek
+
+  - id: micro-logging-tsv-two-streams
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/logging/two-streams.zeek
+
+  - id: micro-logging-json-one-stream
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/logging/one-stream.zeek  LogAscii::use_json=T
+
+  - id: micro-logging-json-two-streams
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/logging/two-streams.zeek LogAscii::use_json=T
+
+  - id: micro-logging-writer-none-one-stream
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/logging/one-stream.zeek Log::default_writer=Log::WRITER_NONE
+
+  - id: micro-logging-writer-none-two-streams
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/logging/two-streams.zeek Log::default_writer=Log::WRITER_NONE
+
+  - id: micro-function-calls-bifs
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/function-calls/bifs.zeek
+
+  - id: micro-function-calls-recursion
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/function-calls/recursion.zeek
+
+  - id: micro-function-calls-many
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/function-calls/many.zeek
+
+  - id: micro-patterns-basic
+    bench_command: /benchmarker/scripts/tiny-benchmark.sh
+    bench_args: -D -b microbenchmarks/patterns/basic.zeek

ci-based/config.yml

@@ -33,116 +33,13 @@ CPU_SET:
 
 # Default number of times each test runs. Can be overridden by providing
 # a runs key within each test.
-RUN_COUNT: 5
+RUN_COUNT: 1
 
 # Max time for unpacking the build archives.
 TAR_TIMEOUT: 20
 
-ZEEK_TESTS:
-  - id: pcap-ixia-ent-data-center-2-30sec-500mbps
-    pcap_file: ixia_RamEntDataCenter2_30sec_500Mbps.pcap
-
-  - id: pcap-zeek-testing-ipv6
-    pcap_file: zeek-testing-ipv6.trace
-
-  - id: pcap-zeek-testing-2009-m57-day11-18
-    pcap_file: 2009-M57-day11-18.trace
-
-  - id: pcap-50k-tcp-conns
-    pcap_file: 50k-tcp-conns.pcap
-
-  - id: pcap-much-alexa-https-top-100
-    pcap_file: much-alexa-https-top-100.pcap
-
-  - id: pcap-much-alexa-dns-top-100
-    pcap_file: much-alexa-dns-top-100.pcap
-
-  - id: pcap-http-many-smaller
-    pcap_file: http_many_smaller.pcap
-
-  - id: pcap-500k-syns
-    pcap_file: 500k-syns.pcap
-
-  - id: micro-misc-zeek-version
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/misc/zeek-version.zeek
-
-  # Same as above, but not bare and load test-all-policy, too.
-  - id: micro-misc-zeek-version-all-policy
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D test-all-policy microbenchmarks/misc/zeek-version.zeek
-
-  - id: micro-vector-ops-simple-value
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/vector-ops/simple-value.zeek
-
-  - id: micro-vector-ops-complex-value
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/vector-ops/complex-value.zeek
-
-  - id: micro-table-ops-simple-key-value
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/table-ops/simple-key-value.zeek
-
-  - id: micro-table-ops-complex-key-value
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/table-ops/complex-key-value.zeek
-
-  - id: micro-table-ops-complex-key-value-2
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/table-ops/complex-key-value-2.zeek
-
-  - id: micro-table-ops-copy
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/table-ops/copy.zeek
-
-  - id: micro-table-ops-small-table-simple-key
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/table-ops/small-table-simple-key.zeek
-
-  - id: micro-table-ops-small-table-complex-key
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/table-ops/small-table-complex-key.zeek
-
-  - id: micro-logging-tsv-one-stream
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/logging/one-stream.zeek
-
-  - id: micro-logging-tsv-two-streams
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/logging/two-streams.zeek
-
-  - id: micro-logging-json-one-stream
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/logging/one-stream.zeek  LogAscii::use_json=T
-
-  - id: micro-logging-json-two-streams
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/logging/two-streams.zeek LogAscii::use_json=T
-
-  - id: micro-logging-writer-none-one-stream
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/logging/one-stream.zeek Log::default_writer=Log::WRITER_NONE
-
-  - id: micro-logging-writer-none-two-streams
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/logging/two-streams.zeek Log::default_writer=Log::WRITER_NONE
-
-  - id: micro-function-calls-bifs
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/function-calls/bifs.zeek
-
-  - id: micro-function-calls-recursion
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/function-calls/recursion.zeek
-
-  - id: micro-function-calls-many
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/function-calls/many.zeek
-
-  - id: micro-patterns-basic
-    bench_command: /benchmarker/scripts/tiny-benchmark.sh
-    bench_args: -D -b microbenchmarks/patterns/basic.zeek
+# Path to a YAML file describing which tests to run.
+TESTS_FILE: ./config-tests.yml
 
 rq:
   logging:

ci-based/docker-compose.yml

@@ -58,6 +58,7 @@ services:
       dockerfile: containers/rq.Dockerfile
     volumes:
       - ./config.yml:/app/config.yml
+      - ./config-tests.yml:/app/config-tests.yml
       - /var/run/docker.sock:/var/run/docker.sock
       - app_spool_data:/app/spool
 

ci-based/tests/test_tasks.py

@@ -143,3 +143,30 @@ def test__runc__tmpfs(self):
         self.assertEqual("/run", run_kwargs["environment"]["RUN_PATH"])
         self.assertEqual("", run_kwargs["tmpfs"]["/mnt/data/tmpfs"])
         self.assertEqual("", run_kwargs["tmpfs"]["/run"])
+
+
+class TestZeekJob(unittest.TestCase):
+    def setUp(self):
+        self.job = zeek_benchmarker.tasks.ZeekJob(
+            build_url="test-url",
+            build_hash="test-hash",
+            original_branch="test-original-branch",
+            normalized_branch="test-normalized-branch",
+            commit="test-commit",
+            job_id="test-job-id",
+        )
+
+    @mock.patch("zeek_benchmarker.tasks.ZeekJob.run_zeek_test")
+    def test_process(self, run_zeek_test_mock):
+        # This uses the Config.get() singleton call to get access
+        # to the
+        # config.yml and confi-tests.yml stored in this repo.
+        # We use that for basic smoke testing that reading the
+        # config worked.
+        self.job._process()
+
+        tests = [c.args[0] for c in run_zeek_test_mock.call_args_list]
+        ids = [t.test_id for t in tests]
+        self.assertGreater(len(ids), 10)
+        self.assertIn("micro-table-ops-copy", ids)
+        self.assertIn("pcap-500k-syns", ids)

ci-based/zeek_benchmarker/config.py

@@ -11,10 +11,11 @@ class SMTPSettings(typing.NamedTuple):
 
 
 class Config:
-    _config: "Config" = None
+    _config: typing.Optional["Config"] = None
 
     def __init__(self, d: dict[str, typing.Any]):
         self._d = d
+        self._tests_d: dict[str, typing.Any] | None = None
 
     @property
     def work_dir(self) -> str:
@@ -32,6 +33,14 @@ def zeek_cpus(self) -> str:
     def run_count(self) -> int:
         return self._d["RUN_COUNT"]
 
+    @property
+    def zeek_tests(self) -> list[dict[str, typing.Any]]:
+        if self._tests_d is None:
+            with open(self["TESTS_FILE"]) as fp:
+                self._tests_d = yaml.safe_load(fp)
+
+        return self._tests_d["ZEEK_TESTS"]
+
     def __getitem__(self, k: str, default: typing.Any = None):
         """
         Allow dictionary key lookups.
@@ -51,7 +60,7 @@ def smtp_settings(self) -> SMTPSettings:
         )
 
 
-def get():
+def get() -> Config:
     """
     Lazily load the config.
     """

ci-based/zeek_benchmarker/tasks.py

@@ -496,7 +496,7 @@ def run_zeek_test(self, t):
 
     def _process(self):
         cfg = config.get()
-        for t in cfg["ZEEK_TESTS"]:
+        for t in cfg.zeek_tests:
             zeek_test = ZeekTest.from_dict(cfg, t)
             self.run_zeek_test(zeek_test)