From 5052b55c440f3a778f9db9c854fa385077dd7ab9 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 19 Dec 2023 01:18:54 -0800 Subject: [PATCH] test_models: fuzz test panda and CarState (#30443) * pre-hypothesis * some hypothesis junk * this kinda works but is really slow due to counter check * choose addrs from fingerprint * stash * honda nidec brake pressed mismatches fixed * bump panda * stash * tesla: use DI_torque2 (panda msg) * run * run * ah this honda mismatch too * no more multi can msgs * clean up, remove old file * add todo * prob can remove urandom * stash, huge examples * fix pq standstill mismatch * yuge * yup there's a leak somewhere * try to find leak * skip dashcam (pq and tesla) * PR comments * bump * draft stash * fix alt brake hondas * bump * bump * bump * some clean up * minor clean up * more clean up * stash * fix honda bug * more * 100 examples * revert tesla * no memory leak any more? * bring back tests with skips * parameterize max_examples * skip interceptor * is jenkins on my branch? * ooh that's fast * 50 is not bad for GH CI * 300 might be better with rest of test_models * no more detection * bump * need CS_prev to catch bugs where openpilot changes and panda doesn't (eg. not setting interceptor safety mode) * need to simplify all this * need a warm up first, since some signals are 1 by default (toyota's gas_released!=1) * changes * set honda safety param * set toyota safety param * bump panda * clean up honda * rm interceptor * thought interleaving addrs might help, but we can fine tune later * Revert "thought interleaving addrs might help, but we can fine tune later" This reverts commit 153301384b48c9f33f9e2af3c224241eaeec41c1. * get size from dict * what * add nocapture marker * clean up * try to raise logging level * need to run last as pytest_runtest_call, since it starts capturing * get capman conditionally * mark * type fingerprint * should use gen_empty_fingerprint * no longer needed * draft * no longer need gc * clean that up * test everything! * more clean up * more * no point * fix that * fix errors * bump * nice even 300 examples for 300 segs * final bump :fingers_crossed: * better import order * remove debugging prints * warm up kinda works * Revert "warm up kinda works" This reverts commit 7fc77b07d592edb13eadca77deb49540954a7d69. * random seed * revert * strat strat * add expl comment * cmt * check controls allowed * Revert "check controls allowed" This reverts commit e82a0e5396810dd4670e6847aa555194a709e10f. * not unittests * run tests! * run tests 2! * run tests 3! * seed unused * revert * add shrink phase, and remove health check suppression * hello * oncemore * Update selfdrive/car/tests/test_models.py --- Jenkinsfile | 2 +- conftest.py | 18 +++++-- selfdrive/car/tests/test_models.py | 79 ++++++++++++++++++++++++++++-- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a7f272cd807de4..9868777dfbccc5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -268,7 +268,7 @@ node { 'car tests': { pcStage("car tests") { sh label: "build", script: "selfdrive/manager/build.py" - sh label: "run car tests", script: "cd selfdrive/car/tests && MAX_EXAMPLES=100 INTERNAL_SEG_CNT=250 FILEREADER_CACHE=1 \ + sh label: "run car tests", script: "cd selfdrive/car/tests && MAX_EXAMPLES=300 INTERNAL_SEG_CNT=300 FILEREADER_CACHE=1 \ INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt pytest test_models.py test_car_interfaces.py" } }, diff --git a/conftest.py b/conftest.py index 3c566e36728117..6792bd0c3d38f3 100644 --- a/conftest.py +++ b/conftest.py @@ -12,6 +12,17 @@ def pytest_sessionstart(session): session.config.option.randomly_reorganize = False +@pytest.hookimpl(hookwrapper=True, trylast=True) +def pytest_runtest_call(item): + # ensure we run as a hook after capturemanager's + if item.get_closest_marker("nocapture") is not None: + capmanager = item.config.pluginmanager.getplugin("capturemanager") + with capmanager.global_and_fixture_disabled(): + yield + else: + yield + + @pytest.fixture(scope="function", autouse=True) def openpilot_function_fixture(): starting_env = dict(os.environ) @@ -58,7 +69,8 @@ def pytest_collection_modifyitems(config, items): @pytest.hookimpl(trylast=True) def pytest_configure(config): - config_line = ( - "xdist_group_class_property: group tests by a property of the class that contains them" - ) + config_line = "xdist_group_class_property: group tests by a property of the class that contains them" + config.addinivalue_line("markers", config_line) + + config_line = "nocapture: don't capture test output" config.addinivalue_line("markers", config_line) diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 2103b6ccced8b2..e9c2a4ecd51ae4 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -6,10 +6,12 @@ import random import unittest from collections import defaultdict, Counter +import hypothesis.strategies as st +from hypothesis import Phase, given, settings from typing import List, Optional, Tuple from parameterized import parameterized_class -from cereal import log, car +from cereal import messaging, log, car from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.realtime import DT_CTRL @@ -33,6 +35,7 @@ JOB_ID = int(os.environ.get("JOB_ID", "0")) INTERNAL_SEG_LIST = os.environ.get("INTERNAL_SEG_LIST", "") INTERNAL_SEG_CNT = int(os.environ.get("INTERNAL_SEG_CNT", "0")) +MAX_EXAMPLES = int(os.environ.get("MAX_EXAMPLES", "50")) def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]: @@ -67,6 +70,7 @@ class TestCarModelBase(unittest.TestCase): ci: bool = True can_msgs: List[capnp.lib.capnp._DynamicStructReader] + fingerprint: dict[int, dict[int, int]] elm_frame: Optional[int] car_safety_mode_frame: Optional[int] @@ -105,7 +109,7 @@ def setUpClass(cls): can_msgs = [] cls.elm_frame = None cls.car_safety_mode_frame = None - fingerprint = gen_empty_fingerprint() + cls.fingerprint = gen_empty_fingerprint() experimental_long = False for msg in lr: if msg.which() == "can": @@ -113,7 +117,7 @@ def setUpClass(cls): if len(can_msgs) <= FRAME_FINGERPRINT: for m in msg.can: if m.src < 64: - fingerprint[m.src][m.address] = len(m.dat) + cls.fingerprint[m.src][m.address] = len(m.dat) elif msg.which() == "carParams": car_fw = msg.carParams.carFw @@ -149,7 +153,7 @@ def setUpClass(cls): cls.can_msgs = sorted(can_msgs, key=lambda msg: msg.logMonoTime) cls.CarInterface, cls.CarController, cls.CarState = interfaces[cls.car_model] - cls.CP = cls.CarInterface.get_params(cls.car_model, fingerprint, car_fw, experimental_long, docs=False) + cls.CP = cls.CarInterface.get_params(cls.car_model, cls.fingerprint, car_fw, experimental_long, docs=False) assert cls.CP assert cls.CP.carFingerprint == cls.car_model @@ -297,6 +301,73 @@ def test_car_controller(car_control): CC = car.CarControl.new_message(cruiseControl={'resume': True}) test_car_controller(CC) + # Skip stdout/stderr capture with pytest, causes elevated memory usage + @pytest.mark.nocapture + @settings(max_examples=MAX_EXAMPLES, deadline=None, + phases=(Phase.reuse, Phase.generate, Phase.shrink)) + @given(data=st.data()) + def test_panda_safety_carstate_fuzzy(self, data): + """ + For each example, pick a random CAN message on the bus and fuzz its data, + checking for panda state mismatches. + """ + + if self.CP.dashcamOnly: + self.skipTest("no need to check panda safety for dashcamOnly") + + valid_addrs = [(addr, bus, size) for bus, addrs in self.fingerprint.items() for addr, size in addrs.items()] + address, bus, size = data.draw(st.sampled_from(valid_addrs)) + + msg_strategy = st.binary(min_size=size, max_size=size) + msgs = data.draw(st.lists(msg_strategy, min_size=20)) + + CC = car.CarControl.new_message() + + for dat in msgs: + # due to panda updating state selectively, only edges are expected to match + # TODO: warm up CarState with real CAN messages to check edge of both sources + # (eg. toyota's gasPressed is the inverse of a signal being set) + prev_panda_gas = self.safety.get_gas_pressed_prev() + prev_panda_brake = self.safety.get_brake_pressed_prev() + prev_panda_regen_braking = self.safety.get_regen_braking_prev() + prev_panda_vehicle_moving = self.safety.get_vehicle_moving() + prev_panda_cruise_engaged = self.safety.get_cruise_engaged_prev() + prev_panda_acc_main_on = self.safety.get_acc_main_on() + + to_send = libpanda_py.make_CANPacket(address, bus, dat) + self.safety.safety_rx_hook(to_send) + + can = messaging.new_message('can', 1) + can.can = [log.CanData(address=address, dat=dat, src=bus)] + + CS = self.CI.update(CC, (can.to_bytes(),)) + + if self.safety.get_gas_pressed_prev() != prev_panda_gas: + self.assertEqual(CS.gasPressed, self.safety.get_gas_pressed_prev()) + + if self.safety.get_brake_pressed_prev() != prev_panda_brake: + # TODO: remove this exception once this mismatch is resolved + brake_pressed = CS.brakePressed + if CS.brakePressed and not self.safety.get_brake_pressed_prev(): + if self.CP.carFingerprint in (HONDA.PILOT, HONDA.RIDGELINE) and CS.brake > 0.05: + brake_pressed = False + + self.assertEqual(brake_pressed, self.safety.get_brake_pressed_prev()) + + if self.safety.get_regen_braking_prev() != prev_panda_regen_braking: + self.assertEqual(CS.regenBraking, self.safety.get_regen_braking_prev()) + + if self.safety.get_vehicle_moving() != prev_panda_vehicle_moving: + self.assertEqual(not CS.standstill, self.safety.get_vehicle_moving()) + + if not (self.CP.carName == "honda" and self.CP.carFingerprint not in HONDA_BOSCH): + if self.safety.get_cruise_engaged_prev() != prev_panda_cruise_engaged: + self.assertEqual(CS.cruiseState.enabled, self.safety.get_cruise_engaged_prev()) + + if self.CP.carName == "honda": + if self.safety.get_acc_main_on() != prev_panda_acc_main_on: + self.assertEqual(CS.cruiseState.available, self.safety.get_acc_main_on()) + def test_panda_safety_carstate(self): """ Assert that panda safety matches openpilot's carState