diff --git a/Jenkinsfile b/Jenkinsfile index a7f272cd807de4..9868777dfbccc5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -268,7 +268,7 @@ node { 'car tests': { pcStage("car tests") { sh label: "build", script: "selfdrive/manager/build.py" - sh label: "run car tests", script: "cd selfdrive/car/tests && MAX_EXAMPLES=100 INTERNAL_SEG_CNT=250 FILEREADER_CACHE=1 \ + sh label: "run car tests", script: "cd selfdrive/car/tests && MAX_EXAMPLES=300 INTERNAL_SEG_CNT=300 FILEREADER_CACHE=1 \ INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt pytest test_models.py test_car_interfaces.py" } }, diff --git a/conftest.py b/conftest.py index 3c566e36728117..6792bd0c3d38f3 100644 --- a/conftest.py +++ b/conftest.py @@ -12,6 +12,17 @@ def pytest_sessionstart(session): session.config.option.randomly_reorganize = False +@pytest.hookimpl(hookwrapper=True, trylast=True) +def pytest_runtest_call(item): + # ensure we run as a hook after capturemanager's + if item.get_closest_marker("nocapture") is not None: + capmanager = item.config.pluginmanager.getplugin("capturemanager") + with capmanager.global_and_fixture_disabled(): + yield + else: + yield + + @pytest.fixture(scope="function", autouse=True) def openpilot_function_fixture(): starting_env = dict(os.environ) @@ -58,7 +69,8 @@ def pytest_collection_modifyitems(config, items): @pytest.hookimpl(trylast=True) def pytest_configure(config): - config_line = ( - "xdist_group_class_property: group tests by a property of the class that contains them" - ) + config_line = "xdist_group_class_property: group tests by a property of the class that contains them" + config.addinivalue_line("markers", config_line) + + config_line = "nocapture: don't capture test output" config.addinivalue_line("markers", config_line) diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 2103b6ccced8b2..e9c2a4ecd51ae4 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -6,10 +6,12 @@ import random import unittest from collections import defaultdict, Counter +import hypothesis.strategies as st +from hypothesis import Phase, given, settings from typing import List, Optional, Tuple from parameterized import parameterized_class -from cereal import log, car +from cereal import messaging, log, car from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.realtime import DT_CTRL @@ -33,6 +35,7 @@ JOB_ID = int(os.environ.get("JOB_ID", "0")) INTERNAL_SEG_LIST = os.environ.get("INTERNAL_SEG_LIST", "") INTERNAL_SEG_CNT = int(os.environ.get("INTERNAL_SEG_CNT", "0")) +MAX_EXAMPLES = int(os.environ.get("MAX_EXAMPLES", "50")) def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]: @@ -67,6 +70,7 @@ class TestCarModelBase(unittest.TestCase): ci: bool = True can_msgs: List[capnp.lib.capnp._DynamicStructReader] + fingerprint: dict[int, dict[int, int]] elm_frame: Optional[int] car_safety_mode_frame: Optional[int] @@ -105,7 +109,7 @@ def setUpClass(cls): can_msgs = [] cls.elm_frame = None cls.car_safety_mode_frame = None - fingerprint = gen_empty_fingerprint() + cls.fingerprint = gen_empty_fingerprint() experimental_long = False for msg in lr: if msg.which() == "can": @@ -113,7 +117,7 @@ def setUpClass(cls): if len(can_msgs) <= FRAME_FINGERPRINT: for m in msg.can: if m.src < 64: - fingerprint[m.src][m.address] = len(m.dat) + cls.fingerprint[m.src][m.address] = len(m.dat) elif msg.which() == "carParams": car_fw = msg.carParams.carFw @@ -149,7 +153,7 @@ def setUpClass(cls): cls.can_msgs = sorted(can_msgs, key=lambda msg: msg.logMonoTime) cls.CarInterface, cls.CarController, cls.CarState = interfaces[cls.car_model] - cls.CP = cls.CarInterface.get_params(cls.car_model, fingerprint, car_fw, experimental_long, docs=False) + cls.CP = cls.CarInterface.get_params(cls.car_model, cls.fingerprint, car_fw, experimental_long, docs=False) assert cls.CP assert cls.CP.carFingerprint == cls.car_model @@ -297,6 +301,73 @@ def test_car_controller(car_control): CC = car.CarControl.new_message(cruiseControl={'resume': True}) test_car_controller(CC) + # Skip stdout/stderr capture with pytest, causes elevated memory usage + @pytest.mark.nocapture + @settings(max_examples=MAX_EXAMPLES, deadline=None, + phases=(Phase.reuse, Phase.generate, Phase.shrink)) + @given(data=st.data()) + def test_panda_safety_carstate_fuzzy(self, data): + """ + For each example, pick a random CAN message on the bus and fuzz its data, + checking for panda state mismatches. + """ + + if self.CP.dashcamOnly: + self.skipTest("no need to check panda safety for dashcamOnly") + + valid_addrs = [(addr, bus, size) for bus, addrs in self.fingerprint.items() for addr, size in addrs.items()] + address, bus, size = data.draw(st.sampled_from(valid_addrs)) + + msg_strategy = st.binary(min_size=size, max_size=size) + msgs = data.draw(st.lists(msg_strategy, min_size=20)) + + CC = car.CarControl.new_message() + + for dat in msgs: + # due to panda updating state selectively, only edges are expected to match + # TODO: warm up CarState with real CAN messages to check edge of both sources + # (eg. toyota's gasPressed is the inverse of a signal being set) + prev_panda_gas = self.safety.get_gas_pressed_prev() + prev_panda_brake = self.safety.get_brake_pressed_prev() + prev_panda_regen_braking = self.safety.get_regen_braking_prev() + prev_panda_vehicle_moving = self.safety.get_vehicle_moving() + prev_panda_cruise_engaged = self.safety.get_cruise_engaged_prev() + prev_panda_acc_main_on = self.safety.get_acc_main_on() + + to_send = libpanda_py.make_CANPacket(address, bus, dat) + self.safety.safety_rx_hook(to_send) + + can = messaging.new_message('can', 1) + can.can = [log.CanData(address=address, dat=dat, src=bus)] + + CS = self.CI.update(CC, (can.to_bytes(),)) + + if self.safety.get_gas_pressed_prev() != prev_panda_gas: + self.assertEqual(CS.gasPressed, self.safety.get_gas_pressed_prev()) + + if self.safety.get_brake_pressed_prev() != prev_panda_brake: + # TODO: remove this exception once this mismatch is resolved + brake_pressed = CS.brakePressed + if CS.brakePressed and not self.safety.get_brake_pressed_prev(): + if self.CP.carFingerprint in (HONDA.PILOT, HONDA.RIDGELINE) and CS.brake > 0.05: + brake_pressed = False + + self.assertEqual(brake_pressed, self.safety.get_brake_pressed_prev()) + + if self.safety.get_regen_braking_prev() != prev_panda_regen_braking: + self.assertEqual(CS.regenBraking, self.safety.get_regen_braking_prev()) + + if self.safety.get_vehicle_moving() != prev_panda_vehicle_moving: + self.assertEqual(not CS.standstill, self.safety.get_vehicle_moving()) + + if not (self.CP.carName == "honda" and self.CP.carFingerprint not in HONDA_BOSCH): + if self.safety.get_cruise_engaged_prev() != prev_panda_cruise_engaged: + self.assertEqual(CS.cruiseState.enabled, self.safety.get_cruise_engaged_prev()) + + if self.CP.carName == "honda": + if self.safety.get_acc_main_on() != prev_panda_acc_main_on: + self.assertEqual(CS.cruiseState.available, self.safety.get_acc_main_on()) + def test_panda_safety_carstate(self): """ Assert that panda safety matches openpilot's carState