From e8857a6713db6b8b3b8cd67aca1b8c046a4dccd9 Mon Sep 17 00:00:00 2001 From: Vivek Aithal Date: Thu, 6 Jun 2024 01:14:28 -0700 Subject: [PATCH 01/12] [Lag] Increase Optima Lag (#32583) increase lag in optima to 0.2 --- selfdrive/car/hyundai/interface.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/selfdrive/car/hyundai/interface.py b/selfdrive/car/hyundai/interface.py index 00452a9ae02d08..22c54bce6b5179 100644 --- a/selfdrive/car/hyundai/interface.py +++ b/selfdrive/car/hyundai/interface.py @@ -75,6 +75,9 @@ def _get_params(ret, candidate, fingerprint, car_fw, experimental_long, docs): ret.steerLimitTimer = 0.4 CarInterfaceBase.configure_torque_tune(candidate, ret.lateralTuning) + if candidate == CAR.KIA_OPTIMA_G4_FL: + ret.steerActuatorDelay = 0.2 + # *** longitudinal control *** if candidate in CANFD_CAR: ret.longitudinalTuning.kpV = [0.1] From 4c549778c0b0bd4430f2ec18023c40d3a6252a1a Mon Sep 17 00:00:00 2001 From: Hoang Bui <47828508+bongbui321@users.noreply.github.com> Date: Thu, 6 Jun 2024 15:59:57 -0400 Subject: [PATCH 02/12] CI/tools_test: remove redundant `git lfs pull` (#32642) remove redundant --- .github/workflows/tools_tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index b3cfd56d13499d..10c5b709103e2b 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -47,7 +47,6 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - run: git lfs pull - uses: ./.github/workflows/setup-with-retry - name: Build openpilot run: | From e71ec4e8d7e39a56235f590227e7cb361cd8b79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Thu, 6 Jun 2024 14:31:56 -0700 Subject: [PATCH 03/12] Split cereal into cereal/msgq (#32631) * squash * fix doc * compile device * compile device * Update ref * add msgq to precommit exclusions * No service ports * fix compile * address comments * More comments * Delete logger * Update opendbc * Linting * bump msgq --- .gitignore | 2 + .gitmodules | 4 +- .pre-commit-config.yaml | 12 +- Dockerfile.openpilot | 1 + SConstruct | 3 + cereal | 1 - cereal/.pre-commit-config.yaml | 47 + cereal/Dockerfile | 54 + cereal/LICENSE | 7 + cereal/README.md | 60 + cereal/SConscript | 31 + cereal/__init__.py | 9 + cereal/car.capnp | 706 +++++ cereal/codecov.yml | 8 + cereal/custom.capnp | 39 + cereal/generate_javascript.sh | 26 + cereal/include/c++.capnp | 26 + cereal/legacy.capnp | 574 ++++ cereal/log.capnp | 2369 +++++++++++++++++ cereal/maptile.capnp | 49 + cereal/messaging/.gitignore | 10 + cereal/messaging/__init__.py | 306 +++ cereal/messaging/bridge.cc | 92 + cereal/messaging/demo.cc | 50 + cereal/messaging/demo.py | 29 + cereal/messaging/messaging.h | 112 + cereal/messaging/msgq.md | 54 + cereal/messaging/msgq_tests.cc | 394 +++ cereal/messaging/socketmaster.cc | 211 ++ cereal/messaging/stress.py | 14 + cereal/messaging/test_runner.cc | 2 + cereal/messaging/tests/__init__.py | 0 cereal/messaging/tests/test_fake.py | 193 ++ cereal/messaging/tests/test_messaging.py | 247 ++ cereal/messaging/tests/test_poller.py | 142 + cereal/messaging/tests/test_pub_sub_master.py | 163 ++ cereal/messaging/tests/test_services.py | 25 + cereal/services.py | 118 + docs/c_docs.rst | 4 +- msgq | 1 + opendbc | 2 +- selfdrive/controls/controlsd.py | 2 +- selfdrive/locationd/SConscript | 4 +- selfdrive/modeld/dmonitoringmodeld.py | 2 +- selfdrive/modeld/modeld.py | 2 +- selfdrive/modeld/models/commonmodel.pxd | 2 +- selfdrive/modeld/models/commonmodel_pyx.pxd | 4 +- selfdrive/modeld/models/commonmodel_pyx.pyx | 4 +- selfdrive/modeld/runners/snpemodel.pxd | 2 +- selfdrive/modeld/runners/thneedmodel.pxd | 2 +- selfdrive/modeld/tests/test_modeld.py | 2 +- selfdrive/navd/SConscript | 4 +- selfdrive/navd/map_renderer.h | 2 +- selfdrive/navd/tests/test_map_renderer.py | 2 +- selfdrive/pandad/SConscript | 4 +- .../test/process_replay/process_replay.py | 2 +- selfdrive/test/process_replay/vision_meta.py | 2 +- selfdrive/ui/SConscript | 4 +- selfdrive/ui/qt/widgets/cameraview.h | 2 +- selfdrive/ui/tests/test_ui/run.py | 2 +- system/camerad/SConscript | 4 +- system/camerad/cameras/camera_common.h | 2 +- system/camerad/snapshot/snapshot.py | 2 +- system/hardware/base.h | 2 +- system/logcatd/SConscript | 4 +- system/loggerd/SConscript | 4 +- system/loggerd/encoder/encoder.h | 2 +- system/loggerd/loggerd.cc | 2 +- system/loggerd/loggerd.h | 2 +- system/loggerd/tests/test_loggerd.py | 2 +- system/proclogd/SConscript | 4 +- system/sensord/SConscript | 4 +- system/ubloxd/SConscript | 4 +- tools/cabana/SConscript | 4 +- tools/camerastream/compressed_vipc.py | 2 +- tools/replay/SConscript | 4 +- tools/replay/camera.h | 2 +- tools/replay/framereader.h | 2 +- tools/replay/ui.py | 2 +- tools/sim/Dockerfile.sim | 1 + tools/sim/lib/camerad.py | 2 +- tools/webcam/camerad.py | 2 +- 82 files changed, 6240 insertions(+), 66 deletions(-) delete mode 160000 cereal create mode 100644 cereal/.pre-commit-config.yaml create mode 100644 cereal/Dockerfile create mode 100644 cereal/LICENSE create mode 100644 cereal/README.md create mode 100644 cereal/SConscript create mode 100644 cereal/__init__.py create mode 100644 cereal/car.capnp create mode 100644 cereal/codecov.yml create mode 100644 cereal/custom.capnp create mode 100755 cereal/generate_javascript.sh create mode 100644 cereal/include/c++.capnp create mode 100644 cereal/legacy.capnp create mode 100644 cereal/log.capnp create mode 100644 cereal/maptile.capnp create mode 100644 cereal/messaging/.gitignore create mode 100644 cereal/messaging/__init__.py create mode 100644 cereal/messaging/bridge.cc create mode 100644 cereal/messaging/demo.cc create mode 100644 cereal/messaging/demo.py create mode 100644 cereal/messaging/messaging.h create mode 100644 cereal/messaging/msgq.md create mode 100644 cereal/messaging/msgq_tests.cc create mode 100644 cereal/messaging/socketmaster.cc create mode 100644 cereal/messaging/stress.py create mode 100644 cereal/messaging/test_runner.cc create mode 100644 cereal/messaging/tests/__init__.py create mode 100644 cereal/messaging/tests/test_fake.py create mode 100755 cereal/messaging/tests/test_messaging.py create mode 100644 cereal/messaging/tests/test_poller.py create mode 100755 cereal/messaging/tests/test_pub_sub_master.py create mode 100755 cereal/messaging/tests/test_services.py create mode 100755 cereal/services.py create mode 160000 msgq diff --git a/.gitignore b/.gitignore index 4bc5fd3a24555e..370335e6a05af2 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,8 @@ compare_runtime*.html persist selfdrive/pandad/pandad +cereal/services.h +cereal/gen selfdrive/logcatd/logcatd selfdrive/mapd/default_speeds_by_region.json system/proclogd/proclogd diff --git a/.gitmodules b/.gitmodules index 73f832b1d6ce9a..ee65939b6b6eda 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,8 @@ [submodule "opendbc"] path = opendbc url = ../../commaai/opendbc.git -[submodule "cereal"] - path = cereal +[submodule "msgq"] + path = msgq url = ../../commaai/cereal.git [submodule "rednose_repo"] path = rednose_repo diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 68d303cd616c0d..75ddcfb616d6b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: rev: v2.2.6 hooks: - id: codespell - exclude: '^(third_party/)|(body/)|(cereal/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)' + exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)' args: # if you've got a short variable name that's getting flagged, add it here - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints @@ -36,7 +36,7 @@ repos: rev: v0.4.4 hooks: - id: ruff - exclude: '^(third_party/)|(cereal/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' + exclude: '^(third_party/)|(msgq/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' - repo: local hooks: - id: mypy @@ -47,7 +47,7 @@ repos: args: - --local-partial-types - --explicit-package-bases - exclude: '^(third_party/)|(body/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' + exclude: '^(third_party/)|(body/)|(msgq/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' - repo: local hooks: - id: cppcheck @@ -55,7 +55,7 @@ repos: entry: cppcheck language: system types: [c++] - exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)|(installer/)' + exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)|(installer/)' args: - --error-exitcode=1 - --language=c++ @@ -66,7 +66,7 @@ repos: rev: 1.6.1 hooks: - id: cpplint - exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)' + exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)' args: - --quiet - --counting=total @@ -78,7 +78,7 @@ repos: rev: v0.16.2 hooks: - id: cython-lint - exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)' + exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)' args: - --max-line-length=240 - --ignore=E111, E302, E305 diff --git a/Dockerfile.openpilot b/Dockerfile.openpilot index 1e15722bb95bbd..9aa1aa8b74105d 100644 --- a/Dockerfile.openpilot +++ b/Dockerfile.openpilot @@ -20,6 +20,7 @@ COPY ./release ${OPENPILOT_PATH}/release COPY ./common ${OPENPILOT_PATH}/common COPY ./opendbc ${OPENPILOT_PATH}/opendbc COPY ./cereal ${OPENPILOT_PATH}/cereal +COPY ./msgq ${OPENPILOT_PATH}/msgq COPY ./panda ${OPENPILOT_PATH}/panda COPY ./selfdrive ${OPENPILOT_PATH}/selfdrive COPY ./system ${OPENPILOT_PATH}/system diff --git a/SConstruct b/SConstruct index df34d5af992a12..ba49d926aa340e 100644 --- a/SConstruct +++ b/SConstruct @@ -209,6 +209,7 @@ env = Environment( "#third_party/qrcode", "#third_party", "#cereal", + "#msgq", "#opendbc/can", "#third_party/maplibre-native-qt/include", f"#third_party/maplibre-native-qt/{arch}/include" @@ -224,6 +225,7 @@ env = Environment( CXXFLAGS=["-std=c++1z"] + cxxflags, LIBPATH=libpath + [ "#cereal", + "#msgq", "#third_party", "#opendbc/can", "#selfdrive/pandad", @@ -357,6 +359,7 @@ gpucommon = [_gpucommon] Export('common', 'gpucommon') # Build cereal and messaging +SConscript(['msgq/SConscript']) SConscript(['cereal/SConscript']) # Build other submodules diff --git a/cereal b/cereal deleted file mode 160000 index d0ceac712f531c..00000000000000 --- a/cereal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d0ceac712f531c125923cbdc4ec7bedbdc5a7baf diff --git a/cereal/.pre-commit-config.yaml b/cereal/.pre-commit-config.yaml new file mode 100644 index 00000000000000..03360e0f9fa677 --- /dev/null +++ b/cereal/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-ast + - id: check-yaml + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.5 + hooks: + - id: ruff +- repo: local + hooks: + - id: cppcheck + name: cppcheck + entry: cppcheck + language: system + types: [c++] + exclude: '^(messaging/msgq_tests.cc|messaging/test_runner.cc)' + args: + - --error-exitcode=1 + - --inline-suppr + - --language=c++ + - --force + - --quiet + - -j4 +- repo: https://github.com/cpplint/cpplint + rev: 1.6.1 + hooks: + - id: cpplint + args: + - --quiet + - --counting=detailed + - --linelength=240 + - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces +- repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + args: + - -L ned + - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US diff --git a/cereal/Dockerfile b/cereal/Dockerfile new file mode 100644 index 00000000000000..d2e3ab591be74c --- /dev/null +++ b/cereal/Dockerfile @@ -0,0 +1,54 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + autoconf \ + build-essential \ + ca-certificates \ + capnproto \ + clang \ + cppcheck \ + curl \ + git \ + libbz2-dev \ + libcapnp-dev \ + libclang-rt-dev \ + libffi-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libzmq3-dev \ + llvm \ + make \ + cmake \ + ocl-icd-opencl-dev \ + opencl-headers \ + python3-dev \ + python3-pip \ + tk-dev \ + wget \ + xz-utils \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy + +WORKDIR /project/ +RUN cd /tmp/ && \ + git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \ + cd Catch2 && \ + mv single_include/catch2/ /project/ && \ + cd .. \ + rm -rf Catch2 + +WORKDIR /project/cereal + +ENV PYTHONPATH=/project + +COPY . . +RUN rm -rf .git && \ + scons -c && scons -j$(nproc) diff --git a/cereal/LICENSE b/cereal/LICENSE new file mode 100644 index 00000000000000..f1fd199c632b14 --- /dev/null +++ b/cereal/LICENSE @@ -0,0 +1,7 @@ +Copyright (c) 2020, Comma.ai, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cereal/README.md b/cereal/README.md new file mode 100644 index 00000000000000..e3326aab0e98ca --- /dev/null +++ b/cereal/README.md @@ -0,0 +1,60 @@ +# What is cereal? [![cereal tests](https://github.com/commaai/cereal/workflows/tests/badge.svg?event=push)](https://github.com/commaai/cereal/actions) [![codecov](https://codecov.io/gh/commaai/cereal/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/cereal) + +cereal is both a messaging spec for robotics systems as well as generic high performance IPC pub sub messaging with a single publisher and multiple subscribers. + +Imagine this use case: +* A sensor process reads gyro measurements directly from an IMU and publishes a `sensorEvents` packet +* A calibration process subscribes to the `sensorEvents` packet to use the IMU +* A localization process subscribes to the `sensorEvents` packet to use the IMU also + + +## Messaging Spec + +You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`. + +All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type. + +### Best Practices + +- **All fields must describe quantities in SI units**, unless otherwise specified in the field name. +- In the context of the message they are in, field names should be completely unambiguous. +- All values should be easy to plot and be human-readable with minimal parsing. + +### Maintaining backwards-compatibility + +When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can +be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other +things are not. Read more details [here](https://capnproto.org/language.html). + +### Custom forks + +Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging +spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot +then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in +[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your +fork will remain backwards-compatible with all versions of mainline cereal/openpilot and your fork.** + +## Pub Sub Backends + +cereal supports two backends, one based on [zmq](https://zeromq.org/) and another called [msgq](messaging/msgq.cc), a custom pub sub based on shared memory that doesn't require the bytes to pass through the kernel. + +Example +--- +```python +import cereal.messaging as messaging + +# in subscriber +sm = messaging.SubMaster(['sensorEvents']) +while 1: + sm.update() + print(sm['sensorEvents']) + +``` + +```python +# in publisher +pm = messaging.PubMaster(['sensorEvents']) +dat = messaging.new_message('sensorEvents', size=1) +dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}} +pm.send('sensorEvents', dat) +``` diff --git a/cereal/SConscript b/cereal/SConscript new file mode 100644 index 00000000000000..5a71abbb81fb16 --- /dev/null +++ b/cereal/SConscript @@ -0,0 +1,31 @@ +Import('env', 'envCython', 'arch', 'common', 'messaging') + +import shutil + +cereal_dir = Dir('.') +gen_dir = Dir('gen') +other_dir = Dir('#msgq/messaging') + +# Build cereal +schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp'] +env.Command(["gen/c/include/c++.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS") +env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files], + schema_files, + f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/") + +# TODO: remove non shared cereal and messaging +cereal_objects = env.SharedObject([f'gen/cpp/{s}.c++' for s in schema_files]) + +cereal = env.Library('cereal', cereal_objects) +env.SharedLibrary('cereal_shared', cereal_objects) + +# Build messaging + +services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET') +env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging, 'zmq', common]) + + +socketmaster = env.SharedObject(['messaging/socketmaster.cc']) +socketmaster = env.Library('socketmaster', socketmaster) + +Export('cereal', 'socketmaster') diff --git a/cereal/__init__.py b/cereal/__init__.py new file mode 100644 index 00000000000000..89c5cf38e390e7 --- /dev/null +++ b/cereal/__init__.py @@ -0,0 +1,9 @@ +import os +import capnp + +CEREAL_PATH = os.path.dirname(os.path.abspath(__file__)) +capnp.remove_import_hook() + +log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp")) +car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp")) +custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp")) diff --git a/cereal/car.capnp b/cereal/car.capnp new file mode 100644 index 00000000000000..d7ec1b2484a390 --- /dev/null +++ b/cereal/car.capnp @@ -0,0 +1,706 @@ +using Cxx = import "./include/c++.capnp"; +$Cxx.namespace("cereal"); + +@0x8e2af1e708af8b8d; + +# ******* events causing controls state machine transition ******* + +struct CarEvent @0x9b1657f34caf3ad3 { + name @0 :EventName; + + # event types + enable @1 :Bool; + noEntry @2 :Bool; + warning @3 :Bool; # alerts presented only when enabled or soft disabling + userDisable @4 :Bool; + softDisable @5 :Bool; + immediateDisable @6 :Bool; + preEnable @7 :Bool; + permanent @8 :Bool; # alerts presented regardless of openpilot state + overrideLateral @10 :Bool; + overrideLongitudinal @9 :Bool; + + enum EventName @0xbaa8c5d505f727de { + canError @0; + steerUnavailable @1; + wrongGear @4; + doorOpen @5; + seatbeltNotLatched @6; + espDisabled @7; + wrongCarMode @8; + steerTempUnavailable @9; + reverseGear @10; + buttonCancel @11; + buttonEnable @12; + pedalPressed @13; # exits active state + preEnableStandstill @73; # added during pre-enable state with brake + gasPressedOverride @108; # added when user is pressing gas with no disengage on gas + steerOverride @114; + cruiseDisabled @14; + speedTooLow @17; + outOfSpace @18; + overheat @19; + calibrationIncomplete @20; + calibrationInvalid @21; + calibrationRecalibrating @117; + controlsMismatch @22; + pcmEnable @23; + pcmDisable @24; + radarFault @26; + brakeHold @28; + parkBrake @29; + manualRestart @30; + lowSpeedLockout @31; + joystickDebug @34; + steerTempUnavailableSilent @35; + resumeRequired @36; + preDriverDistracted @37; + promptDriverDistracted @38; + driverDistracted @39; + preDriverUnresponsive @43; + promptDriverUnresponsive @44; + driverUnresponsive @45; + belowSteerSpeed @46; + lowBattery @48; + accFaulted @51; + sensorDataInvalid @52; + commIssue @53; + commIssueAvgFreq @109; + tooDistracted @54; + posenetInvalid @55; + soundsUnavailable @56; + preLaneChangeLeft @57; + preLaneChangeRight @58; + laneChange @59; + lowMemory @63; + stockAeb @64; + ldw @65; + carUnrecognized @66; + invalidLkasSetting @69; + speedTooHigh @70; + laneChangeBlocked @71; + relayMalfunction @72; + stockFcw @74; + startup @75; + startupNoCar @76; + startupNoControl @77; + startupMaster @78; + startupNoFw @104; + fcw @79; + steerSaturated @80; + belowEngageSpeed @84; + noGps @85; + wrongCruiseMode @87; + modeldLagging @89; + deviceFalling @90; + fanMalfunction @91; + cameraMalfunction @92; + cameraFrameRate @110; + processNotRunning @95; + dashcamMode @96; + controlsInitializing @98; + usbError @99; + roadCameraError @100; + driverCameraError @101; + wideRoadCameraError @102; + highCpuUsage @105; + cruiseMismatch @106; + lkasDisabled @107; + canBusMissing @111; + controlsdLagging @112; + resumeBlocked @113; + steerTimeLimit @115; + vehicleSensorsInvalid @116; + locationdTemporaryError @103; + locationdPermanentError @118; + paramsdTemporaryError @50; + paramsdPermanentError @119; + actuatorsApiUnavailable @120; + + radarCanErrorDEPRECATED @15; + communityFeatureDisallowedDEPRECATED @62; + radarCommIssueDEPRECATED @67; + driverMonitorLowAccDEPRECATED @68; + gasUnavailableDEPRECATED @3; + dataNeededDEPRECATED @16; + modelCommIssueDEPRECATED @27; + ipasOverrideDEPRECATED @33; + geofenceDEPRECATED @40; + driverMonitorOnDEPRECATED @41; + driverMonitorOffDEPRECATED @42; + calibrationProgressDEPRECATED @47; + invalidGiraffeHondaDEPRECATED @49; + invalidGiraffeToyotaDEPRECATED @60; + internetConnectivityNeededDEPRECATED @61; + whitePandaUnsupportedDEPRECATED @81; + commIssueWarningDEPRECATED @83; + focusRecoverActiveDEPRECATED @86; + neosUpdateRequiredDEPRECATED @88; + modelLagWarningDEPRECATED @93; + startupOneplusDEPRECATED @82; + startupFuzzyFingerprintDEPRECATED @97; + noTargetDEPRECATED @25; + brakeUnavailableDEPRECATED @2; + plannerErrorDEPRECATED @32; + gpsMalfunctionDEPRECATED @94; + } +} + +# ******* main car state @ 100hz ******* +# all speeds in m/s + +struct CarState { + events @13 :List(CarEvent); + + # CAN health + canValid @26 :Bool; # invalid counter/checksums + canTimeout @40 :Bool; # CAN bus dropped out + canErrorCounter @48 :UInt32; + canRcvTimeout @49 :Bool; + + # car speed + vEgo @1 :Float32; # best estimate of speed + aEgo @16 :Float32; # best estimate of acceleration + vEgoRaw @17 :Float32; # unfiltered speed from CAN sensors + vEgoCluster @44 :Float32; # best estimate of speed shown on car's instrument cluster, used for UI + + yawRate @22 :Float32; # best estimate of yaw rate + standstill @18 :Bool; + wheelSpeeds @2 :WheelSpeeds; + + # gas pedal, 0.0-1.0 + gas @3 :Float32; # this is user pedal only + gasPressed @4 :Bool; # this is user pedal only + + engineRpm @46 :Float32; + + # brake pedal, 0.0-1.0 + brake @5 :Float32; # this is user pedal only + brakePressed @6 :Bool; # this is user pedal only + regenBraking @45 :Bool; # this is user pedal only + parkingBrake @39 :Bool; + brakeHoldActive @38 :Bool; + + # steering wheel + steeringAngleDeg @7 :Float32; + steeringAngleOffsetDeg @37 :Float32; # Offset betweens sensors in case there multiple + steeringRateDeg @15 :Float32; + steeringTorque @8 :Float32; # TODO: standardize units + steeringTorqueEps @27 :Float32; # TODO: standardize units + steeringPressed @9 :Bool; # if the user is using the steering wheel + steerFaultTemporary @35 :Bool; # temporary EPS fault + steerFaultPermanent @36 :Bool; # permanent EPS fault + stockAeb @30 :Bool; + stockFcw @31 :Bool; + espDisabled @32 :Bool; + accFaulted @42 :Bool; + carFaultedNonCritical @47 :Bool; # some ECU is faulted, but car remains controllable + + # cruise state + cruiseState @10 :CruiseState; + + # gear + gearShifter @14 :GearShifter; + + # button presses + buttonEvents @11 :List(ButtonEvent); + leftBlinker @20 :Bool; + rightBlinker @21 :Bool; + genericToggle @23 :Bool; + + # lock info + doorOpen @24 :Bool; + seatbeltUnlatched @25 :Bool; + + # clutch (manual transmission only) + clutchPressed @28 :Bool; + + # blindspot sensors + leftBlindspot @33 :Bool; # Is there something blocking the left lane change + rightBlindspot @34 :Bool; # Is there something blocking the right lane change + + fuelGauge @41 :Float32; # battery or fuel tank level from 0.0 to 1.0 + charging @43 :Bool; + + # process meta + cumLagMs @50 :Float32; + + struct WheelSpeeds { + # optional wheel speeds + fl @0 :Float32; + fr @1 :Float32; + rl @2 :Float32; + rr @3 :Float32; + } + + struct CruiseState { + enabled @0 :Bool; + speed @1 :Float32; + speedCluster @6 :Float32; # Set speed as shown on instrument cluster + available @2 :Bool; + speedOffset @3 :Float32; + standstill @4 :Bool; + nonAdaptive @5 :Bool; + } + + enum GearShifter { + unknown @0; + park @1; + drive @2; + neutral @3; + reverse @4; + sport @5; + low @6; + brake @7; + eco @8; + manumatic @9; + } + + # send on change + struct ButtonEvent { + pressed @0 :Bool; + type @1 :Type; + + enum Type { + unknown @0; + leftBlinker @1; + rightBlinker @2; + accelCruise @3; + decelCruise @4; + cancel @5; + altButton1 @6; + altButton2 @7; + altButton3 @8; + setCruise @9; + resumeCruise @10; + gapAdjustCruise @11; + } + } + + # deprecated + errorsDEPRECATED @0 :List(CarEvent.EventName); + brakeLightsDEPRECATED @19 :Bool; + steeringRateLimitedDEPRECATED @29 :Bool; + canMonoTimesDEPRECATED @12: List(UInt64); +} + +# ******* radar state @ 20hz ******* + +struct RadarData @0x888ad6581cf0aacb { + errors @0 :List(Error); + points @1 :List(RadarPoint); + + enum Error { + canError @0; + fault @1; + wrongConfig @2; + } + + # similar to LiveTracks + # is one timestamp valid for all? I think so + struct RadarPoint { + trackId @0 :UInt64; # no trackId reuse + + # these 3 are the minimum required + dRel @1 :Float32; # m from the front bumper of the car + yRel @2 :Float32; # m + vRel @3 :Float32; # m/s + + # these are optional and valid if they are not NaN + aRel @4 :Float32; # m/s^2 + yvRel @5 :Float32; # m/s + + # some radars flag measurements VS estimates + measured @6 :Bool; + } + + # deprecated + canMonoTimesDEPRECATED @2 :List(UInt64); +} + +# ******* car controls @ 100hz ******* + +struct CarControl { + # must be true for any actuator commands to work + enabled @0 :Bool; + latActive @11: Bool; + longActive @12: Bool; + + # Actuator commands as computed by controlsd + actuators @6 :Actuators; + + # moved to CarOutput + actuatorsOutputDEPRECATED @10 :Actuators; + + leftBlinker @15: Bool; + rightBlinker @16: Bool; + + orientationNED @13 :List(Float32); + angularVelocity @14 :List(Float32); + + cruiseControl @4 :CruiseControl; + hudControl @5 :HUDControl; + + struct Actuators { + # range from 0.0 - 1.0 + gas @0: Float32; + brake @1: Float32; + # range from -1.0 - 1.0 + steer @2: Float32; + # value sent over can to the car + steerOutputCan @8: Float32; + steeringAngleDeg @3: Float32; + + curvature @7: Float32; + + speed @6: Float32; # m/s + accel @4: Float32; # m/s^2 + longControlState @5: LongControlState; + + enum LongControlState @0xe40f3a917d908282{ + off @0; + pid @1; + stopping @2; + starting @3; + } + } + + struct CruiseControl { + cancel @0: Bool; + resume @1: Bool; + override @4: Bool; + speedOverrideDEPRECATED @2: Float32; + accelOverrideDEPRECATED @3: Float32; + } + + struct HUDControl { + speedVisible @0: Bool; + setSpeed @1: Float32; + lanesVisible @2: Bool; + leadVisible @3: Bool; + visualAlert @4: VisualAlert; + audibleAlert @5: AudibleAlert; + rightLaneVisible @6: Bool; + leftLaneVisible @7: Bool; + rightLaneDepart @8: Bool; + leftLaneDepart @9: Bool; + leadDistanceBars @10: Int8; # 1-3: 1 is closest, 3 is farthest. some ports may utilize 2-4 bars instead + + enum VisualAlert { + # these are the choices from the Honda + # map as good as you can for your car + none @0; + fcw @1; + steerRequired @2; + brakePressed @3; + wrongGear @4; + seatbeltUnbuckled @5; + speedTooHigh @6; + ldw @7; + } + + enum AudibleAlert { + none @0; + + engage @1; + disengage @2; + refuse @3; + + warningSoft @4; + warningImmediate @5; + + prompt @6; + promptRepeat @7; + promptDistracted @8; + } + } + + gasDEPRECATED @1 :Float32; + brakeDEPRECATED @2 :Float32; + steeringTorqueDEPRECATED @3 :Float32; + activeDEPRECATED @7 :Bool; + rollDEPRECATED @8 :Float32; + pitchDEPRECATED @9 :Float32; +} + +struct CarOutput { + # Any car specific rate limits or quirks applied by + # the CarController are reflected in actuatorsOutput + # and matches what is sent to the car + actuatorsOutput @0 :CarControl.Actuators; +} + +# ****** car param ****** + +struct CarParams { + carName @0 :Text; + carFingerprint @1 :Text; + fuzzyFingerprint @55 :Bool; + + notCar @66 :Bool; # flag for non-car robotics platforms + + pcmCruise @3 :Bool; # is openpilot's state tied to the PCM's cruise state? + enableDsu @5 :Bool; # driving support unit + enableBsm @56 :Bool; # blind spot monitoring + flags @64 :UInt32; # flags for car specific quirks + experimentalLongitudinalAvailable @71 :Bool; + + minEnableSpeed @7 :Float32; + minSteerSpeed @8 :Float32; + safetyConfigs @62 :List(SafetyConfig); + alternativeExperience @65 :Int16; # panda flag for features like no disengage on gas + + # Car docs fields + maxLateralAccel @68 :Float32; + autoResumeSng @69 :Bool; # describes whether car can resume from a stop automatically + + # things about the car in the manual + mass @17 :Float32; # [kg] curb weight: all fluids no cargo + wheelbase @18 :Float32; # [m] distance from rear axle to front axle + centerToFront @19 :Float32; # [m] distance from center of mass to front axle + steerRatio @20 :Float32; # [] ratio of steering wheel angle to front wheel angle + steerRatioRear @21 :Float32; # [] ratio of steering wheel angle to rear wheel angle (usually 0) + + # things we can derive + rotationalInertia @22 :Float32; # [kg*m2] body rotational inertia + tireStiffnessFactor @72 :Float32; # scaling factor used in calculating tireStiffness[Front,Rear] + tireStiffnessFront @23 :Float32; # [N/rad] front tire coeff of stiff + tireStiffnessRear @24 :Float32; # [N/rad] rear tire coeff of stiff + + longitudinalTuning @25 :LongitudinalPIDTuning; + lateralParams @48 :LateralParams; + lateralTuning :union { + pid @26 :LateralPIDTuning; + indiDEPRECATED @27 :LateralINDITuning; + lqrDEPRECATED @40 :LateralLQRTuning; + torque @67 :LateralTorqueTuning; + } + + steerLimitAlert @28 :Bool; + steerLimitTimer @47 :Float32; # time before steerLimitAlert is issued + + vEgoStopping @29 :Float32; # Speed at which the car goes into stopping state + vEgoStarting @59 :Float32; # Speed at which the car goes into starting state + stoppingControl @31 :Bool; # Does the car allow full control even at lows speeds when stopping + steerControlType @34 :SteerControlType; + radarUnavailable @35 :Bool; # True when radar objects aren't visible on CAN or aren't parsed out + stopAccel @60 :Float32; # Required acceleration to keep vehicle stationary + stoppingDecelRate @52 :Float32; # m/s^2/s while trying to stop + startAccel @32 :Float32; # Required acceleration to get car moving + startingState @70 :Bool; # Does this car make use of special starting state + + steerActuatorDelay @36 :Float32; # Steering wheel actuator delay in seconds + longitudinalActuatorDelayLowerBound @61 :Float32; # Gas/Brake actuator delay in seconds, lower bound + longitudinalActuatorDelayUpperBound @58 :Float32; # Gas/Brake actuator delay in seconds, upper bound + openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control? + carVin @38 :Text; # VIN number queried during fingerprinting + dashcamOnly @41: Bool; + passive @73: Bool; # is openpilot in control? + transmissionType @43 :TransmissionType; + carFw @44 :List(CarFw); + + radarTimeStep @45: Float32 = 0.05; # time delta between radar updates, 20Hz is very standard + fingerprintSource @49: FingerprintSource; + networkLocation @50 :NetworkLocation; # Where Panda/C2 is integrated into the car's CAN network + + wheelSpeedFactor @63 :Float32; # Multiplier on wheels speeds to computer actual speeds + + struct SafetyConfig { + safetyModel @0 :SafetyModel; + safetyParam @3 :UInt16; + safetyParamDEPRECATED @1 :Int16; + safetyParam2DEPRECATED @2 :UInt32; + } + + struct LateralParams { + torqueBP @0 :List(Int32); + torqueV @1 :List(Int32); + } + + struct LateralPIDTuning { + kpBP @0 :List(Float32); + kpV @1 :List(Float32); + kiBP @2 :List(Float32); + kiV @3 :List(Float32); + kf @4 :Float32; + } + + struct LateralTorqueTuning { + useSteeringAngle @0 :Bool; + kp @1 :Float32; + ki @2 :Float32; + friction @3 :Float32; + kf @4 :Float32; + steeringAngleDeadzoneDeg @5 :Float32; + latAccelFactor @6 :Float32; + latAccelOffset @7 :Float32; + } + + struct LongitudinalPIDTuning { + kpBP @0 :List(Float32); + kpV @1 :List(Float32); + kiBP @2 :List(Float32); + kiV @3 :List(Float32); + kf @6 :Float32; + deadzoneBP @4 :List(Float32); + deadzoneV @5 :List(Float32); + } + + struct LateralINDITuning { + outerLoopGainBP @4 :List(Float32); + outerLoopGainV @5 :List(Float32); + innerLoopGainBP @6 :List(Float32); + innerLoopGainV @7 :List(Float32); + timeConstantBP @8 :List(Float32); + timeConstantV @9 :List(Float32); + actuatorEffectivenessBP @10 :List(Float32); + actuatorEffectivenessV @11 :List(Float32); + + outerLoopGainDEPRECATED @0 :Float32; + innerLoopGainDEPRECATED @1 :Float32; + timeConstantDEPRECATED @2 :Float32; + actuatorEffectivenessDEPRECATED @3 :Float32; + } + + struct LateralLQRTuning { + scale @0 :Float32; + ki @1 :Float32; + dcGain @2 :Float32; + + # State space system + a @3 :List(Float32); + b @4 :List(Float32); + c @5 :List(Float32); + + k @6 :List(Float32); # LQR gain + l @7 :List(Float32); # Kalman gain + } + + enum SafetyModel { + silent @0; + hondaNidec @1; + toyota @2; + elm327 @3; + gm @4; + hondaBoschGiraffe @5; + ford @6; + cadillac @7; + hyundai @8; + chrysler @9; + tesla @10; + subaru @11; + gmPassive @12; + mazda @13; + nissan @14; + volkswagen @15; + toyotaIpas @16; + allOutput @17; + gmAscm @18; + noOutput @19; # like silent but without silent CAN TXs + hondaBosch @20; + volkswagenPq @21; + subaruPreglobal @22; # pre-Global platform + hyundaiLegacy @23; + hyundaiCommunity @24; + volkswagenMlb @25; + hongqi @26; + body @27; + hyundaiCanfd @28; + volkswagenMqbEvo @29; + chryslerCusw @30; + psa @31; + } + + enum SteerControlType { + torque @0; + angle @1; + + curvatureDEPRECATED @2; + } + + enum TransmissionType { + unknown @0; + automatic @1; # Traditional auto, including DSG + manual @2; # True "stick shift" only + direct @3; # Electric vehicle or other direct drive + cvt @4; + } + + struct CarFw { + ecu @0 :Ecu; + fwVersion @1 :Data; + address @2 :UInt32; + subAddress @3 :UInt8; + responseAddress @4 :UInt32; + request @5 :List(Data); + brand @6 :Text; + bus @7 :UInt8; + logging @8 :Bool; + obdMultiplexing @9 :Bool; + } + + enum Ecu { + eps @0; + abs @1; + fwdRadar @2; + fwdCamera @3; + engine @4; + unknown @5; + transmission @8; # Transmission Control Module + hybrid @18; # hybrid control unit, e.g. Chrysler's HCP, Honda's IMA Control Unit, Toyota's hybrid control computer + srs @9; # airbag + gateway @10; # can gateway + hud @11; # heads up display + combinationMeter @12; # instrument cluster + electricBrakeBooster @15; + shiftByWire @16; + adas @19; + cornerRadar @21; + hvac @20; + parkingAdas @7; # parking assist system ECU, e.g. Toyota's IPAS, Hyundai's RSPA, etc. + epb @22; # electronic parking brake + telematics @23; + body @24; # body control module + + # Toyota only + dsu @6; + + # Honda only + vsa @13; # Vehicle Stability Assist + programmedFuelInjection @14; + + debug @17; + } + + enum FingerprintSource { + can @0; + fw @1; + fixed @2; + } + + enum NetworkLocation { + fwdCamera @0; # Standard/default integration at LKAS camera + gateway @1; # Integration at vehicle's CAN gateway + } + + enableGasInterceptorDEPRECATED @2 :Bool; + enableCameraDEPRECATED @4 :Bool; + enableApgsDEPRECATED @6 :Bool; + steerRateCostDEPRECATED @33 :Float32; + isPandaBlackDEPRECATED @39 :Bool; + hasStockCameraDEPRECATED @57 :Bool; + safetyParamDEPRECATED @10 :Int16; + safetyModelDEPRECATED @9 :SafetyModel; + safetyModelPassiveDEPRECATED @42 :SafetyModel = silent; + minSpeedCanDEPRECATED @51 :Float32; + communityFeatureDEPRECATED @46: Bool; + startingAccelRateDEPRECATED @53 :Float32; + steerMaxBPDEPRECATED @11 :List(Float32); + steerMaxVDEPRECATED @12 :List(Float32); + gasMaxBPDEPRECATED @13 :List(Float32); + gasMaxVDEPRECATED @14 :List(Float32); + brakeMaxBPDEPRECATED @15 :List(Float32); + brakeMaxVDEPRECATED @16 :List(Float32); + directAccelControlDEPRECATED @30 :Bool; + maxSteeringAngleDegDEPRECATED @54 :Float32; +} diff --git a/cereal/codecov.yml b/cereal/codecov.yml new file mode 100644 index 00000000000000..83427c3ee8b6e5 --- /dev/null +++ b/cereal/codecov.yml @@ -0,0 +1,8 @@ +comment: false +coverage: + status: + project: + default: + informational: true + patch: off + diff --git a/cereal/custom.capnp b/cereal/custom.capnp new file mode 100644 index 00000000000000..369222add8c622 --- /dev/null +++ b/cereal/custom.capnp @@ -0,0 +1,39 @@ +using Cxx = import "./include/c++.capnp"; +$Cxx.namespace("cereal"); + +@0xb526ba661d550a59; + +# custom.capnp: a home for empty structs reserved for custom forks +# These structs are guaranteed to remain reserved and empty in mainline +# cereal, so use these if you want custom events in your fork. + +# you can rename the struct, but don't change the identifier +struct CustomReserved0 @0x81c2f05a394cf4af { +} + +struct CustomReserved1 @0xaedffd8f31e7b55d { +} + +struct CustomReserved2 @0xf35cc4560bbf6ec2 { +} + +struct CustomReserved3 @0xda96579883444c35 { +} + +struct CustomReserved4 @0x80ae746ee2596b11 { +} + +struct CustomReserved5 @0xa5cd762cd951a455 { +} + +struct CustomReserved6 @0xf98d843bfd7004a3 { +} + +struct CustomReserved7 @0xb86e6369214c01c8 { +} + +struct CustomReserved8 @0xf416ec09499d9d19 { +} + +struct CustomReserved9 @0xa1680744031fdb2d { +} diff --git a/cereal/generate_javascript.sh b/cereal/generate_javascript.sh new file mode 100755 index 00000000000000..d6525a64d95eed --- /dev/null +++ b/cereal/generate_javascript.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +rm -r gen/ts +rm -r gen/js + +mkdir gen/ts +mkdir gen/js + +echo "Installing needed npm modules" +npm i capnpc-ts capnp-ts + +capnpc -o node_modules/.bin/capnpc-ts:gen/ts log.capnp car.capnp +capnpc -o node_modules/.bin/capnpc-ts:gen/ts car.capnp + +cat log.capnp | egrep '\([a-zA-Z]*\.[^\s]+\.[^s]+\)' | sed 's/^.*([a-zA-Z]*\.\([a-zA-Z.]*\)).*/\1/' | while read line +do + TOKEN=`echo $line | sed 's/\./_/g'` + ROOT=`echo $line | sed 's/\..*$//g'` + cat gen/ts/log.capnp.ts | grep '^import.*'${TOKEN} + if [[ "$?" == "1" ]] + then + sed -i 's/^\(import {.*\)'${ROOT}'\(,*\) \(.*\)$/\1'${ROOT}', '${TOKEN}'\2 \3/' ./gen/ts/log.capnp.ts + fi +done + +tsc ./gen/ts/* --lib es2015 --outDir ./gen/js diff --git a/cereal/include/c++.capnp b/cereal/include/c++.capnp new file mode 100644 index 00000000000000..2bda54717920ab --- /dev/null +++ b/cereal/include/c++.capnp @@ -0,0 +1,26 @@ +# Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors +# Licensed under the MIT License: +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +@0xbdf87d7bb8304e81; +$namespace("capnp::annotations"); + +annotation namespace(file): Text; +annotation name(field, enumerant, struct, enum, interface, method, param, group, union): Text; diff --git a/cereal/legacy.capnp b/cereal/legacy.capnp new file mode 100644 index 00000000000000..a8fa5e4a1f7d5d --- /dev/null +++ b/cereal/legacy.capnp @@ -0,0 +1,574 @@ +using Cxx = import "./include/c++.capnp"; +$Cxx.namespace("cereal"); + +@0x80ef1ec4889c2a63; + +# legacy.capnp: a home for deprecated structs + +struct LogRotate @0x9811e1f38f62f2d1 { + segmentNum @0 :Int32; + path @1 :Text; +} + +struct LiveUI @0xc08240f996aefced { + rearViewCam @0 :Bool; + alertText1 @1 :Text; + alertText2 @2 :Text; + awarenessStatus @3 :Float32; +} + +struct UiLayoutState @0x88dcce08ad29dda0 { + activeApp @0 :App; + sidebarCollapsed @1 :Bool; + mapEnabled @2 :Bool; + mockEngaged @3 :Bool; + + enum App @0x9917470acf94d285 { + home @0; + music @1; + nav @2; + settings @3; + none @4; + } +} + +struct OrbslamCorrection @0x8afd33dc9b35e1aa { + correctionMonoTime @0 :UInt64; + prePositionECEF @1 :List(Float64); + postPositionECEF @2 :List(Float64); + prePoseQuatECEF @3 :List(Float32); + postPoseQuatECEF @4 :List(Float32); + numInliers @5 :UInt32; +} + +struct EthernetPacket @0xa99a9d5b33cf5859 { + pkt @0 :Data; + ts @1 :Float32; +} + +struct CellInfo @0xcff7566681c277ce { + timestamp @0 :UInt64; + repr @1 :Text; # android toString() for now +} + +struct WifiScan @0xd4df5a192382ba0b { + bssid @0 :Text; + ssid @1 :Text; + capabilities @2 :Text; + frequency @3 :Int32; + level @4 :Int32; + timestamp @5 :Int64; + + centerFreq0 @6 :Int32; + centerFreq1 @7 :Int32; + channelWidth @8 :ChannelWidth; + operatorFriendlyName @9 :Text; + venueName @10 :Text; + is80211mcResponder @11 :Bool; + passpoint @12 :Bool; + + distanceCm @13 :Int32; + distanceSdCm @14 :Int32; + + enum ChannelWidth @0xcb6a279f015f6b51 { + w20Mhz @0; + w40Mhz @1; + w80Mhz @2; + w160Mhz @3; + w80Plus80Mhz @4; + } +} + +struct LiveEventData @0x94b7baa90c5c321e { + name @0 :Text; + value @1 :Int32; +} + +struct ModelData @0xb8aad62cffef28a9 { + frameId @0 :UInt32; + frameAge @12 :UInt32; + frameDropPerc @13 :Float32; + timestampEof @9 :UInt64; + modelExecutionTime @14 :Float32; + gpuExecutionTime @16 :Float32; + rawPred @15 :Data; + + path @1 :PathData; + leftLane @2 :PathData; + rightLane @3 :PathData; + lead @4 :LeadData; + freePath @6 :List(Float32); + + settings @5 :ModelSettings; + leadFuture @7 :LeadData; + speed @8 :List(Float32); + meta @10 :MetaData; + longitudinal @11 :LongitudinalData; + + struct PathData @0x8817eeea389e9f08 { + points @0 :List(Float32); + prob @1 :Float32; + std @2 :Float32; + stds @3 :List(Float32); + poly @4 :List(Float32); + validLen @5 :Float32; + } + + struct LeadData @0xd1c9bef96d26fa91 { + dist @0 :Float32; + prob @1 :Float32; + std @2 :Float32; + relVel @3 :Float32; + relVelStd @4 :Float32; + relY @5 :Float32; + relYStd @6 :Float32; + relA @7 :Float32; + relAStd @8 :Float32; + } + + struct ModelSettings @0xa26e3710efd3e914 { + bigBoxX @0 :UInt16; + bigBoxY @1 :UInt16; + bigBoxWidth @2 :UInt16; + bigBoxHeight @3 :UInt16; + boxProjection @4 :List(Float32); + yuvCorrection @5 :List(Float32); + inputTransform @6 :List(Float32); + } + + struct MetaData @0x9744f25fb60f2bf8 { + engagedProb @0 :Float32; + desirePrediction @1 :List(Float32); + brakeDisengageProb @2 :Float32; + gasDisengageProb @3 :Float32; + steerOverrideProb @4 :Float32; + desireState @5 :List(Float32); + } + + struct LongitudinalData @0xf98f999c6a071122 { + distances @2 :List(Float32); + speeds @0 :List(Float32); + accelerations @1 :List(Float32); + } +} + +struct ECEFPoint @0xc25bbbd524983447 { + x @0 :Float64; + y @1 :Float64; + z @2 :Float64; +} + +struct ECEFPointDEPRECATED @0xe10e21168db0c7f7 { + x @0 :Float32; + y @1 :Float32; + z @2 :Float32; +} + +struct GPSPlannerPoints @0xab54c59699f8f9f3 { + curPosDEPRECATED @0 :ECEFPointDEPRECATED; + pointsDEPRECATED @1 :List(ECEFPointDEPRECATED); + curPos @6 :ECEFPoint; + points @7 :List(ECEFPoint); + valid @2 :Bool; + trackName @3 :Text; + speedLimit @4 :Float32; + accelTarget @5 :Float32; +} + +struct GPSPlannerPlan @0xf5ad1d90cdc1dd6b { + valid @0 :Bool; + poly @1 :List(Float32); + trackName @2 :Text; + speed @3 :Float32; + acceleration @4 :Float32; + pointsDEPRECATED @5 :List(ECEFPointDEPRECATED); + points @6 :List(ECEFPoint); + xLookahead @7 :Float32; +} + +struct UiNavigationEvent @0x90c8426c3eaddd3b { + type @0: Type; + status @1: Status; + distanceTo @2: Float32; + endRoadPointDEPRECATED @3: ECEFPointDEPRECATED; + endRoadPoint @4: ECEFPoint; + + enum Type @0xe8db07dcf8fcea05 { + none @0; + laneChangeLeft @1; + laneChangeRight @2; + mergeLeft @3; + mergeRight @4; + turnLeft @5; + turnRight @6; + } + + enum Status @0xb9aa88c75ef99a1f { + none @0; + passive @1; + approaching @2; + active @3; + } +} + +struct LiveLocationData @0xb99b2bc7a57e8128 { + status @0 :UInt8; + + # 3D fix + lat @1 :Float64; + lon @2 :Float64; + alt @3 :Float32; # m + + # speed + speed @4 :Float32; # m/s + + # NED velocity components + vNED @5 :List(Float32); + + # roll, pitch, heading (x,y,z) + roll @6 :Float32; # WRT to center of earth? + pitch @7 :Float32; # WRT to center of earth? + heading @8 :Float32; # WRT to north? + + # what are these? + wanderAngle @9 :Float32; + trackAngle @10 :Float32; + + # car frame -- https://upload.wikimedia.org/wikipedia/commons/f/f5/RPY_angles_of_cars.png + + # gyro, in car frame, deg/s + gyro @11 :List(Float32); + + # accel, in car frame, m/s^2 + accel @12 :List(Float32); + + accuracy @13 :Accuracy; + + source @14 :SensorSource; + # if we are fixing a location in the past + fixMonoTime @15 :UInt64; + + gpsWeek @16 :Int32; + timeOfWeek @17 :Float64; + + positionECEF @18 :List(Float64); + poseQuatECEF @19 :List(Float32); + pitchCalibration @20 :Float32; + yawCalibration @21 :Float32; + imuFrame @22 :List(Float32); + + struct Accuracy @0x943dc4625473b03f { + pNEDError @0 :List(Float32); + vNEDError @1 :List(Float32); + rollError @2 :Float32; + pitchError @3 :Float32; + headingError @4 :Float32; + ellipsoidSemiMajorError @5 :Float32; + ellipsoidSemiMinorError @6 :Float32; + ellipsoidOrientationError @7 :Float32; + } + + enum SensorSource @0xc871d3cc252af657 { + applanix @0; + kalman @1; + orbslam @2; + timing @3; + dummy @4; + } +} + +struct OrbOdometry @0xd7700859ed1f5b76 { + # timing first + startMonoTime @0 :UInt64; + endMonoTime @1 :UInt64; + + # fundamental matrix and error + f @2: List(Float64); + err @3: Float64; + + # number of inlier points + inliers @4: Int32; + + # for debug only + # indexed by endMonoTime features + # value is startMonoTime feature match + # -1 if no match + matches @5: List(Int16); +} + +struct OrbFeatures @0xcd60164a8a0159ef { + timestampEof @0 :UInt64; + # transposed arrays of normalized image coordinates + # len(xs) == len(ys) == len(descriptors) * 32 + xs @1 :List(Float32); + ys @2 :List(Float32); + descriptors @3 :Data; + octaves @4 :List(Int8); + + # match index to last OrbFeatures + # -1 if no match + timestampLastEof @5 :UInt64; + matches @6: List(Int16); +} + +struct OrbFeaturesSummary @0xd500d30c5803fa4f { + timestampEof @0 :UInt64; + timestampLastEof @1 :UInt64; + + featureCount @2 :UInt16; + matchCount @3 :UInt16; + computeNs @4 :UInt64; +} + +struct OrbKeyFrame @0xc8233c0345e27e24 { + # this is a globally unique id for the KeyFrame + id @0: UInt64; + + # this is the location of the KeyFrame + pos @1: ECEFPoint; + + # these are the features in the world + # len(dpos) == len(descriptors) * 32 + dpos @2 :List(ECEFPoint); + descriptors @3 :Data; +} + +struct KalmanOdometry @0x92e21bb7ea38793a { + trans @0 :List(Float32); # m/s in device frame + rot @1 :List(Float32); # rad/s in device frame + transStd @2 :List(Float32); # std m/s in device frame + rotStd @3 :List(Float32); # std rad/s in device frame +} + +struct OrbObservation @0x9b326d4e436afec7 { + observationMonoTime @0 :UInt64; + normalizedCoordinates @1 :List(Float32); + locationECEF @2 :List(Float64); + matchDistance @3: UInt32; +} + +struct CalibrationFeatures @0x8fdfadb254ea867a { + frameId @0 :UInt32; + + p0 @1 :List(Float32); + p1 @2 :List(Float32); + status @3 :List(Int8); +} + +struct NavStatus @0xbd8822120928120c { + isNavigating @0 :Bool; + currentAddress @1 :Address; + + struct Address @0xce7cd672cacc7814 { + title @0 :Text; + lat @1 :Float64; + lng @2 :Float64; + house @3 :Text; + address @4 :Text; + street @5 :Text; + city @6 :Text; + state @7 :Text; + country @8 :Text; + } +} + +struct NavUpdate @0xdb98be6565516acb { + isNavigating @0 :Bool; + curSegment @1 :Int32; + segments @2 :List(Segment); + + struct LatLng @0x9eaef9187cadbb9b { + lat @0 :Float64; + lng @1 :Float64; + } + + struct Segment @0xa5b39b4fc4d7da3f { + from @0 :LatLng; + to @1 :LatLng; + updateTime @2 :Int32; + distance @3 :Int32; + crossTime @4 :Int32; + exitNo @5 :Int32; + instruction @6 :Instruction; + + parts @7 :List(LatLng); + + enum Instruction @0xc5417a637451246f { + turnLeft @0; + turnRight @1; + keepLeft @2; + keepRight @3; + straight @4; + roundaboutExitNumber @5; + roundaboutExit @6; + roundaboutTurnLeft @7; + unkn8 @8; + roundaboutStraight @9; + unkn10 @10; + roundaboutTurnRight @11; + unkn12 @12; + roundaboutUturn @13; + unkn14 @14; + arrive @15; + exitLeft @16; + exitRight @17; + unkn18 @18; + uturn @19; + # ... + } + } +} + +struct TrafficEvent @0xacfa74a094e62626 { + type @0 :Type; + distance @1 :Float32; + action @2 :Action; + resuming @3 :Bool; + + enum Type @0xd85d75253435bf4b { + stopSign @0; + lightRed @1; + lightYellow @2; + lightGreen @3; + stopLight @4; + } + + enum Action @0xa6f6ce72165ccb49 { + none @0; + yield @1; + stop @2; + resumeReady @3; + } + +} + + +struct AndroidGnss @0xdfdf30d03fc485bd { + union { + measurements @0 :Measurements; + navigationMessage @1 :NavigationMessage; + } + + struct Measurements @0xa20710d4f428d6cd { + clock @0 :Clock; + measurements @1 :List(Measurement); + + struct Clock @0xa0e27b453a38f450 { + timeNanos @0 :Int64; + hardwareClockDiscontinuityCount @1 :Int32; + + hasTimeUncertaintyNanos @2 :Bool; + timeUncertaintyNanos @3 :Float64; + + hasLeapSecond @4 :Bool; + leapSecond @5 :Int32; + + hasFullBiasNanos @6 :Bool; + fullBiasNanos @7 :Int64; + + hasBiasNanos @8 :Bool; + biasNanos @9 :Float64; + + hasBiasUncertaintyNanos @10 :Bool; + biasUncertaintyNanos @11 :Float64; + + hasDriftNanosPerSecond @12 :Bool; + driftNanosPerSecond @13 :Float64; + + hasDriftUncertaintyNanosPerSecond @14 :Bool; + driftUncertaintyNanosPerSecond @15 :Float64; + } + + struct Measurement @0xd949bf717d77614d { + svId @0 :Int32; + constellation @1 :Constellation; + + timeOffsetNanos @2 :Float64; + state @3 :Int32; + receivedSvTimeNanos @4 :Int64; + receivedSvTimeUncertaintyNanos @5 :Int64; + cn0DbHz @6 :Float64; + pseudorangeRateMetersPerSecond @7 :Float64; + pseudorangeRateUncertaintyMetersPerSecond @8 :Float64; + accumulatedDeltaRangeState @9 :Int32; + accumulatedDeltaRangeMeters @10 :Float64; + accumulatedDeltaRangeUncertaintyMeters @11 :Float64; + + hasCarrierFrequencyHz @12 :Bool; + carrierFrequencyHz @13 :Float32; + hasCarrierCycles @14 :Bool; + carrierCycles @15 :Int64; + hasCarrierPhase @16 :Bool; + carrierPhase @17 :Float64; + hasCarrierPhaseUncertainty @18 :Bool; + carrierPhaseUncertainty @19 :Float64; + hasSnrInDb @20 :Bool; + snrInDb @21 :Float64; + + multipathIndicator @22 :MultipathIndicator; + + enum Constellation @0x9ef1f3ff0deb5ffb { + unknown @0; + gps @1; + sbas @2; + glonass @3; + qzss @4; + beidou @5; + galileo @6; + } + + enum State @0xcbb9490adce12d72 { + unknown @0; + codeLock @1; + bitSync @2; + subframeSync @3; + towDecoded @4; + msecAmbiguous @5; + symbolSync @6; + gloStringSync @7; + gloTodDecoded @8; + bdsD2BitSync @9; + bdsD2SubframeSync @10; + galE1bcCodeLock @11; + galE1c2ndCodeLock @12; + galE1bPageSync @13; + sbasSync @14; + } + + enum MultipathIndicator @0xc04e7b6231d4caa8 { + unknown @0; + detected @1; + notDetected @2; + } + } + } + + struct NavigationMessage @0xe2517b083095fd4e { + type @0 :Int32; + svId @1 :Int32; + messageId @2 :Int32; + submessageId @3 :Int32; + data @4 :Data; + status @5 :Status; + + enum Status @0xec1ff7996b35366f { + unknown @0; + parityPassed @1; + parityRebuilt @2; + } + } +} + +struct LidarPts @0xe3d6685d4e9d8f7a { + r @0 :List(UInt16); # uint16 m*500.0 + theta @1 :List(UInt16); # uint16 deg*100.0 + reflect @2 :List(UInt8); # uint8 0-255 + + # For storing out of file. + idx @3 :UInt64; + + # For storing in file + pkt @4 :Data; +} + + diff --git a/cereal/log.capnp b/cereal/log.capnp new file mode 100644 index 00000000000000..36ca692dcc32c3 --- /dev/null +++ b/cereal/log.capnp @@ -0,0 +1,2369 @@ +using Cxx = import "./include/c++.capnp"; +$Cxx.namespace("cereal"); + +using Car = import "car.capnp"; +using Legacy = import "legacy.capnp"; +using Custom = import "custom.capnp"; + +@0xf3b1f17e25a4285b; + +const logVersion :Int32 = 1; + +struct Map(Key, Value) { + entries @0 :List(Entry); + struct Entry { + key @0 :Key; + value @1 :Value; + } +} + +enum LongitudinalPersonality { + aggressive @0; + standard @1; + relaxed @2; +} + +struct InitData { + kernelArgs @0 :List(Text); + kernelVersion @15 :Text; + osVersion @18 :Text; + + dongleId @2 :Text; + bootlogId @22 :Text; + + deviceType @3 :DeviceType; + version @4 :Text; + gitCommit @10 :Text; + gitCommitDate @21 :Text; + gitBranch @11 :Text; + gitRemote @13 :Text; + + androidProperties @16 :Map(Text, Text); + + pandaInfo @8 :PandaInfo; + + dirty @9 :Bool; + passive @12 :Bool; + params @17 :Map(Text, Data); + + commands @19 :Map(Text, Data); + + wallTimeNanos @20 :UInt64; + + enum DeviceType { + unknown @0; + neo @1; + chffrAndroid @2; + chffrIos @3; + tici @4; + pc @5; + tizi @6; + mici @7; + } + + struct PandaInfo { + hasPanda @0 :Bool; + dongleId @1 :Text; + stVersion @2 :Text; + espVersion @3 :Text; + } + + # ***** deprecated stuff ***** + gctxDEPRECATED @1 :Text; + androidBuildInfo @5 :AndroidBuildInfo; + androidSensorsDEPRECATED @6 :List(AndroidSensor); + chffrAndroidExtraDEPRECATED @7 :ChffrAndroidExtra; + iosBuildInfoDEPRECATED @14 :IosBuildInfo; + + struct AndroidBuildInfo { + board @0 :Text; + bootloader @1 :Text; + brand @2 :Text; + device @3 :Text; + display @4 :Text; + fingerprint @5 :Text; + hardware @6 :Text; + host @7 :Text; + id @8 :Text; + manufacturer @9 :Text; + model @10 :Text; + product @11 :Text; + radioVersion @12 :Text; + serial @13 :Text; + supportedAbis @14 :List(Text); + tags @15 :Text; + time @16 :Int64; + type @17 :Text; + user @18 :Text; + + versionCodename @19 :Text; + versionRelease @20 :Text; + versionSdk @21 :Int32; + versionSecurityPatch @22 :Text; + } + + struct AndroidSensor { + id @0 :Int32; + name @1 :Text; + vendor @2 :Text; + version @3 :Int32; + handle @4 :Int32; + type @5 :Int32; + maxRange @6 :Float32; + resolution @7 :Float32; + power @8 :Float32; + minDelay @9 :Int32; + fifoReservedEventCount @10 :UInt32; + fifoMaxEventCount @11 :UInt32; + stringType @12 :Text; + maxDelay @13 :Int32; + } + + struct ChffrAndroidExtra { + allCameraCharacteristics @0 :Map(Text, Text); + } + + struct IosBuildInfo { + appVersion @0 :Text; + appBuild @1 :UInt32; + osVersion @2 :Text; + deviceModel @3 :Text; + } +} + +struct FrameData { + frameId @0 :UInt32; + frameIdSensor @25 :UInt32; + requestId @28 :UInt32; + encodeId @1 :UInt32; + + frameType @7 :FrameType; + + # Timestamps + timestampEof @2 :UInt64; + timestampSof @8 :UInt64; + processingTime @23 :Float32; + + # Exposure + integLines @4 :Int32; + highConversionGain @20 :Bool; + gain @15 :Float32; # This includes highConversionGain if enabled + measuredGreyFraction @21 :Float32; + targetGreyFraction @22 :Float32; + exposureValPercent @27 :Float32; + + transform @10 :List(Float32); + + image @6 :Data; + + temperaturesC @24 :List(Float32); + + enum FrameType { + unknown @0; + neo @1; + chffrAndroid @2; + front @3; + } + + sensor @26 :ImageSensor; + enum ImageSensor { + unknown @0; + ar0231 @1; + ox03c10 @2; + os04c10 @3; + } + + frameLengthDEPRECATED @3 :Int32; + globalGainDEPRECATED @5 :Int32; + androidCaptureResultDEPRECATED @9 :AndroidCaptureResult; + lensPosDEPRECATED @11 :Int32; + lensSagDEPRECATED @12 :Float32; + lensErrDEPRECATED @13 :Float32; + lensTruePosDEPRECATED @14 :Float32; + focusValDEPRECATED @16 :List(Int16); + focusConfDEPRECATED @17 :List(UInt8); + sharpnessScoreDEPRECATED @18 :List(UInt16); + recoverStateDEPRECATED @19 :Int32; + struct AndroidCaptureResult { + sensitivity @0 :Int32; + frameDuration @1 :Int64; + exposureTime @2 :Int64; + rollingShutterSkew @3 :UInt64; + colorCorrectionTransform @4 :List(Int32); + colorCorrectionGains @5 :List(Float32); + displayRotation @6 :Int8; + } +} + +struct Thumbnail { + frameId @0 :UInt32; + timestampEof @1 :UInt64; + thumbnail @2 :Data; + encoding @3 :Encoding; + + enum Encoding { + unknown @0; + jpeg @1; + keyframe @2; + } +} + +struct GPSNMEAData { + timestamp @0 :Int64; + localWallTime @1 :UInt64; + nmea @2 :Text; +} + +# android sensor_event_t +struct SensorEventData { + version @0 :Int32; + sensor @1 :Int32; + type @2 :Int32; + timestamp @3 :Int64; + uncalibratedDEPRECATED @10 :Bool; + + union { + acceleration @4 :SensorVec; + magnetic @5 :SensorVec; + orientation @6 :SensorVec; + gyro @7 :SensorVec; + pressure @9 :SensorVec; + magneticUncalibrated @11 :SensorVec; + gyroUncalibrated @12 :SensorVec; + proximity @13: Float32; + light @14: Float32; + temperature @15: Float32; + } + source @8 :SensorSource; + + struct SensorVec { + v @0 :List(Float32); + status @1 :Int8; + } + + enum SensorSource { + android @0; + iOS @1; + fiber @2; + velodyne @3; # Velodyne IMU + bno055 @4; # Bosch accelerometer + lsm6ds3 @5; # includes LSM6DS3 and LSM6DS3TR, TR = tape reel + bmp280 @6; # barometer + mmc3416x @7; # magnetometer + bmx055 @8; + rpr0521 @9; + lsm6ds3trc @10; + mmc5603nj @11; + } +} + +# android struct GpsLocation +struct GpsLocationData { + # Contains module-specific flags. + flags @0 :UInt16; + + # Represents latitude in degrees. + latitude @1 :Float64; + + # Represents longitude in degrees. + longitude @2 :Float64; + + # Represents altitude in meters above the WGS 84 reference ellipsoid. + altitude @3 :Float64; + + # Represents speed in meters per second. + speed @4 :Float32; + + # Represents heading in degrees. + bearingDeg @5 :Float32; + + # Represents expected horizontal accuracy in meters. + horizontalAccuracy @6 :Float32; + + unixTimestampMillis @7 :Int64; + + source @8 :SensorSource; + + # Represents NED velocity in m/s. + vNED @9 :List(Float32); + + # Represents expected vertical accuracy in meters. (presumably 1 sigma?) + verticalAccuracy @10 :Float32; + + # Represents bearing accuracy in degrees. (presumably 1 sigma?) + bearingAccuracyDeg @11 :Float32; + + # Represents velocity accuracy in m/s. (presumably 1 sigma?) + speedAccuracy @12 :Float32; + + hasFix @13 :Bool; + + enum SensorSource { + android @0; + iOS @1; + car @2; + velodyne @3; # Velodyne IMU + fusion @4; + external @5; + ublox @6; + trimble @7; + qcomdiag @8; + unicore @9; + } +} + +enum Desire { + none @0; + turnLeft @1; + turnRight @2; + laneChangeLeft @3; + laneChangeRight @4; + keepLeft @5; + keepRight @6; +} + +enum LaneChangeState { + off @0; + preLaneChange @1; + laneChangeStarting @2; + laneChangeFinishing @3; +} + +enum LaneChangeDirection { + none @0; + left @1; + right @2; +} + +struct CanData { + address @0 :UInt32; + busTime @1 :UInt16; + dat @2 :Data; + src @3 :UInt8; +} + +struct DeviceState @0xa4d8b5af2aa492eb { + deviceType @45 :InitData.DeviceType; + + networkType @22 :NetworkType; + networkInfo @31 :NetworkInfo; + networkStrength @24 :NetworkStrength; + networkStats @43 :NetworkStats; + networkMetered @41 :Bool; + lastAthenaPingTime @32 :UInt64; + + started @11 :Bool; + startedMonoTime @13 :UInt64; + + # system utilization + freeSpacePercent @7 :Float32; + memoryUsagePercent @19 :Int8; + gpuUsagePercent @33 :Int8; + cpuUsagePercent @34 :List(Int8); # per-core cpu usage + + # power + offroadPowerUsageUwh @23 :UInt32; + carBatteryCapacityUwh @25 :UInt32; + powerDrawW @40 :Float32; + somPowerDrawW @42 :Float32; + + # device thermals + cpuTempC @26 :List(Float32); + gpuTempC @27 :List(Float32); + memoryTempC @28 :Float32; + nvmeTempC @35 :List(Float32); + modemTempC @36 :List(Float32); + pmicTempC @39 :List(Float32); + maxTempC @44 :Float32; # max of other temps, used to control fan + thermalZones @38 :List(ThermalZone); + thermalStatus @14 :ThermalStatus; + + fanSpeedPercentDesired @10 :UInt16; + screenBrightnessPercent @37 :Int8; + + struct ThermalZone { + name @0 :Text; + temp @1 :Float32; + } + + enum ThermalStatus { + green @0; + yellow @1; + red @2; + danger @3; + } + + enum NetworkType { + none @0; + wifi @1; + cell2G @2; + cell3G @3; + cell4G @4; + cell5G @5; + ethernet @6; + } + + enum NetworkStrength { + unknown @0; + poor @1; + moderate @2; + good @3; + great @4; + } + + struct NetworkInfo { + technology @0 :Text; + operator @1 :Text; + band @2 :Text; + channel @3 :UInt16; + extra @4 :Text; + state @5 :Text; + } + + struct NetworkStats { + wwanTx @0 :Int64; + wwanRx @1 :Int64; + } + + # deprecated + cpu0DEPRECATED @0 :UInt16; + cpu1DEPRECATED @1 :UInt16; + cpu2DEPRECATED @2 :UInt16; + cpu3DEPRECATED @3 :UInt16; + memDEPRECATED @4 :UInt16; + gpuDEPRECATED @5 :UInt16; + batDEPRECATED @6 :UInt32; + pa0DEPRECATED @21 :UInt16; + cpuUsagePercentDEPRECATED @20 :Int8; + batteryStatusDEPRECATED @9 :Text; + batteryVoltageDEPRECATED @16 :Int32; + batteryTempCDEPRECATED @29 :Float32; + batteryPercentDEPRECATED @8 :Int16; + batteryCurrentDEPRECATED @15 :Int32; + chargingErrorDEPRECATED @17 :Bool; + chargingDisabledDEPRECATED @18 :Bool; + usbOnlineDEPRECATED @12 :Bool; + ambientTempCDEPRECATED @30 :Float32; +} + +struct PandaState @0xa7649e2575e4591e { + ignitionLine @2 :Bool; + rxBufferOverflow @7 :UInt32; + txBufferOverflow @8 :UInt32; + pandaType @10 :PandaType; + ignitionCan @13 :Bool; + faultStatus @15 :FaultStatus; + powerSaveEnabled @16 :Bool; + uptime @17 :UInt32; + faults @18 :List(FaultType); + heartbeatLost @22 :Bool; + interruptLoad @25 :Float32; + fanPower @28 :UInt8; + fanStallCount @34 :UInt8; + + spiChecksumErrorCount @33 :UInt16; + + harnessStatus @21 :HarnessStatus; + sbu1Voltage @35 :Float32; + sbu2Voltage @36 :Float32; + + # can health + canState0 @29 :PandaCanState; + canState1 @30 :PandaCanState; + canState2 @31 :PandaCanState; + + # safety stuff + controlsAllowed @3 :Bool; + safetyRxInvalid @19 :UInt32; + safetyTxBlocked @24 :UInt32; + safetyModel @14 :Car.CarParams.SafetyModel; + safetyParam @27 :UInt16; + alternativeExperience @23 :Int16; + safetyRxChecksInvalid @32 :Bool; + + voltage @0 :UInt32; + current @1 :UInt32; + + enum FaultStatus { + none @0; + faultTemp @1; + faultPerm @2; + } + + enum FaultType { + relayMalfunction @0; + unusedInterruptHandled @1; + interruptRateCan1 @2; + interruptRateCan2 @3; + interruptRateCan3 @4; + interruptRateTach @5; + interruptRateGmlanDEPRECATED @6; + interruptRateInterrupts @7; + interruptRateSpiDma @8; + interruptRateSpiCs @9; + interruptRateUart1 @10; + interruptRateUart2 @11; + interruptRateUart3 @12; + interruptRateUart5 @13; + interruptRateUartDma @14; + interruptRateUsb @15; + interruptRateTim1 @16; + interruptRateTim3 @17; + registerDivergent @18; + interruptRateKlineInit @19; + interruptRateClockSource @20; + interruptRateTick @21; + interruptRateExti @22; + interruptRateSpi @23; + interruptRateUart7 @24; + sirenMalfunction @25; + heartbeatLoopWatchdog @26; + # Update max fault type in boardd when adding faults + } + + enum PandaType @0x8a58adf93e5b3751 { + unknown @0; + whitePanda @1; + greyPanda @2; + blackPanda @3; + pedal @4; + uno @5; + dos @6; + redPanda @7; + redPandaV2 @8; + tres @9; + cuatro @10; + } + + enum HarnessStatus { + notConnected @0; + normal @1; + flipped @2; + } + + struct PandaCanState { + busOff @0 :Bool; + busOffCnt @1 :UInt32; + errorWarning @2 :Bool; + errorPassive @3 :Bool; + lastError @4 :LecErrorCode; + lastStoredError @5 :LecErrorCode; + lastDataError @6 :LecErrorCode; + lastDataStoredError @7 :LecErrorCode; + receiveErrorCnt @8 :UInt8; + transmitErrorCnt @9 :UInt8; + totalErrorCnt @10 :UInt32; + totalTxLostCnt @11 :UInt32; + totalRxLostCnt @12 :UInt32; + totalTxCnt @13 :UInt32; + totalRxCnt @14 :UInt32; + totalFwdCnt @15 :UInt32; + canSpeed @16 :UInt16; + canDataSpeed @17 :UInt16; + canfdEnabled @18 :Bool; + brsEnabled @19 :Bool; + canfdNonIso @20 :Bool; + irq0CallRate @21 :UInt32; + irq1CallRate @22 :UInt32; + irq2CallRate @23 :UInt32; + canCoreResetCnt @24 :UInt32; + + enum LecErrorCode { + noError @0; + stuffError @1; + formError @2; + ackError @3; + bit1Error @4; + bit0Error @5; + crcError @6; + noChange @7; + } + } + + gasInterceptorDetectedDEPRECATED @4 :Bool; + startedSignalDetectedDEPRECATED @5 :Bool; + hasGpsDEPRECATED @6 :Bool; + gmlanSendErrsDEPRECATED @9 :UInt32; + fanSpeedRpmDEPRECATED @11 :UInt16; + usbPowerModeDEPRECATED @12 :PeripheralState.UsbPowerModeDEPRECATED; + safetyParamDEPRECATED @20 :Int16; + safetyParam2DEPRECATED @26 :UInt32; +} + +struct PeripheralState { + pandaType @0 :PandaState.PandaType; + voltage @1 :UInt32; + current @2 :UInt32; + fanSpeedRpm @3 :UInt16; + + usbPowerModeDEPRECATED @4 :UsbPowerModeDEPRECATED; + enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 { + none @0; + client @1; + cdp @2; + dcp @3; + } +} + +struct RadarState @0x9a185389d6fdd05f { + mdMonoTime @6 :UInt64; + carStateMonoTime @11 :UInt64; + radarErrors @12 :List(Car.RadarData.Error); + + leadOne @3 :LeadData; + leadTwo @4 :LeadData; + cumLagMs @5 :Float32; + + struct LeadData { + dRel @0 :Float32; + yRel @1 :Float32; + vRel @2 :Float32; + aRel @3 :Float32; + vLead @4 :Float32; + dPath @6 :Float32; + vLat @7 :Float32; + vLeadK @8 :Float32; + aLeadK @9 :Float32; + fcw @10 :Bool; + status @11 :Bool; + aLeadTau @12 :Float32; + modelProb @13 :Float32; + radar @14 :Bool; + radarTrackId @15 :Int32 = -1; + + aLeadDEPRECATED @5 :Float32; + } + + # deprecated + ftMonoTimeDEPRECATED @7 :UInt64; + warpMatrixDEPRECATED @0 :List(Float32); + angleOffsetDEPRECATED @1 :Float32; + calStatusDEPRECATED @2 :Int8; + calCycleDEPRECATED @8 :Int32; + calPercDEPRECATED @9 :Int8; + canMonoTimesDEPRECATED @10 :List(UInt64); +} + +struct LiveCalibrationData { + calStatus @11 :Status; + calCycle @2 :Int32; + calPerc @3 :Int8; + validBlocks @9 :Int32; + + # view_frame_from_road_frame + # ui's is inversed needs new + extrinsicMatrix @4 :List(Float32); + # the direction of travel vector in device frame + rpyCalib @7 :List(Float32); + rpyCalibSpread @8 :List(Float32); + wideFromDeviceEuler @10 :List(Float32); + height @12 :List(Float32); + + warpMatrixDEPRECATED @0 :List(Float32); + calStatusDEPRECATED @1 :Int8; + warpMatrix2DEPRECATED @5 :List(Float32); + warpMatrixBigDEPRECATED @6 :List(Float32); + + enum Status { + uncalibrated @0; + calibrated @1; + invalid @2; + recalibrating @3; + } +} + +struct LiveTracks { + trackId @0 :Int32; + dRel @1 :Float32; + yRel @2 :Float32; + vRel @3 :Float32; + aRel @4 :Float32; + timeStamp @5 :Float32; + status @6 :Float32; + currentTime @7 :Float32; + stationary @8 :Bool; + oncoming @9 :Bool; +} + +struct ControlsState @0x97ff69c53601abf1 { + startMonoTime @48 :UInt64; + longitudinalPlanMonoTime @28 :UInt64; + lateralPlanMonoTime @50 :UInt64; + + state @31 :OpenpilotState; + enabled @19 :Bool; + active @36 :Bool; + + experimentalMode @64 :Bool; + personality @66 :LongitudinalPersonality; + + longControlState @30 :Car.CarControl.Actuators.LongControlState; + vPid @2 :Float32; + vTargetLead @3 :Float32; + vCruise @22 :Float32; # actual set speed + vCruiseCluster @63 :Float32; # set speed to display in the UI + upAccelCmd @4 :Float32; + uiAccelCmd @5 :Float32; + ufAccelCmd @33 :Float32; + aTarget @35 :Float32; + curvature @37 :Float32; # path curvature from vehicle model + desiredCurvature @61 :Float32; # lag adjusted curvatures used by lateral controllers + forceDecel @51 :Bool; + + # UI alerts + alertText1 @24 :Text; + alertText2 @25 :Text; + alertStatus @38 :AlertStatus; + alertSize @39 :AlertSize; + alertBlinkingRate @42 :Float32; + alertType @44 :Text; + alertSound @56 :Car.CarControl.HUDControl.AudibleAlert; + engageable @41 :Bool; # can OP be engaged? + + cumLagMs @15 :Float32; + + lateralControlState :union { + indiState @52 :LateralINDIState; + pidState @53 :LateralPIDState; + angleState @58 :LateralAngleState; + debugState @59 :LateralDebugState; + torqueState @60 :LateralTorqueState; + + curvatureStateDEPRECATED @65 :LateralCurvatureState; + lqrStateDEPRECATED @55 :LateralLQRState; + } + + enum OpenpilotState @0xdbe58b96d2d1ac61 { + disabled @0; + preEnabled @1; + enabled @2; + softDisabling @3; + overriding @4; # superset of overriding with steering or accelerator + } + + enum AlertStatus { + normal @0; # low priority alert for user's convenience + userPrompt @1; # mid priority alert that might require user intervention + critical @2; # high priority alert that needs immediate user intervention + } + + enum AlertSize { + none @0; # don't display the alert + small @1; # small box + mid @2; # mid screen + full @3; # full screen + } + + struct LateralINDIState { + active @0 :Bool; + steeringAngleDeg @1 :Float32; + steeringRateDeg @2 :Float32; + steeringAccelDeg @3 :Float32; + rateSetPoint @4 :Float32; + accelSetPoint @5 :Float32; + accelError @6 :Float32; + delayedOutput @7 :Float32; + delta @8 :Float32; + output @9 :Float32; + saturated @10 :Bool; + steeringAngleDesiredDeg @11 :Float32; + steeringRateDesiredDeg @12 :Float32; + } + + struct LateralPIDState { + active @0 :Bool; + steeringAngleDeg @1 :Float32; + steeringRateDeg @2 :Float32; + angleError @3 :Float32; + p @4 :Float32; + i @5 :Float32; + f @6 :Float32; + output @7 :Float32; + saturated @8 :Bool; + steeringAngleDesiredDeg @9 :Float32; + } + + struct LateralTorqueState { + active @0 :Bool; + error @1 :Float32; + errorRate @8 :Float32; + p @2 :Float32; + i @3 :Float32; + d @4 :Float32; + f @5 :Float32; + output @6 :Float32; + saturated @7 :Bool; + actualLateralAccel @9 :Float32; + desiredLateralAccel @10 :Float32; + } + + struct LateralLQRState { + active @0 :Bool; + steeringAngleDeg @1 :Float32; + i @2 :Float32; + output @3 :Float32; + lqrOutput @4 :Float32; + saturated @5 :Bool; + steeringAngleDesiredDeg @6 :Float32; + } + + struct LateralAngleState { + active @0 :Bool; + steeringAngleDeg @1 :Float32; + output @2 :Float32; + saturated @3 :Bool; + steeringAngleDesiredDeg @4 :Float32; + } + + struct LateralCurvatureState { + active @0 :Bool; + actualCurvature @1 :Float32; + desiredCurvature @2 :Float32; + error @3 :Float32; + p @4 :Float32; + i @5 :Float32; + f @6 :Float32; + output @7 :Float32; + saturated @8 :Bool; + } + + struct LateralDebugState { + active @0 :Bool; + steeringAngleDeg @1 :Float32; + output @2 :Float32; + saturated @3 :Bool; + } + + # deprecated + vEgoDEPRECATED @0 :Float32; + vEgoRawDEPRECATED @32 :Float32; + aEgoDEPRECATED @1 :Float32; + canMonoTimeDEPRECATED @16 :UInt64; + radarStateMonoTimeDEPRECATED @17 :UInt64; + mdMonoTimeDEPRECATED @18 :UInt64; + yActualDEPRECATED @6 :Float32; + yDesDEPRECATED @7 :Float32; + upSteerDEPRECATED @8 :Float32; + uiSteerDEPRECATED @9 :Float32; + ufSteerDEPRECATED @34 :Float32; + aTargetMinDEPRECATED @10 :Float32; + aTargetMaxDEPRECATED @11 :Float32; + rearViewCamDEPRECATED @23 :Bool; + driverMonitoringOnDEPRECATED @43 :Bool; + hudLeadDEPRECATED @14 :Int32; + alertSoundDEPRECATED @45 :Text; + angleModelBiasDEPRECATED @27 :Float32; + gpsPlannerActiveDEPRECATED @40 :Bool; + decelForTurnDEPRECATED @47 :Bool; + decelForModelDEPRECATED @54 :Bool; + awarenessStatusDEPRECATED @26 :Float32; + angleSteersDEPRECATED @13 :Float32; + vCurvatureDEPRECATED @46 :Float32; + mapValidDEPRECATED @49 :Bool; + jerkFactorDEPRECATED @12 :Float32; + steerOverrideDEPRECATED @20 :Bool; + steeringAngleDesiredDegDEPRECATED @29 :Float32; + canMonoTimesDEPRECATED @21 :List(UInt64); + desiredCurvatureRateDEPRECATED @62 :Float32; + canErrorCounterDEPRECATED @57 :UInt32; +} + +# All SI units and in device frame +struct XYZTData @0xc3cbae1fd505ae80 { + x @0 :List(Float32); + y @1 :List(Float32); + z @2 :List(Float32); + t @3 :List(Float32); + xStd @4 :List(Float32); + yStd @5 :List(Float32); + zStd @6 :List(Float32); +} + +struct ModelDataV2 { + frameId @0 :UInt32; + frameIdExtra @20 :UInt32; + frameAge @1 :UInt32; + frameDropPerc @2 :Float32; + timestampEof @3 :UInt64; + modelExecutionTime @15 :Float32; + gpuExecutionTime @17 :Float32; + rawPredictions @16 :Data; + + # predicted future position, orientation, etc.. + position @4 :XYZTData; + orientation @5 :XYZTData; + velocity @6 :XYZTData; + orientationRate @7 :XYZTData; + acceleration @19 :XYZTData; + + # prediction lanelines and road edges + laneLines @8 :List(XYZTData); + laneLineProbs @9 :List(Float32); + laneLineStds @13 :List(Float32); + roadEdges @10 :List(XYZTData); + roadEdgeStds @14 :List(Float32); + + # predicted lead cars + leads @11 :List(LeadDataV2); + leadsV3 @18 :List(LeadDataV3); + + meta @12 :MetaData; + confidence @23: ConfidenceClass; + + # Model perceived motion + temporalPose @21 :Pose; + + navEnabledDEPRECATED @22 :Bool; + locationMonoTimeDEPRECATED @24 :UInt64; + + # e2e lateral planner + lateralPlannerSolutionDEPRECATED @25: LateralPlannerSolution; + action @26: Action; + + struct LeadDataV2 { + prob @0 :Float32; # probability that car is your lead at time t + t @1 :Float32; + + # x and y are relative position in device frame + # v is norm relative speed + # a is norm relative acceleration + xyva @2 :List(Float32); + xyvaStd @3 :List(Float32); + } + + struct LeadDataV3 { + prob @0 :Float32; # probability that car is your lead at time t + probTime @1 :Float32; + t @2 :List(Float32); + + # x and y are relative position in device frame + # v absolute norm speed + # a is derivative of v + x @3 :List(Float32); + xStd @4 :List(Float32); + y @5 :List(Float32); + yStd @6 :List(Float32); + v @7 :List(Float32); + vStd @8 :List(Float32); + a @9 :List(Float32); + aStd @10 :List(Float32); + } + + + struct MetaData { + engagedProb @0 :Float32; + desirePrediction @1 :List(Float32); + desireState @5 :List(Float32); + disengagePredictions @6 :DisengagePredictions; + hardBrakePredicted @7 :Bool; + laneChangeState @8 :LaneChangeState; + laneChangeDirection @9 :LaneChangeDirection; + + + # deprecated + brakeDisengageProbDEPRECATED @2 :Float32; + gasDisengageProbDEPRECATED @3 :Float32; + steerOverrideProbDEPRECATED @4 :Float32; + } + + enum ConfidenceClass { + red @0; + yellow @1; + green @2; + } + + struct DisengagePredictions { + t @0 :List(Float32); + brakeDisengageProbs @1 :List(Float32); + gasDisengageProbs @2 :List(Float32); + steerOverrideProbs @3 :List(Float32); + brake3MetersPerSecondSquaredProbs @4 :List(Float32); + brake4MetersPerSecondSquaredProbs @5 :List(Float32); + brake5MetersPerSecondSquaredProbs @6 :List(Float32); + } + + struct Pose { + trans @0 :List(Float32); # m/s in device frame + rot @1 :List(Float32); # rad/s in device frame + transStd @2 :List(Float32); # std m/s in device frame + rotStd @3 :List(Float32); # std rad/s in device frame + } + + struct LateralPlannerSolution { + x @0 :List(Float32); + y @1 :List(Float32); + yaw @2 :List(Float32); + yawRate @3 :List(Float32); + xStd @4 :List(Float32); + yStd @5 :List(Float32); + yawStd @6 :List(Float32); + yawRateStd @7 :List(Float32); + } + + struct Action { + desiredCurvature @0 :Float32; + } +} + +struct EncodeIndex { + # picture from camera + frameId @0 :UInt32; + type @1 :Type; + # index of encoder from start of route + encodeId @2 :UInt32; + # minute long segment this frame is in + segmentNum @3 :Int32; + # index into camera file in segment in presentation order + segmentId @4 :UInt32; + # index into camera file in segment in encode order + segmentIdEncode @5 :UInt32; + timestampSof @6 :UInt64; + timestampEof @7 :UInt64; + + # encoder metadata + flags @8 :UInt32; + len @9 :UInt32; + + enum Type { + bigBoxLossless @0; + fullHEVC @1; + qcameraH264 @6; + livestreamH264 @7; + + # deprecated + bigBoxHEVCDEPRECATED @2; + chffrAndroidH264DEPRECATED @3; + fullLosslessClipDEPRECATED @4; + frontDEPRECATED @5; + + } +} + +struct AndroidLogEntry { + id @0 :UInt8; + ts @1 :UInt64; + priority @2 :UInt8; + pid @3 :Int32; + tid @4 :Int32; + tag @5 :Text; + message @6 :Text; +} + +struct LongitudinalPlan @0xe00b5b3eba12876c { + modelMonoTime @9 :UInt64; + hasLead @7 :Bool; + fcw @8 :Bool; + longitudinalPlanSource @15 :LongitudinalPlanSource; + processingDelay @29 :Float32; + + # desired speed/accel/jerk over next 2.5s + accels @32 :List(Float32); + speeds @33 :List(Float32); + jerks @34 :List(Float32); + + solverExecutionTime @35 :Float32; + + enum LongitudinalPlanSource { + cruise @0; + lead0 @1; + lead1 @2; + lead2 @3; + e2e @4; + } + + # deprecated + vCruiseDEPRECATED @16 :Float32; + aCruiseDEPRECATED @17 :Float32; + vTargetDEPRECATED @3 :Float32; + vTargetFutureDEPRECATED @14 :Float32; + aTargetDEPRECATED @18 :Float32; + vStartDEPRECATED @26 :Float32; + aStartDEPRECATED @27 :Float32; + vMaxDEPRECATED @20 :Float32; + radarStateMonoTimeDEPRECATED @10 :UInt64; + jerkFactorDEPRECATED @6 :Float32; + hasLeftLaneDEPRECATED @23 :Bool; + hasRightLaneDEPRECATED @24 :Bool; + aTargetMinDEPRECATED @4 :Float32; + aTargetMaxDEPRECATED @5 :Float32; + lateralValidDEPRECATED @0 :Bool; + longitudinalValidDEPRECATED @2 :Bool; + dPolyDEPRECATED @1 :List(Float32); + laneWidthDEPRECATED @11 :Float32; + vCurvatureDEPRECATED @21 :Float32; + decelForTurnDEPRECATED @22 :Bool; + mapValidDEPRECATED @25 :Bool; + radarValidDEPRECATED @28 :Bool; + radarCanErrorDEPRECATED @30 :Bool; + commIssueDEPRECATED @31 :Bool; + eventsDEPRECATED @13 :List(Car.CarEvent); + gpsTrajectoryDEPRECATED @12 :GpsTrajectory; + gpsPlannerActiveDEPRECATED @19 :Bool; + personalityDEPRECATED @36 :LongitudinalPersonality; + + struct GpsTrajectory { + x @0 :List(Float32); + y @1 :List(Float32); + } +} +struct UiPlan { + frameId @2 :UInt32; + position @0 :XYZTData; + accel @1 :List(Float32); +} + +struct LateralPlan @0xe1e9318e2ae8b51e { + modelMonoTime @31 :UInt64; + laneWidthDEPRECATED @0 :Float32; + lProbDEPRECATED @5 :Float32; + rProbDEPRECATED @7 :Float32; + dPathPoints @20 :List(Float32); + dProbDEPRECATED @21 :Float32; + + mpcSolutionValid @9 :Bool; + desire @17 :Desire; + laneChangeState @18 :LaneChangeState; + laneChangeDirection @19 :LaneChangeDirection; + useLaneLines @29 :Bool; + + # desired curvatures over next 2.5s in rad/m + psis @26 :List(Float32); + curvatures @27 :List(Float32); + curvatureRates @28 :List(Float32); + + solverExecutionTime @30 :Float32; + solverCost @32 :Float32; + solverState @33 :SolverState; + + struct SolverState { + x @0 :List(List(Float32)); + u @1 :List(Float32); + } + + # deprecated + curvatureDEPRECATED @22 :Float32; + curvatureRateDEPRECATED @23 :Float32; + rawCurvatureDEPRECATED @24 :Float32; + rawCurvatureRateDEPRECATED @25 :Float32; + cProbDEPRECATED @3 :Float32; + dPolyDEPRECATED @1 :List(Float32); + cPolyDEPRECATED @2 :List(Float32); + lPolyDEPRECATED @4 :List(Float32); + rPolyDEPRECATED @6 :List(Float32); + modelValidDEPRECATED @12 :Bool; + commIssueDEPRECATED @15 :Bool; + posenetValidDEPRECATED @16 :Bool; + sensorValidDEPRECATED @14 :Bool; + paramsValidDEPRECATED @10 :Bool; + steeringAngleDegDEPRECATED @8 :Float32; # deg + steeringRateDegDEPRECATED @13 :Float32; # deg/s + angleOffsetDegDEPRECATED @11 :Float32; +} + +struct LiveLocationKalman { + + # More info on reference frames: + # https://github.com/commaai/openpilot/tree/master/common/transformations + + positionECEF @0 : Measurement; + positionGeodetic @1 : Measurement; + velocityECEF @2 : Measurement; + velocityNED @3 : Measurement; + velocityDevice @4 : Measurement; + accelerationDevice @5: Measurement; + + + # These angles are all eulers and roll, pitch, yaw + # orientationECEF transforms to rot matrix: ecef_from_device + orientationECEF @6 : Measurement; + calibratedOrientationECEF @20 : Measurement; + orientationNED @7 : Measurement; + angularVelocityDevice @8 : Measurement; + + # orientationNEDCalibrated transforms to rot matrix: NED_from_calibrated + calibratedOrientationNED @9 : Measurement; + + # Calibrated frame is simply device frame + # aligned with the vehicle + velocityCalibrated @10 : Measurement; + accelerationCalibrated @11 : Measurement; + angularVelocityCalibrated @12 : Measurement; + + gpsWeek @13 :Int32; + gpsTimeOfWeek @14 :Float64; + status @15 :Status; + unixTimestampMillis @16 :Int64; + inputsOK @17 :Bool = true; + posenetOK @18 :Bool = true; + gpsOK @19 :Bool = true; + sensorsOK @21 :Bool = true; + deviceStable @22 :Bool = true; + timeSinceReset @23 :Float64; + excessiveResets @24 :Bool; + timeToFirstFix @25 :Float32; + + filterState @26 : Measurement; + + enum Status { + uninitialized @0; + uncalibrated @1; + valid @2; + } + + struct Measurement { + value @0 : List(Float64); + std @1 : List(Float64); + valid @2 : Bool; + } +} + +struct ProcLog { + cpuTimes @0 :List(CPUTimes); + mem @1 :Mem; + procs @2 :List(Process); + + struct Process { + pid @0 :Int32; + name @1 :Text; + state @2 :UInt8; + ppid @3 :Int32; + + cpuUser @4 :Float32; + cpuSystem @5 :Float32; + cpuChildrenUser @6 :Float32; + cpuChildrenSystem @7 :Float32; + priority @8 :Int64; + nice @9 :Int32; + numThreads @10 :Int32; + startTime @11 :Float64; + + memVms @12 :UInt64; + memRss @13 :UInt64; + + processor @14 :Int32; + + cmdline @15 :List(Text); + exe @16 :Text; + } + + struct CPUTimes { + cpuNum @0 :Int64; + user @1 :Float32; + nice @2 :Float32; + system @3 :Float32; + idle @4 :Float32; + iowait @5 :Float32; + irq @6 :Float32; + softirq @7 :Float32; + } + + struct Mem { + total @0 :UInt64; + free @1 :UInt64; + available @2 :UInt64; + buffers @3 :UInt64; + cached @4 :UInt64; + active @5 :UInt64; + inactive @6 :UInt64; + shared @7 :UInt64; + } +} + +struct GnssMeasurements { + measTime @0 :UInt64; + gpsWeek @1 :Int16; + gpsTimeOfWeek @2 :Float64; + + correctedMeasurements @3 :List(CorrectedMeasurement); + ephemerisStatuses @9 :List(EphemerisStatus); + + kalmanPositionECEF @4 :LiveLocationKalman.Measurement; + kalmanVelocityECEF @5 :LiveLocationKalman.Measurement; + positionECEF @6 :LiveLocationKalman.Measurement; + velocityECEF @7 :LiveLocationKalman.Measurement; + timeToFirstFix @8 :Float32; + # Todo sync this with timing pulse of ublox + + struct EphemerisStatus { + constellationId @0 :ConstellationId; + svId @1 :UInt8; + type @2 :EphemerisType; + source @3 :EphemerisSource; + gpsWeek @4 : UInt16; + tow @5 :Float64; + } + + struct CorrectedMeasurement { + constellationId @0 :ConstellationId; + svId @1 :UInt8; + # Is 0 when not Glonass constellation. + glonassFrequency @2 :Int8; + pseudorange @3 :Float64; + pseudorangeStd @4 :Float64; + pseudorangeRate @5 :Float64; + pseudorangeRateStd @6 :Float64; + # Satellite position and velocity [x,y,z] + satPos @7 :List(Float64); + satVel @8 :List(Float64); + ephemerisSourceDEPRECATED @9 :EphemerisSourceDEPRECATED; + } + + struct EphemerisSourceDEPRECATED { + type @0 :EphemerisType; + # first epoch in file: + gpsWeek @1 :Int16; # -1 if Nav + gpsTimeOfWeek @2 :Int32; # -1 if Nav. Integer for seconds is good enough for logs. + } + + enum ConstellationId { + # Satellite Constellation using the Ublox gnssid as index + gps @0; + sbas @1; + galileo @2; + beidou @3; + imes @4; + qznss @5; + glonass @6; + } + + enum EphemerisType { + nav @0; + # Different ultra-rapid files: + nasaUltraRapid @1; + glonassIacUltraRapid @2; + qcom @3; + } + + enum EphemerisSource { + gnssChip @0; + internet @1; + cache @2; + unknown @3; + } +} + +struct UbloxGnss { + union { + measurementReport @0 :MeasurementReport; + ephemeris @1 :Ephemeris; + ionoData @2 :IonoData; + hwStatus @3 :HwStatus; + hwStatus2 @4 :HwStatus2; + glonassEphemeris @5 :GlonassEphemeris; + satReport @6 :SatReport; + } + + struct SatReport { + #received time of week in gps time in seconds and gps week + iTow @0 :UInt32; + svs @1 :List(SatInfo); + + struct SatInfo { + svId @0 :UInt8; + gnssId @1 :UInt8; + flagsBitfield @2 :UInt32; + } + } + + struct MeasurementReport { + #received time of week in gps time in seconds and gps week + rcvTow @0 :Float64; + gpsWeek @1 :UInt16; + # leap seconds in seconds + leapSeconds @2 :UInt16; + # receiver status + receiverStatus @3 :ReceiverStatus; + # num of measurements to follow + numMeas @4 :UInt8; + measurements @5 :List(Measurement); + + struct ReceiverStatus { + # leap seconds have been determined + leapSecValid @0 :Bool; + # Clock reset applied + clkReset @1 :Bool; + } + + struct Measurement { + svId @0 :UInt8; + trackingStatus @1 :TrackingStatus; + # pseudorange in meters + pseudorange @2 :Float64; + # carrier phase measurement in cycles + carrierCycles @3 :Float64; + # doppler measurement in Hz + doppler @4 :Float32; + # GNSS id, 0 is gps + gnssId @5 :UInt8; + glonassFrequencyIndex @6 :UInt8; + # carrier phase locktime counter in ms + locktime @7 :UInt16; + # Carrier-to-noise density ratio (signal strength) in dBHz + cno @8 :UInt8; + # pseudorange standard deviation in meters + pseudorangeStdev @9 :Float32; + # carrier phase standard deviation in cycles + carrierPhaseStdev @10 :Float32; + # doppler standard deviation in Hz + dopplerStdev @11 :Float32; + sigId @12 :UInt8; + + struct TrackingStatus { + # pseudorange valid + pseudorangeValid @0 :Bool; + # carrier phase valid + carrierPhaseValid @1 :Bool; + # half cycle valid + halfCycleValid @2 :Bool; + # half cycle subtracted from phase + halfCycleSubtracted @3 :Bool; + } + } + } + + struct Ephemeris { + # This is according to the rinex (2?) format + svId @0 :UInt16; + year @1 :UInt16; + month @2 :UInt16; + day @3 :UInt16; + hour @4 :UInt16; + minute @5 :UInt16; + second @6 :Float32; + af0 @7 :Float64; + af1 @8 :Float64; + af2 @9 :Float64; + + iode @10 :Float64; + crs @11 :Float64; + deltaN @12 :Float64; + m0 @13 :Float64; + + cuc @14 :Float64; + ecc @15 :Float64; + cus @16 :Float64; + a @17 :Float64; # note that this is not the root!! + + toe @18 :Float64; + cic @19 :Float64; + omega0 @20 :Float64; + cis @21 :Float64; + + i0 @22 :Float64; + crc @23 :Float64; + omega @24 :Float64; + omegaDot @25 :Float64; + + iDot @26 :Float64; + codesL2 @27 :Float64; + gpsWeekDEPRECATED @28 :Float64; + l2 @29 :Float64; + + svAcc @30 :Float64; + svHealth @31 :Float64; + tgd @32 :Float64; + iodc @33 :Float64; + + transmissionTime @34 :Float64; + fitInterval @35 :Float64; + + toc @36 :Float64; + + ionoCoeffsValid @37 :Bool; + ionoAlpha @38 :List(Float64); + ionoBeta @39 :List(Float64); + + towCount @40 :UInt32; + toeWeek @41 :UInt16; + tocWeek @42 :UInt16; + } + + struct IonoData { + svHealth @0 :UInt32; + tow @1 :Float64; + gpsWeek @2 :Float64; + + ionoAlpha @3 :List(Float64); + ionoBeta @4 :List(Float64); + + healthValid @5 :Bool; + ionoCoeffsValid @6 :Bool; + } + + struct HwStatus { + noisePerMS @0 :UInt16; + agcCnt @1 :UInt16; + aStatus @2 :AntennaSupervisorState; + aPower @3 :AntennaPowerStatus; + jamInd @4 :UInt8; + flags @5 :UInt8; + + enum AntennaSupervisorState { + init @0; + dontknow @1; + ok @2; + short @3; + open @4; + } + + enum AntennaPowerStatus { + off @0; + on @1; + dontknow @2; + } + } + + struct HwStatus2 { + ofsI @0 :Int8; + magI @1 :UInt8; + ofsQ @2 :Int8; + magQ @3 :UInt8; + cfgSource @4 :ConfigSource; + lowLevCfg @5 :UInt32; + postStatus @6 :UInt32; + + enum ConfigSource { + undefined @0; + rom @1; + otp @2; + configpins @3; + flash @4; + } + } + + struct GlonassEphemeris { + svId @0 :UInt16; + year @1 :UInt16; + dayInYear @2 :UInt16; + hour @3 :UInt16; + minute @4 :UInt16; + second @5 :Float32; + + x @6 :Float64; + xVel @7 :Float64; + xAccel @8 :Float64; + y @9 :Float64; + yVel @10 :Float64; + yAccel @11 :Float64; + z @12 :Float64; + zVel @13 :Float64; + zAccel @14 :Float64; + + svType @15 :UInt8; + svURA @16 :Float32; + age @17 :UInt8; + + svHealth @18 :UInt8; + tkDEPRECATED @19 :UInt16; + tb @20 :UInt16; + + tauN @21 :Float64; + deltaTauN @22 :Float64; + gammaN @23 :Float64; + + p1 @24 :UInt8; + p2 @25 :UInt8; + p3 @26 :UInt8; + p4 @27 :UInt8; + + freqNumDEPRECATED @28 :UInt32; + + n4 @29 :UInt8; + nt @30 :UInt16; + freqNum @31 :Int16; + tkSeconds @32 :UInt32; + } +} + +struct QcomGnss @0xde94674b07ae51c1 { + logTs @0 :UInt64; + union { + measurementReport @1 :MeasurementReport; + clockReport @2 :ClockReport; + drMeasurementReport @3 :DrMeasurementReport; + drSvPoly @4 :DrSvPolyReport; + rawLog @5 :Data; + } + + enum MeasurementSource @0xd71a12b6faada7ee { + gps @0; + glonass @1; + beidou @2; + unknown3 @3; + unknown4 @4; + unknown5 @5; + sbas @6; + } + + enum SVObservationState @0xe81e829a0d6c83e9 { + idle @0; + search @1; + searchVerify @2; + bitEdge @3; + trackVerify @4; + track @5; + restart @6; + dpo @7; + glo10msBe @8; + glo10msAt @9; + } + + struct MeasurementStatus @0xe501010e1bcae83b { + subMillisecondIsValid @0 :Bool; + subBitTimeIsKnown @1 :Bool; + satelliteTimeIsKnown @2 :Bool; + bitEdgeConfirmedFromSignal @3 :Bool; + measuredVelocity @4 :Bool; + fineOrCoarseVelocity @5 :Bool; + lockPointValid @6 :Bool; + lockPointPositive @7 :Bool; + lastUpdateFromDifference @8 :Bool; + lastUpdateFromVelocityDifference @9 :Bool; + strongIndicationOfCrossCorelation @10 :Bool; + tentativeMeasurement @11 :Bool; + measurementNotUsable @12 :Bool; + sirCheckIsNeeded @13 :Bool; + probationMode @14 :Bool; + + glonassMeanderBitEdgeValid @15 :Bool; + glonassTimeMarkValid @16 :Bool; + + gpsRoundRobinRxDiversity @17 :Bool; + gpsRxDiversity @18 :Bool; + gpsLowBandwidthRxDiversityCombined @19 :Bool; + gpsHighBandwidthNu4 @20 :Bool; + gpsHighBandwidthNu8 @21 :Bool; + gpsHighBandwidthUniform @22 :Bool; + multipathIndicator @23 :Bool; + + imdJammingIndicator @24 :Bool; + lteB13TxJammingIndicator @25 :Bool; + freshMeasurementIndicator @26 :Bool; + + multipathEstimateIsValid @27 :Bool; + directionIsValid @28 :Bool; + } + + struct MeasurementReport @0xf580d7d86b7b8692 { + source @0 :MeasurementSource; + + fCount @1 :UInt32; + + gpsWeek @2 :UInt16; + glonassCycleNumber @3 :UInt8; + glonassNumberOfDays @4 :UInt16; + + milliseconds @5 :UInt32; + timeBias @6 :Float32; + clockTimeUncertainty @7 :Float32; + clockFrequencyBias @8 :Float32; + clockFrequencyUncertainty @9 :Float32; + + sv @10 :List(SV); + + struct SV @0xf10c595ae7bb2c27 { + svId @0 :UInt8; + observationState @2 :SVObservationState; + observations @3 :UInt8; + goodObservations @4 :UInt8; + gpsParityErrorCount @5 :UInt16; + glonassFrequencyIndex @1 :Int8; + glonassHemmingErrorCount @6 :UInt8; + filterStages @7 :UInt8; + carrierNoise @8 :UInt16; + latency @9 :Int16; + predetectInterval @10 :UInt8; + postdetections @11 :UInt16; + + unfilteredMeasurementIntegral @12 :UInt32; + unfilteredMeasurementFraction @13 :Float32; + unfilteredTimeUncertainty @14 :Float32; + unfilteredSpeed @15 :Float32; + unfilteredSpeedUncertainty @16 :Float32; + measurementStatus @17 :MeasurementStatus; + multipathEstimate @18 :UInt32; + azimuth @19 :Float32; + elevation @20 :Float32; + carrierPhaseCyclesIntegral @21 :Int32; + carrierPhaseCyclesFraction @22 :UInt16; + fineSpeed @23 :Float32; + fineSpeedUncertainty @24 :Float32; + cycleSlipCount @25 :UInt8; + } + + } + + struct ClockReport @0xca965e4add8f4f0b { + hasFCount @0 :Bool; + fCount @1 :UInt32; + + hasGpsWeek @2 :Bool; + gpsWeek @3 :UInt16; + hasGpsMilliseconds @4 :Bool; + gpsMilliseconds @5 :UInt32; + gpsTimeBias @6 :Float32; + gpsClockTimeUncertainty @7 :Float32; + gpsClockSource @8 :UInt8; + + hasGlonassYear @9 :Bool; + glonassYear @10 :UInt8; + hasGlonassDay @11 :Bool; + glonassDay @12 :UInt16; + hasGlonassMilliseconds @13 :Bool; + glonassMilliseconds @14 :UInt32; + glonassTimeBias @15 :Float32; + glonassClockTimeUncertainty @16 :Float32; + glonassClockSource @17 :UInt8; + + bdsWeek @18 :UInt16; + bdsMilliseconds @19 :UInt32; + bdsTimeBias @20 :Float32; + bdsClockTimeUncertainty @21 :Float32; + bdsClockSource @22 :UInt8; + + galWeek @23 :UInt16; + galMilliseconds @24 :UInt32; + galTimeBias @25 :Float32; + galClockTimeUncertainty @26 :Float32; + galClockSource @27 :UInt8; + + clockFrequencyBias @28 :Float32; + clockFrequencyUncertainty @29 :Float32; + frequencySource @30 :UInt8; + gpsLeapSeconds @31 :UInt8; + gpsLeapSecondsUncertainty @32 :UInt8; + gpsLeapSecondsSource @33 :UInt8; + + gpsToGlonassTimeBiasMilliseconds @34 :Float32; + gpsToGlonassTimeBiasMillisecondsUncertainty @35 :Float32; + gpsToBdsTimeBiasMilliseconds @36 :Float32; + gpsToBdsTimeBiasMillisecondsUncertainty @37 :Float32; + bdsToGloTimeBiasMilliseconds @38 :Float32; + bdsToGloTimeBiasMillisecondsUncertainty @39 :Float32; + gpsToGalTimeBiasMilliseconds @40 :Float32; + gpsToGalTimeBiasMillisecondsUncertainty @41 :Float32; + galToGloTimeBiasMilliseconds @42 :Float32; + galToGloTimeBiasMillisecondsUncertainty @43 :Float32; + galToBdsTimeBiasMilliseconds @44 :Float32; + galToBdsTimeBiasMillisecondsUncertainty @45 :Float32; + + hasRtcTime @46 :Bool; + systemRtcTime @47 :UInt32; + fCountOffset @48 :UInt32; + lpmRtcCount @49 :UInt32; + clockResets @50 :UInt32; + } + + struct DrMeasurementReport @0x8053c39445c6c75c { + + reason @0 :UInt8; + seqNum @1 :UInt8; + seqMax @2 :UInt8; + rfLoss @3 :UInt16; + + systemRtcValid @4 :Bool; + fCount @5 :UInt32; + clockResets @6 :UInt32; + systemRtcTime @7 :UInt64; + + gpsLeapSeconds @8 :UInt8; + gpsLeapSecondsUncertainty @9 :UInt8; + gpsToGlonassTimeBiasMilliseconds @10 :Float32; + gpsToGlonassTimeBiasMillisecondsUncertainty @11 :Float32; + + gpsWeek @12 :UInt16; + gpsMilliseconds @13 :UInt32; + gpsTimeBiasMs @14 :UInt32; + gpsClockTimeUncertaintyMs @15 :UInt32; + gpsClockSource @16 :UInt8; + + glonassClockSource @17 :UInt8; + glonassYear @18 :UInt8; + glonassDay @19 :UInt16; + glonassMilliseconds @20 :UInt32; + glonassTimeBias @21 :Float32; + glonassClockTimeUncertainty @22 :Float32; + + clockFrequencyBias @23 :Float32; + clockFrequencyUncertainty @24 :Float32; + frequencySource @25 :UInt8; + + source @26 :MeasurementSource; + + sv @27 :List(SV); + + struct SV @0xf08b81df8cbf459c { + svId @0 :UInt8; + glonassFrequencyIndex @1 :Int8; + observationState @2 :SVObservationState; + observations @3 :UInt8; + goodObservations @4 :UInt8; + filterStages @5 :UInt8; + predetectInterval @6 :UInt8; + cycleSlipCount @7 :UInt8; + postdetections @8 :UInt16; + + measurementStatus @9 :MeasurementStatus; + + carrierNoise @10 :UInt16; + rfLoss @11 :UInt16; + latency @12 :Int16; + + filteredMeasurementFraction @13 :Float32; + filteredMeasurementIntegral @14 :UInt32; + filteredTimeUncertainty @15 :Float32; + filteredSpeed @16 :Float32; + filteredSpeedUncertainty @17 :Float32; + + unfilteredMeasurementFraction @18 :Float32; + unfilteredMeasurementIntegral @19 :UInt32; + unfilteredTimeUncertainty @20 :Float32; + unfilteredSpeed @21 :Float32; + unfilteredSpeedUncertainty @22 :Float32; + + multipathEstimate @23 :UInt32; + azimuth @24 :Float32; + elevation @25 :Float32; + dopplerAcceleration @26 :Float32; + fineSpeed @27 :Float32; + fineSpeedUncertainty @28 :Float32; + + carrierPhase @29 :Float64; + fCount @30 :UInt32; + + parityErrorCount @31 :UInt16; + goodParity @32 :Bool; + } + } + + struct DrSvPolyReport @0xb1fb80811a673270 { + svId @0 :UInt16; + frequencyIndex @1 :Int8; + + hasPosition @2 :Bool; + hasIono @3 :Bool; + hasTropo @4 :Bool; + hasElevation @5 :Bool; + polyFromXtra @6 :Bool; + hasSbasIono @7 :Bool; + + iode @8 :UInt16; + t0 @9 :Float64; + xyz0 @10 :List(Float64); + xyzN @11 :List(Float64); + other @12 :List(Float32); + + positionUncertainty @13 :Float32; + ionoDelay @14 :Float32; + ionoDot @15 :Float32; + sbasIonoDelay @16 :Float32; + sbasIonoDot @17 :Float32; + tropoDelay @18 :Float32; + elevation @19 :Float32; + elevationDot @20 :Float32; + elevationUncertainty @21 :Float32; + velocityCoeff @22 :List(Float64); + + gpsWeek @23 :UInt16; + gpsTow @24 :Float64; + } +} + +struct Clocks { + wallTimeNanos @3 :UInt64; # unix epoch time + + bootTimeNanosDEPRECATED @0 :UInt64; + monotonicNanosDEPRECATED @1 :UInt64; + monotonicRawNanosDEPRECATD @2 :UInt64; + modemUptimeMillisDEPRECATED @4 :UInt64; +} + +struct LiveMpcData { + x @0 :List(Float32); + y @1 :List(Float32); + psi @2 :List(Float32); + curvature @3 :List(Float32); + qpIterations @4 :UInt32; + calculationTime @5 :UInt64; + cost @6 :Float64; +} + +struct LiveLongitudinalMpcData { + xEgo @0 :List(Float32); + vEgo @1 :List(Float32); + aEgo @2 :List(Float32); + xLead @3 :List(Float32); + vLead @4 :List(Float32); + aLead @5 :List(Float32); + aLeadTau @6 :Float32; # lead accel time constant + qpIterations @7 :UInt32; + mpcId @8 :UInt32; + calculationTime @9 :UInt64; + cost @10 :Float64; +} + +struct Joystick { + # convenient for debug and live tuning + axes @0: List(Float32); + buttons @1: List(Bool); +} + +struct DriverStateV2 { + frameId @0 :UInt32; + modelExecutionTime @1 :Float32; + dspExecutionTime @2 :Float32; + rawPredictions @3 :Data; + + poorVisionProb @4 :Float32; + wheelOnRightProb @5 :Float32; + + leftDriverData @6 :DriverData; + rightDriverData @7 :DriverData; + + struct DriverData { + faceOrientation @0 :List(Float32); + faceOrientationStd @1 :List(Float32); + facePosition @2 :List(Float32); + facePositionStd @3 :List(Float32); + faceProb @4 :Float32; + leftEyeProb @5 :Float32; + rightEyeProb @6 :Float32; + leftBlinkProb @7 :Float32; + rightBlinkProb @8 :Float32; + sunglassesProb @9 :Float32; + occludedProb @10 :Float32; + readyProb @11 :List(Float32); + notReadyProb @12 :List(Float32); + } +} + +struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 { + frameId @0 :UInt32; + modelExecutionTime @14 :Float32; + dspExecutionTime @16 :Float32; + rawPredictions @15 :Data; + + faceOrientation @3 :List(Float32); + facePosition @4 :List(Float32); + faceProb @5 :Float32; + leftEyeProb @6 :Float32; + rightEyeProb @7 :Float32; + leftBlinkProb @8 :Float32; + rightBlinkProb @9 :Float32; + faceOrientationStd @11 :List(Float32); + facePositionStd @12 :List(Float32); + sunglassesProb @13 :Float32; + poorVision @17 :Float32; + partialFace @18 :Float32; + distractedPose @19 :Float32; + distractedEyes @20 :Float32; + eyesOnRoad @21 :Float32; + phoneUse @22 :Float32; + occludedProb @23 :Float32; + + readyProb @24 :List(Float32); + notReadyProb @25 :List(Float32); + + irPwrDEPRECATED @10 :Float32; + descriptorDEPRECATED @1 :List(Float32); + stdDEPRECATED @2 :Float32; +} + +struct DriverMonitoringState @0xb83cda094a1da284 { + events @0 :List(Car.CarEvent); + faceDetected @1 :Bool; + isDistracted @2 :Bool; + distractedType @17 :UInt32; + awarenessStatus @3 :Float32; + posePitchOffset @6 :Float32; + posePitchValidCount @7 :UInt32; + poseYawOffset @8 :Float32; + poseYawValidCount @9 :UInt32; + stepChange @10 :Float32; + awarenessActive @11 :Float32; + awarenessPassive @12 :Float32; + isLowStd @13 :Bool; + hiStdCount @14 :UInt32; + isActiveMode @16 :Bool; + isRHD @4 :Bool; + + isPreviewDEPRECATED @15 :Bool; + rhdCheckedDEPRECATED @5 :Bool; +} + +struct Boot { + wallTimeNanos @0 :UInt64; + pstore @4 :Map(Text, Data); + commands @5 :Map(Text, Data); + launchLog @3 :Text; + + lastKmsgDEPRECATED @1 :Data; + lastPmsgDEPRECATED @2 :Data; +} + +struct LiveParametersData { + valid @0 :Bool; + gyroBias @1 :Float32; + angleOffsetDeg @2 :Float32; + angleOffsetAverageDeg @3 :Float32; + stiffnessFactor @4 :Float32; + steerRatio @5 :Float32; + sensorValid @6 :Bool; + posenetSpeed @8 :Float32; + posenetValid @9 :Bool; + angleOffsetFastStd @10 :Float32; + angleOffsetAverageStd @11 :Float32; + stiffnessFactorStd @12 :Float32; + steerRatioStd @13 :Float32; + roll @14 :Float32; + filterState @15 :LiveLocationKalman.Measurement; + + yawRateDEPRECATED @7 :Float32; +} + +struct LiveTorqueParametersData { + liveValid @0 :Bool; + latAccelFactorRaw @1 :Float32; + latAccelOffsetRaw @2 :Float32; + frictionCoefficientRaw @3 :Float32; + latAccelFactorFiltered @4 :Float32; + latAccelOffsetFiltered @5 :Float32; + frictionCoefficientFiltered @6 :Float32; + totalBucketPoints @7 :Float32; + decay @8 :Float32; + maxResets @9 :Float32; + points @10 :List(List(Float32)); + version @11 :Int32; + useParams @12 :Bool; +} + +struct LiveMapDataDEPRECATED { + speedLimitValid @0 :Bool; + speedLimit @1 :Float32; + speedAdvisoryValid @12 :Bool; + speedAdvisory @13 :Float32; + speedLimitAheadValid @14 :Bool; + speedLimitAhead @15 :Float32; + speedLimitAheadDistance @16 :Float32; + curvatureValid @2 :Bool; + curvature @3 :Float32; + wayId @4 :UInt64; + roadX @5 :List(Float32); + roadY @6 :List(Float32); + lastGps @7: GpsLocationData; + roadCurvatureX @8 :List(Float32); + roadCurvature @9 :List(Float32); + distToTurn @10 :Float32; + mapValid @11 :Bool; +} + +struct CameraOdometry { + frameId @4 :UInt32; + timestampEof @5 :UInt64; + trans @0 :List(Float32); # m/s in device frame + rot @1 :List(Float32); # rad/s in device frame + transStd @2 :List(Float32); # std m/s in device frame + rotStd @3 :List(Float32); # std rad/s in device frame + wideFromDeviceEuler @6 :List(Float32); + wideFromDeviceEulerStd @7 :List(Float32); + roadTransformTrans @8 :List(Float32); + roadTransformTransStd @9 :List(Float32); +} + +struct Sentinel { + enum SentinelType { + endOfSegment @0; + endOfRoute @1; + startOfSegment @2; + startOfRoute @3; + } + type @0 :SentinelType; + signal @1 :Int32; +} + +struct UIDebug { + drawTimeMillis @0 :Float32; +} + +struct ManagerState { + processes @0 :List(ProcessState); + + struct ProcessState { + name @0 :Text; + pid @1 :Int32; + running @2 :Bool; + shouldBeRunning @4 :Bool; + exitCode @3 :Int32; + } +} + +struct UploaderState { + immediateQueueSize @0 :UInt32; + immediateQueueCount @1 :UInt32; + rawQueueSize @2 :UInt32; + rawQueueCount @3 :UInt32; + + # stats for last successfully uploaded file + lastTime @4 :Float32; # s + lastSpeed @5 :Float32; # MB/s + lastFilename @6 :Text; +} + +struct NavInstruction { + maneuverPrimaryText @0 :Text; + maneuverSecondaryText @1 :Text; + maneuverDistance @2 :Float32; # m + maneuverType @3 :Text; # TODO: Make Enum + maneuverModifier @4 :Text; # TODO: Make Enum + + distanceRemaining @5 :Float32; # m + timeRemaining @6 :Float32; # s + timeRemainingTypical @7 :Float32; # s + + lanes @8 :List(Lane); + showFull @9 :Bool; + + speedLimit @10 :Float32; # m/s + speedLimitSign @11 :SpeedLimitSign; + + allManeuvers @12 :List(Maneuver); + + struct Lane { + directions @0 :List(Direction); + active @1 :Bool; + activeDirection @2 :Direction; + } + + enum Direction { + none @0; + left @1; + right @2; + straight @3; + slightLeft @4; + slightRight @5; + } + + enum SpeedLimitSign { + mutcd @0; # US Style + vienna @1; # EU Style + } + + struct Maneuver { + distance @0 :Float32; + type @1 :Text; + modifier @2 :Text; + } +} + +struct NavRoute { + coordinates @0 :List(Coordinate); + + struct Coordinate { + latitude @0 :Float32; + longitude @1 :Float32; + } +} + +struct MapRenderState { + locationMonoTime @0 :UInt64; + renderTime @1 :Float32; + frameId @2: UInt32; +} + +struct NavModelData { + frameId @0 :UInt32; + locationMonoTime @6 :UInt64; + modelExecutionTime @1 :Float32; + dspExecutionTime @2 :Float32; + features @3 :List(Float32); + # predicted future position + position @4 :XYData; + desirePrediction @5 :List(Float32); + + # All SI units and in device frame + struct XYData { + x @0 :List(Float32); + y @1 :List(Float32); + xStd @2 :List(Float32); + yStd @3 :List(Float32); + } +} + +struct EncodeData { + idx @0 :EncodeIndex; + data @1 :Data; + header @2 :Data; + unixTimestampNanos @3 :UInt64; + width @4 :UInt32; + height @5 :UInt32; +} + +struct UserFlag { +} + +struct Microphone { + soundPressure @0 :Float32; + + # uncalibrated, A-weighted + soundPressureWeighted @3 :Float32; + soundPressureWeightedDb @1 :Float32; + filteredSoundPressureWeightedDb @2 :Float32; +} + +struct Event { + logMonoTime @0 :UInt64; # nanoseconds + valid @67 :Bool = true; + + union { + # *********** log metadata *********** + initData @1 :InitData; + sentinel @73 :Sentinel; + + # *********** bootlog *********** + boot @60 :Boot; + + # ********** openpilot daemon msgs ********** + gpsNMEA @3 :GPSNMEAData; + can @5 :List(CanData); + controlsState @7 :ControlsState; + gyroscope @99 :SensorEventData; + gyroscope2 @100 :SensorEventData; + accelerometer @98 :SensorEventData; + accelerometer2 @101 :SensorEventData; + magnetometer @95 :SensorEventData; + lightSensor @96 :SensorEventData; + temperatureSensor @97 :SensorEventData; + temperatureSensor2 @123 :SensorEventData; + pandaStates @81 :List(PandaState); + peripheralState @80 :PeripheralState; + radarState @13 :RadarState; + liveTracks @16 :List(LiveTracks); + sendcan @17 :List(CanData); + liveCalibration @19 :LiveCalibrationData; + carState @22 :Car.CarState; + carControl @23 :Car.CarControl; + carOutput @127 :Car.CarOutput; + longitudinalPlan @24 :LongitudinalPlan; + uiPlan @106 :UiPlan; + ubloxGnss @34 :UbloxGnss; + ubloxRaw @39 :Data; + qcomGnss @31 :QcomGnss; + gpsLocationExternal @48 :GpsLocationData; + gpsLocation @21 :GpsLocationData; + gnssMeasurements @91 :GnssMeasurements; + liveParameters @61 :LiveParametersData; + liveTorqueParameters @94 :LiveTorqueParametersData; + cameraOdometry @63 :CameraOdometry; + thumbnail @66: Thumbnail; + onroadEvents @68: List(Car.CarEvent); + carParams @69: Car.CarParams; + driverMonitoringState @71: DriverMonitoringState; + liveLocationKalman @72 :LiveLocationKalman; + modelV2 @75 :ModelDataV2; + driverStateV2 @92 :DriverStateV2; + + # camera stuff, each camera state has a matching encode idx + roadCameraState @2 :FrameData; + driverCameraState @70: FrameData; + wideRoadCameraState @74: FrameData; + roadEncodeIdx @15 :EncodeIndex; + driverEncodeIdx @76 :EncodeIndex; + wideRoadEncodeIdx @77 :EncodeIndex; + qRoadEncodeIdx @90 :EncodeIndex; + + livestreamRoadEncodeIdx @117 :EncodeIndex; + livestreamWideRoadEncodeIdx @118 :EncodeIndex; + livestreamDriverEncodeIdx @119 :EncodeIndex; + + # microphone data + microphone @103 :Microphone; + + # systems stuff + androidLog @20 :AndroidLogEntry; + managerState @78 :ManagerState; + uploaderState @79 :UploaderState; + procLog @33 :ProcLog; + clocks @35 :Clocks; + deviceState @6 :DeviceState; + logMessage @18 :Text; + errorLogMessage @85 :Text; + + # navigation + navInstruction @82 :NavInstruction; + navRoute @83 :NavRoute; + navThumbnail @84: Thumbnail; + mapRenderState @105: MapRenderState; + + # UI services + userFlag @93 :UserFlag; + uiDebug @102 :UIDebug; + + # *********** debug *********** + testJoystick @52 :Joystick; + roadEncodeData @86 :EncodeData; + driverEncodeData @87 :EncodeData; + wideRoadEncodeData @88 :EncodeData; + qRoadEncodeData @89 :EncodeData; + + livestreamRoadEncodeData @120 :EncodeData; + livestreamWideRoadEncodeData @121 :EncodeData; + livestreamDriverEncodeData @122 :EncodeData; + + customReservedRawData0 @124 :Data; + customReservedRawData1 @125 :Data; + customReservedRawData2 @126 :Data; + + # *********** Custom: reserved for forks *********** + customReserved0 @107 :Custom.CustomReserved0; + customReserved1 @108 :Custom.CustomReserved1; + customReserved2 @109 :Custom.CustomReserved2; + customReserved3 @110 :Custom.CustomReserved3; + customReserved4 @111 :Custom.CustomReserved4; + customReserved5 @112 :Custom.CustomReserved5; + customReserved6 @113 :Custom.CustomReserved6; + customReserved7 @114 :Custom.CustomReserved7; + customReserved8 @115 :Custom.CustomReserved8; + customReserved9 @116 :Custom.CustomReserved9; + + # *********** legacy + deprecated *********** + model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated + liveMpcDEPRECATED @36 :LiveMpcData; + liveLongitudinalMpcDEPRECATED @37 :LiveLongitudinalMpcData; + liveLocationKalmanDEPRECATED @51 :Legacy.LiveLocationData; + orbslamCorrectionDEPRECATED @45 :Legacy.OrbslamCorrection; + liveUIDEPRECATED @14 :Legacy.LiveUI; + sensorEventDEPRECATED @4 :SensorEventData; + liveEventDEPRECATED @8 :List(Legacy.LiveEventData); + liveLocationDEPRECATED @25 :Legacy.LiveLocationData; + ethernetDataDEPRECATED @26 :List(Legacy.EthernetPacket); + cellInfoDEPRECATED @28 :List(Legacy.CellInfo); + wifiScanDEPRECATED @29 :List(Legacy.WifiScan); + uiNavigationEventDEPRECATED @50 :Legacy.UiNavigationEvent; + liveMapDataDEPRECATED @62 :LiveMapDataDEPRECATED; + gpsPlannerPointsDEPRECATED @40 :Legacy.GPSPlannerPoints; + gpsPlannerPlanDEPRECATED @41 :Legacy.GPSPlannerPlan; + applanixRawDEPRECATED @42 :Data; + androidGnssDEPRECATED @30 :Legacy.AndroidGnss; + lidarPtsDEPRECATED @32 :Legacy.LidarPts; + navStatusDEPRECATED @38 :Legacy.NavStatus; + trafficEventsDEPRECATED @43 :List(Legacy.TrafficEvent); + liveLocationTimingDEPRECATED @44 :Legacy.LiveLocationData; + liveLocationCorrectedDEPRECATED @46 :Legacy.LiveLocationData; + navUpdateDEPRECATED @27 :Legacy.NavUpdate; + orbObservationDEPRECATED @47 :List(Legacy.OrbObservation); + locationDEPRECATED @49 :Legacy.LiveLocationData; + orbOdometryDEPRECATED @53 :Legacy.OrbOdometry; + orbFeaturesDEPRECATED @54 :Legacy.OrbFeatures; + applanixLocationDEPRECATED @55 :Legacy.LiveLocationData; + orbKeyFrameDEPRECATED @56 :Legacy.OrbKeyFrame; + orbFeaturesSummaryDEPRECATED @58 :Legacy.OrbFeaturesSummary; + featuresDEPRECATED @10 :Legacy.CalibrationFeatures; + kalmanOdometryDEPRECATED @65 :Legacy.KalmanOdometry; + uiLayoutStateDEPRECATED @57 :Legacy.UiLayoutState; + pandaStateDEPRECATED @12 :PandaState; + driverStateDEPRECATED @59 :DriverStateDEPRECATED; + sensorEventsDEPRECATED @11 :List(SensorEventData); + lateralPlanDEPRECATED @64 :LateralPlan; + navModelDEPRECATED @104 :NavModelData; + } +} diff --git a/cereal/maptile.capnp b/cereal/maptile.capnp new file mode 100644 index 00000000000000..c8a23a1822b808 --- /dev/null +++ b/cereal/maptile.capnp @@ -0,0 +1,49 @@ +using Cxx = import "./include/c++.capnp"; +$Cxx.namespace("cereal"); + +@0xa086df597ef5d7a0; + +# Geometry +struct Point { + x @0: Float64; + y @1: Float64; + z @2: Float64; +} + +struct PolyLine { + points @0: List(Point); +} + +# Map features +struct Lane { + id @0 :Text; + + leftBoundary @1 :LaneBoundary; + rightBoundary @2 :LaneBoundary; + + leftAdjacentId @3 :Text; + rightAdjacentId @4 :Text; + + inboundIds @5 :List(Text); + outboundIds @6 :List(Text); + + struct LaneBoundary { + polyLine @0 :PolyLine; + startHeading @1 :Float32; # WRT north + } +} + +# Map tiles +struct TileSummary { + version @0 :Text; + updatedAt @1 :UInt64; # Millis since epoch + + level @2 :UInt8; + x @3 :UInt16; + y @4 :UInt16; +} + +struct MapTile { + summary @0 :TileSummary; + lanes @1 :List(Lane); +} diff --git a/cereal/messaging/.gitignore b/cereal/messaging/.gitignore new file mode 100644 index 00000000000000..dbbe8e22aee488 --- /dev/null +++ b/cereal/messaging/.gitignore @@ -0,0 +1,10 @@ +demo +bridge +test_runner +*.o +*.os +*.d +*.a +*.so +messaging_pyx.cpp +build/ diff --git a/cereal/messaging/__init__.py b/cereal/messaging/__init__.py new file mode 100644 index 00000000000000..d8115d18189063 --- /dev/null +++ b/cereal/messaging/__init__.py @@ -0,0 +1,306 @@ +# must be built with scons +from msgq.messaging.messaging_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ + set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event +from msgq.messaging.messaging_pyx import MultiplePublishersError, MessagingError + +import os +import capnp +import time + +from typing import Optional, List, Union, Dict, Deque +from collections import deque + +from cereal import log +from cereal.services import SERVICE_LIST + +assert MultiplePublishersError +assert MessagingError +assert toggle_fake_events +assert set_fake_prefix +assert get_fake_prefix +assert delete_fake_prefix +assert wait_for_one_event + +NO_TRAVERSAL_LIMIT = 2**64-1 + +context = Context() + + +def fake_event_handle(endpoint: str, identifier: Optional[str] = None, override: bool = True, enable: bool = False) -> SocketEventHandle: + identifier = identifier or get_fake_prefix() + handle = SocketEventHandle(endpoint, identifier, override) + if override: + handle.enabled = enable + + return handle + + +def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader: + with log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg: + return msg + + +def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder: + args = { + 'valid': False, + 'logMonoTime': int(time.monotonic() * 1e9), + **kwargs + } + dat = log.Event.new_message(**args) + if service is not None: + if size is None: + dat.init(service) + else: + dat.init(service, size) + return dat + + +def pub_sock(endpoint: str) -> PubSocket: + sock = PubSocket() + sock.connect(context, endpoint) + return sock + + +def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1", + conflate: bool = False, timeout: Optional[int] = None) -> SubSocket: + sock = SubSocket() + sock.connect(context, endpoint, addr.encode('utf8'), conflate) + + if timeout is not None: + sock.setTimeout(timeout) + + if poller is not None: + poller.registerSocket(sock) + return sock + + +def drain_sock_raw(sock: SubSocket, wait_for_one: bool = False) -> List[bytes]: + """Receive all message currently available on the queue""" + ret: List[bytes] = [] + while 1: + if wait_for_one and len(ret) == 0: + dat = sock.receive() + else: + dat = sock.receive(non_blocking=True) + + if dat is None: + break + + ret.append(dat) + + return ret + + +def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]: + """Receive all message currently available on the queue""" + msgs = drain_sock_raw(sock, wait_for_one=wait_for_one) + return [log_from_bytes(m) for m in msgs] + + +# TODO: print when we drop packets? +def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]: + """Same as drain sock, but only returns latest message. Consider using conflate instead.""" + dat = None + + while 1: + if wait and dat is None: + recv = sock.receive() + else: + recv = sock.receive(non_blocking=True) + + if recv is None: # Timeout hit + break + + dat = recv + + if dat is not None: + dat = log_from_bytes(dat) + + return dat + + +def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]: + dat = sock.receive() + if dat is not None: + dat = log_from_bytes(dat) + return dat + + +def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]: + dat = sock.receive(non_blocking=True) + if dat is not None: + dat = log_from_bytes(dat) + return dat + + +def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader: + """Keep receiving until we get a message""" + while True: + dat = sock.receive() + if dat is not None: + return log_from_bytes(dat) + + +class SubMaster: + def __init__(self, services: List[str], poll: Optional[str] = None, + ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None, + ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None): + self.frame = -1 + self.seen = {s: False for s in services} + self.updated = {s: False for s in services} + self.recv_time = {s: 0. for s in services} + self.recv_frame = {s: 0 for s in services} + self.alive = {s: False for s in services} + self.freq_ok = {s: False for s in services} + self.recv_dts: Dict[str, Deque[float]] = {} + self.sock = {} + self.data = {} + self.valid = {} + self.logMonoTime = {} + + self.max_freq = {} + self.min_freq = {} + + self.poller = Poller() + polled_services = set([poll, ] if poll is not None else services) + self.non_polled_services = set(services) - polled_services + + self.ignore_average_freq = [] if ignore_avg_freq is None else ignore_avg_freq + self.ignore_alive = [] if ignore_alive is None else ignore_alive + self.ignore_valid = [] if ignore_valid is None else ignore_valid + + self.simulation = bool(int(os.getenv("SIMULATION", "0"))) + + # if freq and poll aren't specified, assume the max to be conservative + assert frequency is None or poll is None, "Do not specify 'frequency' - frequency of the polled service will be used." + self.update_freq = frequency or max([SERVICE_LIST[s].frequency for s in polled_services]) + + for s in services: + p = self.poller if s not in self.non_polled_services else None + self.sock[s] = sub_sock(s, poller=p, addr=addr, conflate=True) + + try: + data = new_message(s) + except capnp.lib.capnp.KjException: + data = new_message(s, 0) # lists + + self.data[s] = getattr(data.as_reader(), s) + self.logMonoTime[s] = 0 + self.valid[s] = True # FIXME: this should default to False + + freq = max(min([SERVICE_LIST[s].frequency, self.update_freq]), 1.) + if s == poll: + max_freq = freq + min_freq = freq + else: + max_freq = min(freq, self.update_freq) + if SERVICE_LIST[s].frequency >= 2*self.update_freq: + min_freq = self.update_freq + elif self.update_freq >= 2*SERVICE_LIST[s].frequency: + min_freq = freq + else: + min_freq = min(freq, freq / 2.) + self.max_freq[s] = max_freq*1.2 + self.min_freq[s] = min_freq*0.8 + self.recv_dts[s] = deque(maxlen=int(10*freq)) + + def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader: + return self.data[s] + + def _check_avg_freq(self, s: str) -> bool: + return SERVICE_LIST[s].frequency > 0.99 and (s not in self.ignore_average_freq) and (s not in self.ignore_alive) + + def update(self, timeout: int = 100) -> None: + msgs = [] + for sock in self.poller.poll(timeout): + msgs.append(recv_one_or_none(sock)) + + # non-blocking receive for non-polled sockets + for s in self.non_polled_services: + msgs.append(recv_one_or_none(self.sock[s])) + self.update_msgs(time.monotonic(), msgs) + + def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None: + self.frame += 1 + self.updated = dict.fromkeys(self.updated, False) + for msg in msgs: + if msg is None: + continue + + s = msg.which() + self.seen[s] = True + self.updated[s] = True + + if self.recv_time[s] > 1e-5: + self.recv_dts[s].append(cur_time - self.recv_time[s]) + self.recv_time[s] = cur_time + self.recv_frame[s] = self.frame + self.data[s] = getattr(msg, s) + self.logMonoTime[s] = msg.logMonoTime + self.valid[s] = msg.valid + + for s in self.data: + if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation: + # alive if delay is within 10x the expected frequency + self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) + + # check average frequency; slow to fall, quick to recover + dts = self.recv_dts[s] + assert dts.maxlen is not None + recent_dts = list(dts)[-int(dts.maxlen / 10):] + try: + avg_freq = 1 / (sum(dts) / len(dts)) + avg_freq_recent = 1 / (sum(recent_dts) / len(recent_dts)) + except ZeroDivisionError: + avg_freq = 0 + avg_freq_recent = 0 + + avg_freq_ok = self.min_freq[s] <= avg_freq <= self.max_freq[s] + recent_freq_ok = self.min_freq[s] <= avg_freq_recent <= self.max_freq[s] + self.freq_ok[s] = avg_freq_ok or recent_freq_ok + else: + self.freq_ok[s] = True + if self.simulation: + self.alive[s] = self.seen[s] # alive is defined as seen when simulation flag set + else: + self.alive[s] = True + + def all_alive(self, service_list: Optional[List[str]] = None) -> bool: + if service_list is None: + service_list = list(self.sock.keys()) + return all(self.alive[s] for s in service_list if s not in self.ignore_alive) + + def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool: + if service_list is None: + service_list = list(self.sock.keys()) + return all(self.freq_ok[s] for s in service_list if self._check_avg_freq(s)) + + def all_valid(self, service_list: Optional[List[str]] = None) -> bool: + if service_list is None: + service_list = list(self.sock.keys()) + return all(self.valid[s] for s in service_list if s not in self.ignore_valid) + + def all_checks(self, service_list: Optional[List[str]] = None) -> bool: + return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list) + + +class PubMaster: + def __init__(self, services: List[str]): + self.sock = {} + for s in services: + self.sock[s] = pub_sock(s) + + def send(self, s: str, dat: Union[bytes, capnp.lib.capnp._DynamicStructBuilder]) -> None: + if not isinstance(dat, bytes): + dat = dat.to_bytes() + self.sock[s].send(dat) + + def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool: + for _ in range(int(timeout*(1./dt))): + if self.sock[s].all_readers_updated(): + return True + time.sleep(dt) + return False + + def all_readers_updated(self, s: str) -> bool: + return self.sock[s].all_readers_updated() # type: ignore diff --git a/cereal/messaging/bridge.cc b/cereal/messaging/bridge.cc new file mode 100644 index 00000000000000..b16548314b5148 --- /dev/null +++ b/cereal/messaging/bridge.cc @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include +#include + +typedef void (*sighandler_t)(int sig); + +#include "cereal/services.h" +#include "msgq/messaging/impl_msgq.h" +#include "msgq/messaging/impl_zmq.h" + +std::atomic do_exit = false; +static void set_do_exit(int sig) { + do_exit = true; +} + +void sigpipe_handler(int sig) { + assert(sig == SIGPIPE); + std::cout << "SIGPIPE received" << std::endl; +} + +static std::vector get_services(std::string whitelist_str, bool zmq_to_msgq) { + std::vector service_list; + for (const auto& it : services) { + std::string name = it.second.name; + bool in_whitelist = whitelist_str.find(name) != std::string::npos; + if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) { + continue; + } + service_list.push_back(name); + } + return service_list; +} + +int main(int argc, char** argv) { + signal(SIGPIPE, (sighandler_t)sigpipe_handler); + signal(SIGINT, (sighandler_t)set_do_exit); + signal(SIGTERM, (sighandler_t)set_do_exit); + + bool zmq_to_msgq = argc > 2; + std::string ip = zmq_to_msgq ? argv[1] : "127.0.0.1"; + std::string whitelist_str = zmq_to_msgq ? std::string(argv[2]) : ""; + + Poller *poller; + Context *pub_context; + Context *sub_context; + if (zmq_to_msgq) { // republishes zmq debugging messages as msgq + poller = new ZMQPoller(); + pub_context = new MSGQContext(); + sub_context = new ZMQContext(); + } else { + poller = new MSGQPoller(); + pub_context = new ZMQContext(); + sub_context = new MSGQContext(); + } + + std::map sub2pub; + for (auto endpoint : get_services(whitelist_str, zmq_to_msgq)) { + PubSocket * pub_sock; + SubSocket * sub_sock; + if (zmq_to_msgq) { + pub_sock = new MSGQPubSocket(); + sub_sock = new ZMQSubSocket(); + } else { + pub_sock = new ZMQPubSocket(); + sub_sock = new MSGQSubSocket(); + } + pub_sock->connect(pub_context, endpoint); + sub_sock->connect(sub_context, endpoint, ip, false); + + poller->registerSocket(sub_sock); + sub2pub[sub_sock] = pub_sock; + } + + while (!do_exit) { + for (auto sub_sock : poller->poll(100)) { + Message * msg = sub_sock->receive(); + if (msg == NULL) continue; + int ret; + do { + ret = sub2pub[sub_sock]->sendMessage(msg); + } while (ret == -1 && errno == EINTR && !do_exit); + assert(ret >= 0 || do_exit); + delete msg; + + if (do_exit) break; + } + } + return 0; +} diff --git a/cereal/messaging/demo.cc b/cereal/messaging/demo.cc new file mode 100644 index 00000000000000..5b4d24468279d3 --- /dev/null +++ b/cereal/messaging/demo.cc @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include + +#include "cereal/messaging/messaging.h" +#include "cereal/messaging/impl_zmq.h" + +#define MSGS 1e5 + +int main() { + Context * c = Context::create(); + SubSocket * sub_sock = SubSocket::create(c, "controlsState"); + PubSocket * pub_sock = PubSocket::create(c, "controlsState"); + + char data[8]; + + Poller * poller = Poller::create({sub_sock}); + + auto start = std::chrono::steady_clock::now(); + + for (uint64_t i = 0; i < MSGS; i++){ + *(uint64_t*)data = i; + pub_sock->send(data, 8); + + auto r = poller->poll(100); + + for (auto p : r){ + Message * m = p->receive(); + uint64_t ii = *(uint64_t*)m->getData(); + assert(i == ii); + delete m; + } + } + + + auto end = std::chrono::steady_clock::now(); + double elapsed = std::chrono::duration_cast(end - start).count() / 1e9; + double throughput = ((double) MSGS / (double) elapsed); + std::cout << throughput << " msg/s" << std::endl; + + delete poller; + delete sub_sock; + delete pub_sock; + delete c; + + + return 0; +} diff --git a/cereal/messaging/demo.py b/cereal/messaging/demo.py new file mode 100644 index 00000000000000..e4850e32b9cb9c --- /dev/null +++ b/cereal/messaging/demo.py @@ -0,0 +1,29 @@ +import time + +from messaging_pyx import Context, Poller, SubSocket, PubSocket + +MSGS = 1e5 + +if __name__ == "__main__": + c = Context() + sub_sock = SubSocket() + pub_sock = PubSocket() + + sub_sock.connect(c, "controlsState") + pub_sock.connect(c, "controlsState") + + poller = Poller() + poller.registerSocket(sub_sock) + + t = time.time() + for i in range(int(MSGS)): + bts = i.to_bytes(4, 'little') + pub_sock.send(bts) + + for s in poller.poll(100): + dat = s.receive() + ii = int.from_bytes(dat, 'little') + assert(i == ii) + + dt = time.time() - t + print("%.1f msg/s" % (MSGS / dt)) diff --git a/cereal/messaging/messaging.h b/cereal/messaging/messaging.h new file mode 100644 index 00000000000000..b0f8b5535565c3 --- /dev/null +++ b/cereal/messaging/messaging.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "cereal/gen/cpp/log.capnp.h" +#include "msgq/messaging/messaging.h" + +#ifdef __APPLE__ +#define CLOCK_BOOTTIME CLOCK_MONOTONIC +#endif + +#define MSG_MULTIPLE_PUBLISHERS 100 + + +class SubMaster { +public: + SubMaster(const std::vector &service_list, const std::vector &poll = {}, + const char *address = nullptr, const std::vector &ignore_alive = {}); + void update(int timeout = 1000); + void update_msgs(uint64_t current_time, const std::vector> &messages); + inline bool allAlive(const std::vector &service_list = {}) { return all_(service_list, false, true); } + inline bool allValid(const std::vector &service_list = {}) { return all_(service_list, true, false); } + inline bool allAliveAndValid(const std::vector &service_list = {}) { return all_(service_list, true, true); } + void drain(); + ~SubMaster(); + + uint64_t frame = 0; + bool updated(const char *name) const; + bool alive(const char *name) const; + bool valid(const char *name) const; + uint64_t rcv_frame(const char *name) const; + uint64_t rcv_time(const char *name) const; + cereal::Event::Reader &operator[](const char *name) const; + +private: + bool all_(const std::vector &service_list, bool valid, bool alive); + Poller *poller_ = nullptr; + struct SubMessage; + std::map messages_; + std::map services_; +}; + +class MessageBuilder : public capnp::MallocMessageBuilder { +public: + MessageBuilder() = default; + + cereal::Event::Builder initEvent(bool valid = true) { + cereal::Event::Builder event = initRoot(); + struct timespec t; + clock_gettime(CLOCK_BOOTTIME, &t); + uint64_t current_time = t.tv_sec * 1000000000ULL + t.tv_nsec; + event.setLogMonoTime(current_time); + event.setValid(valid); + return event; + } + + kj::ArrayPtr toBytes() { + heapArray_ = capnp::messageToFlatArray(*this); + return heapArray_.asBytes(); + } + + size_t getSerializedSize() { + return capnp::computeSerializedSizeInWords(*this) * sizeof(capnp::word); + } + + int serializeToBuffer(unsigned char *buffer, size_t buffer_size) { + size_t serialized_size = getSerializedSize(); + if (serialized_size > buffer_size) { return -1; } + kj::ArrayOutputStream out(kj::ArrayPtr(buffer, buffer_size)); + capnp::writeMessage(out, *this); + return serialized_size; + } + +private: + kj::Array heapArray_; +}; + +class PubMaster { +public: + PubMaster(const std::vector &service_list); + inline int send(const char *name, capnp::byte *data, size_t size) { return sockets_.at(name)->send((char *)data, size); } + int send(const char *name, MessageBuilder &msg); + ~PubMaster(); + +private: + std::map sockets_; +}; + +class AlignedBuffer { +public: + kj::ArrayPtr align(const char *data, const size_t size) { + words_size = size / sizeof(capnp::word) + 1; + if (aligned_buf.size() < words_size) { + aligned_buf = kj::heapArray(words_size < 512 ? 512 : words_size); + } + memcpy(aligned_buf.begin(), data, size); + return aligned_buf.slice(0, words_size); + } + inline kj::ArrayPtr align(Message *m) { + return align(m->getData(), m->getSize()); + } +private: + kj::Array aligned_buf; + size_t words_size; +}; diff --git a/cereal/messaging/msgq.md b/cereal/messaging/msgq.md new file mode 100644 index 00000000000000..34fe35682fb542 --- /dev/null +++ b/cereal/messaging/msgq.md @@ -0,0 +1,54 @@ +# MSGQ: A lock free single producer multi consumer message queue + +## What is MSGQ? +MSGQ is a system to pass messages from a single producer to multiple consumers. All the consumers need to be able to receive all the messages. It is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. + +## Storage +The storage for the queue consists of an area of metadata, and the actual buffer. The metadata contains: + +1. A counter to the number of readers that are active +2. A pointer to the head of the queue for writing. From now on referred to as *write pointer* +3. A cycle counter for the writer. This counter is incremented when the writer wraps around +4. N pointers, pointing to the current read position for all the readers. From now on referred to as *read pointer* +5. N counters, counting the number of cycles for all the readers +6. N booleans, indicating validity for all the readers. From now on referred to as *validity flag* + +The counter and the pointer are both 32 bit values, packed into 64 bit so they can be read and written atomically. + +The data buffer is a ring buffer. All messages are prefixed by an 8 byte size field, followed by the data. A size of -1 indicates a wrap-around, and means the next message is stored at the beginning of the buffer. + + +## Writing +Writing involves the following steps: + +1. Check if the area that is to be written overlaps with any of the read pointers, mark those readers as invalid by clearing the validity flag. +2. Write the message +3. Increase the write pointer by the size of the message + +In case there is not enough space at the end of the buffer, a special empty message with a prefix of -1 is written. The cycle counter is incremented by one. In this case step 1 will check there are no read pointers pointing to the remainder of the buffer. Then another write cycle will start with the actual message. + +There always needs to be 8 bytes of empty space at the end of the buffer. By doing this there is always space to write the -1. + +## Reset reader +When the reader is lagging too much behind the read pointer becomes invalid and no longer points to the beginning of a valid message. To reset a reader to the current write pointer, the following steps are performed: + +1. Set valid flag +2. Set read cycle counter to that of the writer +3. Set read pointer to write pointer + +## Reading +Reading involves the following steps: + +1. Read the size field at the current read pointer +2. Read the validity flag +3. Copy the data out of the buffer +4. Increase the read pointer by the size of the message +5. Check the validity flag again + +Before starting the copy, the valid flag is checked. This is to prevent a race condition where the size prefix was invalid, and the read could read outside of the buffer. Make sure that step 1 and 2 are not reordered by your compiler or CPU. + +If a writer overwrites the data while it's being copied out, the data will be invalid. Therefore the validity flag is also checked after reading it. The order of step 4 and 5 does not matter. + +If at steps 2 or 5 the validity flag is not set, the reader is reset. Any data that was already read is discarded. After the reader is reset, the reading starts from the beginning. + +If a message with size -1 is encountered, step 3 and 4 are replaced by increasing the cycle counter and setting the read pointer to the beginning of the buffer. After that another read is performed. diff --git a/cereal/messaging/msgq_tests.cc b/cereal/messaging/msgq_tests.cc new file mode 100644 index 00000000000000..d33facbc047531 --- /dev/null +++ b/cereal/messaging/msgq_tests.cc @@ -0,0 +1,394 @@ +#include "catch2/catch.hpp" +#include "cereal/messaging/msgq.h" + +TEST_CASE("ALIGN"){ + REQUIRE(ALIGN(0) == 0); + REQUIRE(ALIGN(1) == 8); + REQUIRE(ALIGN(7) == 8); + REQUIRE(ALIGN(8) == 8); + REQUIRE(ALIGN(99999) == 100000); +} + +TEST_CASE("msgq_msg_init_size"){ + const size_t msg_size = 30; + msgq_msg_t msg; + + msgq_msg_init_size(&msg, msg_size); + REQUIRE(msg.size == msg_size); + + msgq_msg_close(&msg); +} + +TEST_CASE("msgq_msg_init_data"){ + const size_t msg_size = 30; + char * data = new char[msg_size]; + + for (size_t i = 0; i < msg_size; i++){ + data[i] = i; + } + + msgq_msg_t msg; + msgq_msg_init_data(&msg, data, msg_size); + + REQUIRE(msg.size == msg_size); + REQUIRE(memcmp(msg.data, data, msg_size) == 0); + + delete[] data; + msgq_msg_close(&msg); +} + + +TEST_CASE("msgq_init_subscriber"){ + remove("/dev/shm/test_queue"); + msgq_queue_t q; + msgq_new_queue(&q, "test_queue", 1024); + REQUIRE(*q.num_readers == 0); + + q.reader_id = 1; + *q.read_valids[0] = false; + *q.read_pointers[0] = ((uint64_t)1 << 32); + + *q.write_pointer = 255; + + msgq_init_subscriber(&q); + REQUIRE(q.read_conflate == false); + REQUIRE(*q.read_valids[0] == true); + REQUIRE((*q.read_pointers[0] >> 32) == 0); + REQUIRE((*q.read_pointers[0] & 0xFFFFFFFF) == 255); +} + +TEST_CASE("msgq_msg_send first message"){ + remove("/dev/shm/test_queue"); + msgq_queue_t q; + msgq_new_queue(&q, "test_queue", 1024); + msgq_init_publisher(&q); + + REQUIRE(*q.write_pointer == 0); + + size_t msg_size = 128; + + SECTION("Aligned message size"){ + } + SECTION("Unaligned message size"){ + msg_size--; + } + + char * data = new char[msg_size]; + + for (size_t i = 0; i < msg_size; i++){ + data[i] = i; + } + + msgq_msg_t msg; + msgq_msg_init_data(&msg, data, msg_size); + + + msgq_msg_send(&msg, &q); + REQUIRE(*(int64_t*)q.data == msg_size); // Check size tag + REQUIRE(*q.write_pointer == 128 + sizeof(int64_t)); + REQUIRE(memcmp(q.data + sizeof(int64_t), data, msg_size) == 0); + + delete[] data; + msgq_msg_close(&msg); +} + +TEST_CASE("msgq_msg_send test wraparound"){ + remove("/dev/shm/test_queue"); + msgq_queue_t q; + msgq_new_queue(&q, "test_queue", 1024); + msgq_init_publisher(&q); + + REQUIRE((*q.write_pointer & 0xFFFFFFFF) == 0); + REQUIRE((*q.write_pointer >> 32) == 0); + + const size_t msg_size = 120; + msgq_msg_t msg; + msgq_msg_init_size(&msg, msg_size); + + for (int i = 0; i < 8; i++) { + msgq_msg_send(&msg, &q); + } + // Check 8th message was written at the beginning + REQUIRE((*q.write_pointer & 0xFFFFFFFF) == msg_size + sizeof(int64_t)); + + // Check cycle count + REQUIRE((*q.write_pointer >> 32) == 1); + + // Check wraparound tag + char * tag_location = q.data; + tag_location += 7 * (msg_size + sizeof(int64_t)); + REQUIRE(*(int64_t*)tag_location == -1); + + msgq_msg_close(&msg); +} + +TEST_CASE("msgq_msg_recv test wraparound"){ + remove("/dev/shm/test_queue"); + msgq_queue_t q_pub, q_sub; + msgq_new_queue(&q_pub, "test_queue", 1024); + msgq_new_queue(&q_sub, "test_queue", 1024); + + msgq_init_publisher(&q_pub); + msgq_init_subscriber(&q_sub); + + REQUIRE((*q_pub.write_pointer >> 32) == 0); + REQUIRE((*q_sub.read_pointers[0] >> 32) == 0); + + const size_t msg_size = 120; + msgq_msg_t msg1; + msgq_msg_init_size(&msg1, msg_size); + + + SECTION("Check cycle counter after reset") { + for (int i = 0; i < 8; i++) { + msgq_msg_send(&msg1, &q_pub); + } + + msgq_msg_t msg2; + msgq_msg_recv(&msg2, &q_sub); + REQUIRE(msg2.size == 0); // Reader had to reset + msgq_msg_close(&msg2); + } + SECTION("Check cycle counter while keeping up with writer") { + for (int i = 0; i < 8; i++) { + msgq_msg_send(&msg1, &q_pub); + + msgq_msg_t msg2; + msgq_msg_recv(&msg2, &q_sub); + REQUIRE(msg2.size > 0); + msgq_msg_close(&msg2); + } + } + + REQUIRE((*q_sub.read_pointers[0] >> 32) == 1); + msgq_msg_close(&msg1); +} + +TEST_CASE("msgq_msg_send test invalidation"){ + remove("/dev/shm/test_queue"); + msgq_queue_t q_pub, q_sub; + msgq_new_queue(&q_pub, "test_queue", 1024); + msgq_new_queue(&q_sub, "test_queue", 1024); + + msgq_init_publisher(&q_pub); + msgq_init_subscriber(&q_sub); + *q_sub.write_pointer = (uint64_t)1 << 32; + + REQUIRE(*q_sub.read_valids[0] == true); + + SECTION("read pointer in tag"){ + *q_sub.read_pointers[0] = 0; + } + SECTION("read pointer in data section"){ + *q_sub.read_pointers[0] = 64; + } + SECTION("read pointer in wraparound section"){ + *q_pub.write_pointer = ((uint64_t)1 << 32) | 1000; // Writer is one cycle ahead + *q_sub.read_pointers[0] = 1020; + } + + msgq_msg_t msg; + msgq_msg_init_size(&msg, 128); + msgq_msg_send(&msg, &q_pub); + + REQUIRE(*q_sub.read_valids[0] == false); + + msgq_msg_close(&msg); +} + +TEST_CASE("msgq_init_subscriber init 2 subscribers"){ + remove("/dev/shm/test_queue"); + msgq_queue_t q1, q2; + msgq_new_queue(&q1, "test_queue", 1024); + msgq_new_queue(&q2, "test_queue", 1024); + + *q1.num_readers = 0; + + REQUIRE(*q1.num_readers == 0); + REQUIRE(*q2.num_readers == 0); + + msgq_init_subscriber(&q1); + REQUIRE(*q1.num_readers == 1); + REQUIRE(*q2.num_readers == 1); + REQUIRE(q1.reader_id == 0); + + msgq_init_subscriber(&q2); + REQUIRE(*q1.num_readers == 2); + REQUIRE(*q2.num_readers == 2); + REQUIRE(q2.reader_id == 1); +} + + +TEST_CASE("Write 1 msg, read 1 msg", "[integration]"){ + remove("/dev/shm/test_queue"); + const size_t msg_size = 128; + msgq_queue_t writer, reader; + + msgq_new_queue(&writer, "test_queue", 1024); + msgq_new_queue(&reader, "test_queue", 1024); + + msgq_init_publisher(&writer); + msgq_init_subscriber(&reader); + + // Build 128 byte message + msgq_msg_t outgoing_msg; + msgq_msg_init_size(&outgoing_msg, msg_size); + + for (size_t i = 0; i < msg_size; i++){ + outgoing_msg.data[i] = i; + } + + REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); + + msgq_msg_t incoming_msg1; + REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size); + REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0); + + // Verify that there are no more messages + msgq_msg_t incoming_msg2; + REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == 0); + + msgq_msg_close(&outgoing_msg); + msgq_msg_close(&incoming_msg1); + msgq_msg_close(&incoming_msg2); +} + +TEST_CASE("Write 2 msg, read 2 msg - conflate = false", "[integration]"){ + remove("/dev/shm/test_queue"); + const size_t msg_size = 128; + msgq_queue_t writer, reader; + + msgq_new_queue(&writer, "test_queue", 1024); + msgq_new_queue(&reader, "test_queue", 1024); + + msgq_init_publisher(&writer); + msgq_init_subscriber(&reader); + + // Build 128 byte message + msgq_msg_t outgoing_msg; + msgq_msg_init_size(&outgoing_msg, msg_size); + + for (size_t i = 0; i < msg_size; i++){ + outgoing_msg.data[i] = i; + } + + REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); + REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); + + msgq_msg_t incoming_msg1; + REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size); + REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0); + + msgq_msg_t incoming_msg2; + REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == msg_size); + REQUIRE(memcmp(incoming_msg2.data, outgoing_msg.data, msg_size) == 0); + + msgq_msg_close(&outgoing_msg); + msgq_msg_close(&incoming_msg1); + msgq_msg_close(&incoming_msg2); +} + +TEST_CASE("Write 2 msg, read 2 msg - conflate = true", "[integration]"){ + remove("/dev/shm/test_queue"); + const size_t msg_size = 128; + msgq_queue_t writer, reader; + + msgq_new_queue(&writer, "test_queue", 1024); + msgq_new_queue(&reader, "test_queue", 1024); + + msgq_init_publisher(&writer); + msgq_init_subscriber(&reader); + reader.read_conflate = true; + + // Build 128 byte message + msgq_msg_t outgoing_msg; + msgq_msg_init_size(&outgoing_msg, msg_size); + + for (size_t i = 0; i < msg_size; i++){ + outgoing_msg.data[i] = i; + } + + REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); + REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); + + msgq_msg_t incoming_msg1; + REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size); + REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0); + + // Verify that there are no more messages + msgq_msg_t incoming_msg2; + REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == 0); + + msgq_msg_close(&outgoing_msg); + msgq_msg_close(&incoming_msg1); + msgq_msg_close(&incoming_msg2); +} + +TEST_CASE("1 publisher, 1 slow subscriber", "[integration]"){ + remove("/dev/shm/test_queue"); + msgq_queue_t writer, reader; + + msgq_new_queue(&writer, "test_queue", 1024); + msgq_new_queue(&reader, "test_queue", 1024); + + msgq_init_publisher(&writer); + msgq_init_subscriber(&reader); + + int n_received = 0; + int n_skipped = 0; + + for (uint64_t i = 0; i < 1e5; i++) { + msgq_msg_t outgoing_msg; + msgq_msg_init_data(&outgoing_msg, (char*)&i, sizeof(uint64_t)); + msgq_msg_send(&outgoing_msg, &writer); + msgq_msg_close(&outgoing_msg); + + if (i % 10 == 0){ + msgq_msg_t msg1; + msgq_msg_recv(&msg1, &reader); + + if (msg1.size == 0){ + n_skipped++; + } else { + n_received++; + } + msgq_msg_close(&msg1); + } + } + + // TODO: verify these numbers by hand + REQUIRE(n_received == 8572); + REQUIRE(n_skipped == 1428); +} + +TEST_CASE("1 publisher, 2 subscribers", "[integration]"){ + remove("/dev/shm/test_queue"); + msgq_queue_t writer, reader1, reader2; + + msgq_new_queue(&writer, "test_queue", 1024); + msgq_new_queue(&reader1, "test_queue", 1024); + msgq_new_queue(&reader2, "test_queue", 1024); + + msgq_init_publisher(&writer); + msgq_init_subscriber(&reader1); + msgq_init_subscriber(&reader2); + + for (uint64_t i = 0; i < 1024 * 3; i++) { + msgq_msg_t outgoing_msg; + msgq_msg_init_data(&outgoing_msg, (char*)&i, sizeof(uint64_t)); + msgq_msg_send(&outgoing_msg, &writer); + + msgq_msg_t msg1, msg2; + msgq_msg_recv(&msg1, &reader1); + msgq_msg_recv(&msg2, &reader2); + + REQUIRE(msg1.size == sizeof(uint64_t)); + REQUIRE(msg2.size == sizeof(uint64_t)); + REQUIRE(*(uint64_t*)msg1.data == i); + REQUIRE(*(uint64_t*)msg2.data == i); + + msgq_msg_close(&outgoing_msg); + msgq_msg_close(&msg1); + msgq_msg_close(&msg2); + } +} diff --git a/cereal/messaging/socketmaster.cc b/cereal/messaging/socketmaster.cc new file mode 100644 index 00000000000000..cd697b1f51ed20 --- /dev/null +++ b/cereal/messaging/socketmaster.cc @@ -0,0 +1,211 @@ +#include +#include +#include +#include +#include + +#include "cereal/services.h" +#include "cereal/messaging/messaging.h" + + +const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1"); + +static inline uint64_t nanos_since_boot() { + struct timespec t; + clock_gettime(CLOCK_BOOTTIME, &t); + return t.tv_sec * 1000000000ULL + t.tv_nsec; +} + +static inline bool inList(const std::vector &list, const char *value) { + for (auto &v : list) { + if (strcmp(value, v) == 0) return true; + } + return false; +} + +class MessageContext { +public: + MessageContext() : ctx_(nullptr) {} + ~MessageContext() { delete ctx_; } + inline Context *context() { + std::call_once(init_flag, [=]() { ctx_ = Context::create(); }); + return ctx_; + } +private: + Context *ctx_; + std::once_flag init_flag; +}; + +MessageContext message_context; + +struct SubMaster::SubMessage { + std::string name; + SubSocket *socket = nullptr; + int freq = 0; + bool updated = false, alive = false, valid = true, ignore_alive; + uint64_t rcv_time = 0, rcv_frame = 0; + void *allocated_msg_reader = nullptr; + bool is_polled = false; + capnp::FlatArrayMessageReader *msg_reader = nullptr; + AlignedBuffer aligned_buf; + cereal::Event::Reader event; +}; + +SubMaster::SubMaster(const std::vector &service_list, const std::vector &poll, + const char *address, const std::vector &ignore_alive) { + poller_ = Poller::create(); + for (auto name : service_list) { + assert(services.count(std::string(name)) > 0); + + service serv = services.at(std::string(name)); + SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true); + assert(socket != 0); + bool is_polled = inList(poll, name) || poll.empty(); + if (is_polled) poller_->registerSocket(socket); + SubMessage *m = new SubMessage{ + .name = name, + .socket = socket, + .freq = serv.frequency, + .ignore_alive = inList(ignore_alive, name), + .allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)), + .is_polled = is_polled}; + m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({}); + messages_[socket] = m; + services_[name] = m; + } +} + +void SubMaster::update(int timeout) { + for (auto &kv : messages_) kv.second->updated = false; + + auto sockets = poller_->poll(timeout); + + // add non-polled sockets for non-blocking receive + for (auto &kv : messages_) { + SubMessage *m = kv.second; + SubSocket *s = kv.first; + if (!m->is_polled) sockets.push_back(s); + } + + uint64_t current_time = nanos_since_boot(); + + std::vector> messages; + + for (auto s : sockets) { + Message *msg = s->receive(true); + if (msg == nullptr) continue; + + SubMessage *m = messages_.at(s); + + m->msg_reader->~FlatArrayMessageReader(); + capnp::ReaderOptions options; + options.traversalLimitInWords = kj::maxValue; // Don't limit + m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg), options); + delete msg; + messages.push_back({m->name, m->msg_reader->getRoot()}); + } + + update_msgs(current_time, messages); +} + +void SubMaster::update_msgs(uint64_t current_time, const std::vector> &messages){ + if (++frame == UINT64_MAX) frame = 1; + + for (auto &kv : messages) { + auto m_find = services_.find(kv.first); + if (m_find == services_.end()){ + continue; + } + SubMessage *m = m_find->second; + m->event = kv.second; + m->updated = true; + m->rcv_time = current_time; + m->rcv_frame = frame; + m->valid = m->event.getValid(); + if (SIMULATION) m->alive = true; + } + + if (!SIMULATION) { + for (auto &kv : messages_) { + SubMessage *m = kv.second; + m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq)); + } + } +} + +bool SubMaster::all_(const std::vector &service_list, bool valid, bool alive) { + int found = 0; + for (auto &kv : messages_) { + SubMessage *m = kv.second; + if (service_list.size() == 0 || inList(service_list, m->name.c_str())) { + found += (!valid || m->valid) && (!alive || (m->alive || m->ignore_alive)); + } + } + return service_list.size() == 0 ? found == messages_.size() : found == service_list.size(); +} + +void SubMaster::drain() { + while (true) { + auto polls = poller_->poll(0); + if (polls.size() == 0) + break; + + for (auto sock : polls) { + Message *msg = sock->receive(true); + delete msg; + } + } +} + +bool SubMaster::updated(const char *name) const { + return services_.at(name)->updated; +} + +bool SubMaster::alive(const char *name) const { + return services_.at(name)->alive; +} + +bool SubMaster::valid(const char *name) const { + return services_.at(name)->valid; +} + +uint64_t SubMaster::rcv_frame(const char *name) const { + return services_.at(name)->rcv_frame; +} + +uint64_t SubMaster::rcv_time(const char *name) const { + return services_.at(name)->rcv_time; +} + +cereal::Event::Reader &SubMaster::operator[](const char *name) const { + return services_.at(name)->event; +} + +SubMaster::~SubMaster() { + delete poller_; + for (auto &kv : messages_) { + SubMessage *m = kv.second; + m->msg_reader->~FlatArrayMessageReader(); + free(m->allocated_msg_reader); + delete m->socket; + delete m; + } +} + +PubMaster::PubMaster(const std::vector &service_list) { + for (auto name : service_list) { + assert(services.count(name) > 0); + PubSocket *socket = PubSocket::create(message_context.context(), name); + assert(socket); + sockets_[name] = socket; + } +} + +int PubMaster::send(const char *name, MessageBuilder &msg) { + auto bytes = msg.toBytes(); + return send(name, bytes.begin(), bytes.size()); +} + +PubMaster::~PubMaster() { + for (auto s : sockets_) delete s.second; +} diff --git a/cereal/messaging/stress.py b/cereal/messaging/stress.py new file mode 100644 index 00000000000000..1a27e520edd01e --- /dev/null +++ b/cereal/messaging/stress.py @@ -0,0 +1,14 @@ +from messaging_pyx import Context, SubSocket, PubSocket + +if __name__ == "__main__": + c = Context() + pub_sock = PubSocket() + pub_sock.connect(c, "controlsState") + + for i in range(int(1e10)): + print(i) + sub_sock = SubSocket() + sub_sock.connect(c, "controlsState") + + pub_sock.send(b'a') + print(sub_sock.receive()) diff --git a/cereal/messaging/test_runner.cc b/cereal/messaging/test_runner.cc new file mode 100644 index 00000000000000..62bf7476a18996 --- /dev/null +++ b/cereal/messaging/test_runner.cc @@ -0,0 +1,2 @@ +#define CATCH_CONFIG_MAIN +#include "catch2/catch.hpp" diff --git a/cereal/messaging/tests/__init__.py b/cereal/messaging/tests/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/cereal/messaging/tests/test_fake.py b/cereal/messaging/tests/test_fake.py new file mode 100644 index 00000000000000..1d3521745d0399 --- /dev/null +++ b/cereal/messaging/tests/test_fake.py @@ -0,0 +1,193 @@ +import os +import unittest +import multiprocessing +import platform +from parameterized import parameterized_class +from typing import Optional + +import cereal.messaging as messaging + +WAIT_TIMEOUT = 5 + + +@unittest.skipIf(platform.system() == "Darwin", "Events not supported on macOS") +class TestEvents(unittest.TestCase): + + def test_mutation(self): + handle = messaging.fake_event_handle("carState") + event = handle.recv_called_event + + self.assertFalse(event.peek()) + event.set() + self.assertTrue(event.peek()) + event.clear() + self.assertFalse(event.peek()) + + del event + + def test_wait(self): + handle = messaging.fake_event_handle("carState") + event = handle.recv_called_event + + event.set() + try: + event.wait(WAIT_TIMEOUT) + self.assertTrue(event.peek()) + except RuntimeError: + self.fail("event.wait() timed out") + + def test_wait_multiprocess(self): + handle = messaging.fake_event_handle("carState") + event = handle.recv_called_event + + def set_event_run(): + event.set() + + try: + p = multiprocessing.Process(target=set_event_run) + p.start() + event.wait(WAIT_TIMEOUT) + self.assertTrue(event.peek()) + except RuntimeError: + self.fail("event.wait() timed out") + + p.kill() + + def test_wait_zero_timeout(self): + handle = messaging.fake_event_handle("carState") + event = handle.recv_called_event + + try: + event.wait(0) + self.fail("event.wait() did not time out") + except RuntimeError: + self.assertFalse(event.peek()) + + +@unittest.skipIf(platform.system() == "Darwin", "FakeSockets not supported on macOS") +@unittest.skipIf("ZMQ" in os.environ, "FakeSockets not supported on ZMQ") +@parameterized_class([{"prefix": None}, {"prefix": "test"}]) +class TestFakeSockets(unittest.TestCase): + prefix: Optional[str] = None + + def setUp(self): + messaging.toggle_fake_events(True) + if self.prefix is not None: + messaging.set_fake_prefix(self.prefix) + else: + messaging.delete_fake_prefix() + + def tearDown(self): + messaging.toggle_fake_events(False) + messaging.delete_fake_prefix() + + def test_event_handle_init(self): + handle = messaging.fake_event_handle("controlsState", override=True) + + self.assertFalse(handle.enabled) + self.assertGreaterEqual(handle.recv_called_event.fd, 0) + self.assertGreaterEqual(handle.recv_ready_event.fd, 0) + + def test_non_managed_socket_state(self): + # non managed socket should have zero state + _ = messaging.pub_sock("ubloxGnss") + + handle = messaging.fake_event_handle("ubloxGnss", override=False) + + self.assertFalse(handle.enabled) + self.assertEqual(handle.recv_called_event.fd, 0) + self.assertEqual(handle.recv_ready_event.fd, 0) + + def test_managed_socket_state(self): + # managed socket should not change anything about the state + handle = messaging.fake_event_handle("ubloxGnss") + handle.enabled = True + + expected_enabled = handle.enabled + expected_recv_called_fd = handle.recv_called_event.fd + expected_recv_ready_fd = handle.recv_ready_event.fd + + _ = messaging.pub_sock("ubloxGnss") + + self.assertEqual(handle.enabled, expected_enabled) + self.assertEqual(handle.recv_called_event.fd, expected_recv_called_fd) + self.assertEqual(handle.recv_ready_event.fd, expected_recv_ready_fd) + + def test_sockets_enable_disable(self): + carState_handle = messaging.fake_event_handle("ubloxGnss", enable=True) + recv_called = carState_handle.recv_called_event + recv_ready = carState_handle.recv_ready_event + + pub_sock = messaging.pub_sock("ubloxGnss") + sub_sock = messaging.sub_sock("ubloxGnss") + + try: + carState_handle.enabled = True + recv_ready.set() + pub_sock.send(b"test") + _ = sub_sock.receive() + self.assertTrue(recv_called.peek()) + recv_called.clear() + + carState_handle.enabled = False + recv_ready.set() + pub_sock.send(b"test") + _ = sub_sock.receive() + self.assertFalse(recv_called.peek()) + except RuntimeError: + self.fail("event.wait() timed out") + + def test_synced_pub_sub(self): + def daemon_repub_process_run(): + pub_sock = messaging.pub_sock("ubloxGnss") + sub_sock = messaging.sub_sock("carState") + + frame = -1 + while True: + frame += 1 + msg = sub_sock.receive(non_blocking=True) + if msg is None: + print("none received") + continue + + bts = frame.to_bytes(8, 'little') + pub_sock.send(bts) + + carState_handle = messaging.fake_event_handle("carState", enable=True) + recv_called = carState_handle.recv_called_event + recv_ready = carState_handle.recv_ready_event + + p = multiprocessing.Process(target=daemon_repub_process_run) + p.start() + + pub_sock = messaging.pub_sock("carState") + sub_sock = messaging.sub_sock("ubloxGnss") + + try: + for i in range(10): + recv_called.wait(WAIT_TIMEOUT) + recv_called.clear() + + if i == 0: + sub_sock.receive(non_blocking=True) + + bts = i.to_bytes(8, 'little') + pub_sock.send(bts) + + recv_ready.set() + recv_called.wait(WAIT_TIMEOUT) + + msg = sub_sock.receive(non_blocking=True) + self.assertIsNotNone(msg) + self.assertEqual(len(msg), 8) + + frame = int.from_bytes(msg, 'little') + self.assertEqual(frame, i) + except RuntimeError: + self.fail("event.wait() timed out") + finally: + p.kill() + + +if __name__ == "__main__": + unittest.main() diff --git a/cereal/messaging/tests/test_messaging.py b/cereal/messaging/tests/test_messaging.py new file mode 100755 index 00000000000000..381cec03ff1a95 --- /dev/null +++ b/cereal/messaging/tests/test_messaging.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +import os +import capnp +import multiprocessing +import numbers +import random +import threading +import time +import unittest +from parameterized import parameterized + +from cereal import log, car +import cereal.messaging as messaging +from cereal.services import SERVICE_LIST + +events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()] + +def random_sock(): + return random.choice(events) + +def random_socks(num_socks=10): + return list({random_sock() for _ in range(num_socks)}) + +def random_bytes(length=1000): + return bytes([random.randrange(0xFF) for _ in range(length)]) + +def zmq_sleep(t=1): + if "ZMQ" in os.environ: + time.sleep(t) + +def zmq_expected_failure(func): + if "ZMQ" in os.environ: + return unittest.expectedFailure(func) + else: + return func + +# TODO: this should take any capnp struct and returrn a msg with random populated data +def random_carstate(): + fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"] + msg = messaging.new_message("carState") + cs = msg.carState + for f in fields: + setattr(cs, f, random.random() * 10) + return msg + +# TODO: this should compare any capnp structs +def assert_carstate(cs1, cs2): + for f in car.CarState.schema.non_union_fields: + # TODO: check all types + val1, val2 = getattr(cs1, f), getattr(cs2, f) + if isinstance(val1, numbers.Number): + assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'" + +def delayed_send(delay, sock, dat): + def send_func(): + sock.send(dat) + threading.Timer(delay, send_func).start() + +class TestPubSubSockets(unittest.TestCase): + + def setUp(self): + # ZMQ pub socket takes too long to die + # sleep to prevent multiple publishers error between tests + zmq_sleep() + + def test_pub_sub(self): + sock = random_sock() + pub_sock = messaging.pub_sock(sock) + sub_sock = messaging.sub_sock(sock, conflate=False, timeout=None) + zmq_sleep(3) + + for _ in range(1000): + msg = random_bytes() + pub_sock.send(msg) + recvd = sub_sock.receive() + self.assertEqual(msg, recvd) + + def test_conflate(self): + sock = random_sock() + pub_sock = messaging.pub_sock(sock) + for conflate in [True, False]: + for _ in range(10): + num_msgs = random.randint(3, 10) + sub_sock = messaging.sub_sock(sock, conflate=conflate, timeout=None) + zmq_sleep() + + sent_msgs = [] + for __ in range(num_msgs): + msg = random_bytes() + pub_sock.send(msg) + sent_msgs.append(msg) + time.sleep(0.1) + recvd_msgs = messaging.drain_sock_raw(sub_sock) + if conflate: + self.assertEqual(len(recvd_msgs), 1) + else: + # TODO: compare actual data + self.assertEqual(len(recvd_msgs), len(sent_msgs)) + + def test_receive_timeout(self): + sock = random_sock() + for _ in range(10): + timeout = random.randrange(200) + sub_sock = messaging.sub_sock(sock, timeout=timeout) + zmq_sleep() + + start_time = time.monotonic() + recvd = sub_sock.receive() + self.assertLess(time.monotonic() - start_time, 0.2) + assert recvd is None + +class TestMessaging(unittest.TestCase): + + def setUp(self): + # TODO: ZMQ tests are too slow; all sleeps will need to be + # replaced with logic to block on the necessary condition + if "ZMQ" in os.environ: + raise unittest.SkipTest + + # ZMQ pub socket takes too long to die + # sleep to prevent multiple publishers error between tests + zmq_sleep() + + @parameterized.expand(events) + def test_new_message(self, evt): + try: + msg = messaging.new_message(evt) + except capnp.lib.capnp.KjException: + msg = messaging.new_message(evt, random.randrange(200)) + self.assertLess(time.monotonic() - msg.logMonoTime, 0.1) + self.assertFalse(msg.valid) + self.assertEqual(evt, msg.which()) + + @parameterized.expand(events) + def test_pub_sock(self, evt): + messaging.pub_sock(evt) + + @parameterized.expand(events) + def test_sub_sock(self, evt): + messaging.sub_sock(evt) + + @parameterized.expand([ + (messaging.drain_sock, capnp._DynamicStructReader), + (messaging.drain_sock_raw, bytes), + ]) + def test_drain_sock(self, func, expected_type): + sock = "carState" + pub_sock = messaging.pub_sock(sock) + sub_sock = messaging.sub_sock(sock, timeout=1000) + zmq_sleep() + + # no wait and no msgs in queue + msgs = func(sub_sock) + self.assertIsInstance(msgs, list) + self.assertEqual(len(msgs), 0) + + # no wait but msgs are queued up + num_msgs = random.randrange(3, 10) + for _ in range(num_msgs): + pub_sock.send(messaging.new_message(sock).to_bytes()) + time.sleep(0.1) + msgs = func(sub_sock) + self.assertIsInstance(msgs, list) + self.assertTrue(all(isinstance(msg, expected_type) for msg in msgs)) + self.assertEqual(len(msgs), num_msgs) + + def test_recv_sock(self): + sock = "carState" + pub_sock = messaging.pub_sock(sock) + sub_sock = messaging.sub_sock(sock, timeout=100) + zmq_sleep() + + # no wait and no msg in queue, socket should timeout + recvd = messaging.recv_sock(sub_sock) + self.assertTrue(recvd is None) + + # no wait and one msg in queue + msg = random_carstate() + pub_sock.send(msg.to_bytes()) + time.sleep(0.01) + recvd = messaging.recv_sock(sub_sock) + self.assertIsInstance(recvd, capnp._DynamicStructReader) + # https://github.com/python/mypy/issues/13038 + assert_carstate(msg.carState, recvd.carState) + + def test_recv_one(self): + sock = "carState" + pub_sock = messaging.pub_sock(sock) + sub_sock = messaging.sub_sock(sock, timeout=1000) + zmq_sleep() + + # no msg in queue, socket should timeout + recvd = messaging.recv_one(sub_sock) + self.assertTrue(recvd is None) + + # one msg in queue + msg = random_carstate() + pub_sock.send(msg.to_bytes()) + recvd = messaging.recv_one(sub_sock) + self.assertIsInstance(recvd, capnp._DynamicStructReader) + assert_carstate(msg.carState, recvd.carState) + + @zmq_expected_failure + def test_recv_one_or_none(self): + sock = "carState" + pub_sock = messaging.pub_sock(sock) + sub_sock = messaging.sub_sock(sock) + zmq_sleep() + + # no msg in queue, socket shouldn't block + recvd = messaging.recv_one_or_none(sub_sock) + self.assertTrue(recvd is None) + + # one msg in queue + msg = random_carstate() + pub_sock.send(msg.to_bytes()) + recvd = messaging.recv_one_or_none(sub_sock) + self.assertIsInstance(recvd, capnp._DynamicStructReader) + assert_carstate(msg.carState, recvd.carState) + + def test_recv_one_retry(self): + sock = "carState" + sock_timeout = 0.1 + pub_sock = messaging.pub_sock(sock) + sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000)) + zmq_sleep() + + # this test doesn't work with ZMQ since multiprocessing interrupts it + if "ZMQ" not in os.environ: + # wait 15 socket timeouts and make sure it's still retrying + p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,)) + p.start() + time.sleep(sock_timeout*15) + self.assertTrue(p.is_alive()) + p.terminate() + + # wait 15 socket timeouts before sending + msg = random_carstate() + delayed_send(sock_timeout*15, pub_sock, msg.to_bytes()) + start_time = time.monotonic() + recvd = messaging.recv_one_retry(sub_sock) + self.assertGreaterEqual(time.monotonic() - start_time, sock_timeout*15) + self.assertIsInstance(recvd, capnp._DynamicStructReader) + assert_carstate(msg.carState, recvd.carState) + +if __name__ == "__main__": + unittest.main() diff --git a/cereal/messaging/tests/test_poller.py b/cereal/messaging/tests/test_poller.py new file mode 100644 index 00000000000000..bcff5e40caebcf --- /dev/null +++ b/cereal/messaging/tests/test_poller.py @@ -0,0 +1,142 @@ +import unittest +import time +import cereal.messaging as messaging + +import concurrent.futures + + +def poller(): + context = messaging.Context() + + p = messaging.Poller() + + sub = messaging.SubSocket() + sub.connect(context, 'controlsState') + p.registerSocket(sub) + + socks = p.poll(10000) + r = [s.receive(non_blocking=True) for s in socks] + + return r + + +class TestPoller(unittest.TestCase): + def test_poll_once(self): + context = messaging.Context() + + pub = messaging.PubSocket() + pub.connect(context, 'controlsState') + + with concurrent.futures.ThreadPoolExecutor() as e: + poll = e.submit(poller) + + time.sleep(0.1) # Slow joiner syndrome + + # Send message + pub.send(b"a") + + # Wait for poll result + result = poll.result() + + del pub + context.term() + + self.assertEqual(result, [b"a"]) + + def test_poll_and_create_many_subscribers(self): + context = messaging.Context() + + pub = messaging.PubSocket() + pub.connect(context, 'controlsState') + + with concurrent.futures.ThreadPoolExecutor() as e: + poll = e.submit(poller) + + time.sleep(0.1) # Slow joiner syndrome + c = messaging.Context() + for _ in range(10): + messaging.SubSocket().connect(c, 'controlsState') + + time.sleep(0.1) + + # Send message + pub.send(b"a") + + # Wait for poll result + result = poll.result() + + del pub + context.term() + + self.assertEqual(result, [b"a"]) + + def test_multiple_publishers_exception(self): + context = messaging.Context() + + with self.assertRaises(messaging.MultiplePublishersError): + pub1 = messaging.PubSocket() + pub1.connect(context, 'controlsState') + + pub2 = messaging.PubSocket() + pub2.connect(context, 'controlsState') + + pub1.send(b"a") + + del pub1 + del pub2 + context.term() + + def test_multiple_messages(self): + context = messaging.Context() + + pub = messaging.PubSocket() + pub.connect(context, 'controlsState') + + sub = messaging.SubSocket() + sub.connect(context, 'controlsState') + + time.sleep(0.1) # Slow joiner + + for i in range(1, 100): + pub.send(b'a'*i) + + msg_seen = False + i = 1 + while True: + r = sub.receive(non_blocking=True) + + if r is not None: + self.assertEqual(b'a'*i, r) + + msg_seen = True + i += 1 + + if r is None and msg_seen: # ZMQ sometimes receives nothing on the first receive + break + + del pub + del sub + context.term() + + def test_conflate(self): + context = messaging.Context() + + pub = messaging.PubSocket() + pub.connect(context, 'controlsState') + + sub = messaging.SubSocket() + sub.connect(context, 'controlsState', conflate=True) + + time.sleep(0.1) # Slow joiner + pub.send(b'a') + pub.send(b'b') + + self.assertEqual(b'b', sub.receive()) + + del pub + del sub + context.term() + + +if __name__ == "__main__": + unittest.main() diff --git a/cereal/messaging/tests/test_pub_sub_master.py b/cereal/messaging/tests/test_pub_sub_master.py new file mode 100755 index 00000000000000..81a1cf2d57c28d --- /dev/null +++ b/cereal/messaging/tests/test_pub_sub_master.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +import random +import time +from typing import Sized, cast +import unittest + +import cereal.messaging as messaging +from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \ + random_bytes, random_carstate, assert_carstate, \ + zmq_sleep + + +class TestSubMaster(unittest.TestCase): + + def setUp(self): + # ZMQ pub socket takes too long to die + # sleep to prevent multiple publishers error between tests + zmq_sleep(3) + + def test_init(self): + sm = messaging.SubMaster(events) + for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive, + sm.sock, sm.data, sm.logMonoTime, sm.valid]: + self.assertEqual(len(cast(Sized, p)), len(events)) + + def test_init_state(self): + socks = random_socks() + sm = messaging.SubMaster(socks) + self.assertEqual(sm.frame, -1) + self.assertFalse(any(sm.updated.values())) + self.assertFalse(any(sm.alive.values())) + self.assertTrue(all(t == 0. for t in sm.recv_time.values())) + self.assertTrue(all(f == 0 for f in sm.recv_frame.values())) + self.assertTrue(all(t == 0 for t in sm.logMonoTime.values())) + + for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive, + sm.sock, sm.data, sm.logMonoTime, sm.valid]: + self.assertEqual(len(cast(Sized, p)), len(socks)) + + def test_getitem(self): + sock = "carState" + pub_sock = messaging.pub_sock(sock) + sm = messaging.SubMaster([sock,]) + zmq_sleep() + + msg = random_carstate() + pub_sock.send(msg.to_bytes()) + sm.update(1000) + assert_carstate(msg.carState, sm[sock]) + + # TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs + def test_update(self): + sock = "carState" + pub_sock = messaging.pub_sock(sock) + sm = messaging.SubMaster([sock,]) + zmq_sleep() + + for i in range(10): + msg = messaging.new_message(sock) + pub_sock.send(msg.to_bytes()) + sm.update(1000) + self.assertEqual(sm.frame, i) + self.assertTrue(all(sm.updated.values())) + + def test_update_timeout(self): + sock = random_sock() + sm = messaging.SubMaster([sock,]) + for _ in range(5): + timeout = random.randrange(1000, 5000) + start_time = time.monotonic() + sm.update(timeout) + t = time.monotonic() - start_time + self.assertGreaterEqual(t, timeout/1000.) + self.assertLess(t, 5) + self.assertFalse(any(sm.updated.values())) + + def test_avg_frequency_checks(self): + for poll in (True, False): + sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"], + poll=("modelV2" if poll else None), + frequency=(20. if not poll else None)) + + checks = { + "carState": (20, 20), + "modelV2": (20, 20 if poll else 10), + "cameraOdometry": (20, 10), + "liveCalibration": (4, 4), + "carParams": (None, None), + } + + for service, (max_freq, min_freq) in checks.items(): + if max_freq is not None: + assert sm._check_avg_freq(service) + assert sm.max_freq[service] == max_freq*1.2 + assert sm.min_freq[service] == min_freq*0.8 + else: + assert not sm._check_avg_freq(service) + + def test_alive(self): + pass + + def test_ignore_alive(self): + pass + + def test_valid(self): + pass + + # SubMaster should always conflate + def test_conflate(self): + sock = "carState" + pub_sock = messaging.pub_sock(sock) + sm = messaging.SubMaster([sock,]) + + n = 10 + for i in range(n+1): + msg = messaging.new_message(sock) + msg.carState.vEgo = i + pub_sock.send(msg.to_bytes()) + time.sleep(0.01) + sm.update(1000) + self.assertEqual(sm[sock].vEgo, n) + + +class TestPubMaster(unittest.TestCase): + + def setUp(self): + # ZMQ pub socket takes too long to die + # sleep to prevent multiple publishers error between tests + zmq_sleep(3) + + def test_init(self): + messaging.PubMaster(events) + + def test_send(self): + socks = random_socks() + pm = messaging.PubMaster(socks) + sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks} + zmq_sleep() + + # PubMaster accepts either a capnp msg builder or bytes + for capnp in [True, False]: + for i in range(100): + sock = socks[i % len(socks)] + + if capnp: + try: + msg = messaging.new_message(sock) + except Exception: + msg = messaging.new_message(sock, random.randrange(50)) + else: + msg = random_bytes() + + pm.send(sock, msg) + recvd = sub_socks[sock].receive() + + if capnp: + msg.clear_write_flag() + msg = msg.to_bytes() + self.assertEqual(msg, recvd, i) + + +if __name__ == "__main__": + unittest.main() diff --git a/cereal/messaging/tests/test_services.py b/cereal/messaging/tests/test_services.py new file mode 100755 index 00000000000000..0fac379e439994 --- /dev/null +++ b/cereal/messaging/tests/test_services.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import os +import tempfile +from typing import Dict +import unittest +from parameterized import parameterized + +import cereal.services as services +from cereal.services import SERVICE_LIST + + +class TestServices(unittest.TestCase): + + @parameterized.expand(SERVICE_LIST.keys()) + def test_services(self, s): + service = SERVICE_LIST[s] + self.assertTrue(service.frequency <= 104) + + def test_generated_header(self): + with tempfile.NamedTemporaryFile(suffix=".h") as f: + ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}") + self.assertEqual(ret, 0, "generated services header is not valid C") + +if __name__ == "__main__": + unittest.main() diff --git a/cereal/services.py b/cereal/services.py new file mode 100755 index 00000000000000..2ab28f6d523ba3 --- /dev/null +++ b/cereal/services.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +from typing import Optional + + +class Service: + def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None): + self.should_log = should_log + self.frequency = frequency + self.decimation = decimation + + +_services: dict[str, tuple] = { + # service: (should_log, frequency, qlog decimation (optional)) + # note: the "EncodeIdx" packets will still be in the log + "gyroscope": (True, 104., 104), + "gyroscope2": (True, 100., 100), + "accelerometer": (True, 104., 104), + "accelerometer2": (True, 100., 100), + "magnetometer": (True, 25., 25), + "lightSensor": (True, 100., 100), + "temperatureSensor": (True, 2., 200), + "temperatureSensor2": (True, 2., 200), + "gpsNMEA": (True, 9.), + "deviceState": (True, 2., 1), + "can": (True, 100., 1223), # decimation gives ~5 msgs in a full segment + "controlsState": (True, 100., 10), + "pandaStates": (True, 10., 1), + "peripheralState": (True, 2., 1), + "radarState": (True, 20., 5), + "roadEncodeIdx": (False, 20., 1), + "liveTracks": (True, 20.), + "sendcan": (True, 100., 139), + "logMessage": (True, 0.), + "errorLogMessage": (True, 0., 1), + "liveCalibration": (True, 4., 4), + "liveTorqueParameters": (True, 4., 1), + "androidLog": (True, 0.), + "carState": (True, 100., 10), + "carControl": (True, 100., 10), + "carOutput": (True, 100., 10), + "longitudinalPlan": (True, 20., 5), + "procLog": (True, 0.5, 15), + "gpsLocationExternal": (True, 10., 10), + "gpsLocation": (True, 1., 1), + "ubloxGnss": (True, 10.), + "qcomGnss": (True, 2.), + "gnssMeasurements": (True, 10., 10), + "clocks": (True, 0.1, 1), + "ubloxRaw": (True, 20.), + "liveLocationKalman": (True, 20., 5), + "liveParameters": (True, 20., 5), + "cameraOdometry": (True, 20., 5), + "thumbnail": (True, 0.2, 1), + "onroadEvents": (True, 1., 1), + "carParams": (True, 0.02, 1), + "roadCameraState": (True, 20., 20), + "driverCameraState": (True, 20., 20), + "driverEncodeIdx": (False, 20., 1), + "driverStateV2": (True, 20., 10), + "driverMonitoringState": (True, 20., 10), + "wideRoadEncodeIdx": (False, 20., 1), + "wideRoadCameraState": (True, 20., 20), + "modelV2": (True, 20., 40), + "managerState": (True, 2., 1), + "uploaderState": (True, 0., 1), + "navInstruction": (True, 1., 10), + "navRoute": (True, 0.), + "navThumbnail": (True, 0.), + "uiPlan": (True, 20., 40.), + "qRoadEncodeIdx": (False, 20.), + "userFlag": (True, 0., 1), + "microphone": (True, 10., 10), + + # debug + "uiDebug": (True, 0., 1), + "testJoystick": (True, 0.), + "roadEncodeData": (False, 20.), + "driverEncodeData": (False, 20.), + "wideRoadEncodeData": (False, 20.), + "qRoadEncodeData": (False, 20.), + "livestreamWideRoadEncodeIdx": (False, 20.), + "livestreamRoadEncodeIdx": (False, 20.), + "livestreamDriverEncodeIdx": (False, 20.), + "livestreamWideRoadEncodeData": (False, 20.), + "livestreamRoadEncodeData": (False, 20.), + "livestreamDriverEncodeData": (False, 20.), + "customReservedRawData0": (True, 0.), + "customReservedRawData1": (True, 0.), + "customReservedRawData2": (True, 0.), +} +SERVICE_LIST = {name: Service(*vals) for + idx, (name, vals) in enumerate(_services.items())} + + +def build_header(): + h = "" + h += "/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */\n" + h += "#ifndef __SERVICES_H\n" + h += "#define __SERVICES_H\n" + + h += "#include \n" + h += "#include \n" + + h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n" + h += "static std::map services = {\n" + for k, v in SERVICE_LIST.items(): + should_log = "true" if v.should_log else "false" + decimation = -1 if v.decimation is None else v.decimation + h += ' { "%s", {"%s", %s, %d, %d}},\n' % \ + (k, k, should_log, v.frequency, decimation) + h += "};\n" + + h += "#endif\n" + return h + + +if __name__ == "__main__": + print(build_header()) diff --git a/docs/c_docs.rst b/docs/c_docs.rst index 7303c89be85550..1027bac1be49a1 100644 --- a/docs/c_docs.rst +++ b/docs/c_docs.rst @@ -14,12 +14,12 @@ cereal messaging ^^^^^^^^^ .. autodoxygenindex:: - :project: cereal_messaging + :project: msgq_messaging visionipc ^^^^^^^^^ .. autodoxygenindex:: - :project: cereal_visionipc + :project: msgq_visionipc selfdrive diff --git a/msgq b/msgq new file mode 160000 index 00000000000000..6d2cc6e22229a9 --- /dev/null +++ b/msgq @@ -0,0 +1 @@ +Subproject commit 6d2cc6e22229a9c855d8474e5643b26fbf2b5976 diff --git a/opendbc b/opendbc index 9a92d7a232e3ab..f58bc3392cfffd 160000 --- a/opendbc +++ b/opendbc @@ -1 +1 @@ -Subproject commit 9a92d7a232e3abe54c501e01c960462dd04588ea +Subproject commit f58bc3392cfffd25b311e093aa72f0ef3a89f6e8 diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index c8cc76cfc929f9..bc79c1fb41808c 100755 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -8,7 +8,7 @@ import cereal.messaging as messaging from cereal import car, log -from cereal.visionipc import VisionIpcClient, VisionStreamType +from msgq.visionipc import VisionIpcClient, VisionStreamType from openpilot.common.conversions import Conversions as CV diff --git a/selfdrive/locationd/SConscript b/selfdrive/locationd/SConscript index 07555a6087f678..d3cfeb2c3a430e 100644 --- a/selfdrive/locationd/SConscript +++ b/selfdrive/locationd/SConscript @@ -1,6 +1,6 @@ -Import('env', 'arch', 'common', 'cereal', 'messaging', 'rednose', 'transformations') +Import('env', 'arch', 'common', 'cereal', 'messaging', 'rednose', 'transformations', 'socketmaster') -loc_libs = [cereal, messaging, 'zmq', common, 'capnp', 'kj', 'pthread', 'dl'] +loc_libs = [cereal, socketmaster, messaging, 'zmq', common, 'capnp', 'kj', 'pthread', 'dl'] # build ekf models rednose_gen_dir = 'models/generated' diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index ef403b44fc648f..a388bf089cb9b9 100755 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -9,7 +9,7 @@ from cereal import messaging from cereal.messaging import PubMaster, SubMaster -from cereal.visionipc import VisionIpcClient, VisionStreamType, VisionBuf +from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf from openpilot.common.swaglog import cloudlog from openpilot.common.params import Params from openpilot.common.realtime import set_realtime_priority diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 526fd4d2189477..3ac80aad91110a 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -8,7 +8,7 @@ from pathlib import Path from setproctitle import setproctitle from cereal.messaging import PubMaster, SubMaster -from cereal.visionipc import VisionIpcClient, VisionStreamType, VisionBuf +from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf from openpilot.common.swaglog import cloudlog from openpilot.common.params import Params from openpilot.common.filter_simple import FirstOrderFilter diff --git a/selfdrive/modeld/models/commonmodel.pxd b/selfdrive/modeld/models/commonmodel.pxd index 57b79aeafb20a3..7c3eb0b3d7e4d5 100644 --- a/selfdrive/modeld/models/commonmodel.pxd +++ b/selfdrive/modeld/models/commonmodel.pxd @@ -1,6 +1,6 @@ # distutils: language = c++ -from cereal.visionipc.visionipc cimport cl_device_id, cl_context, cl_mem +from msgq.visionipc.visionipc cimport cl_device_id, cl_context, cl_mem cdef extern from "common/mat.h": cdef struct mat3: diff --git a/selfdrive/modeld/models/commonmodel_pyx.pxd b/selfdrive/modeld/models/commonmodel_pyx.pxd index 97e391458892ad..0bb798625be28d 100644 --- a/selfdrive/modeld/models/commonmodel_pyx.pxd +++ b/selfdrive/modeld/models/commonmodel_pyx.pxd @@ -1,7 +1,7 @@ # distutils: language = c++ -from cereal.visionipc.visionipc cimport cl_mem -from cereal.visionipc.visionipc_pyx cimport CLContext as BaseCLContext +from msgq.visionipc.visionipc cimport cl_mem +from msgq.visionipc.visionipc_pyx cimport CLContext as BaseCLContext cdef class CLContext(BaseCLContext): pass diff --git a/selfdrive/modeld/models/commonmodel_pyx.pyx b/selfdrive/modeld/models/commonmodel_pyx.pyx index bc59b7a233f4bc..e292bb0d2da6c5 100644 --- a/selfdrive/modeld/models/commonmodel_pyx.pyx +++ b/selfdrive/modeld/models/commonmodel_pyx.pyx @@ -5,8 +5,8 @@ import numpy as np cimport numpy as cnp from libc.string cimport memcpy -from cereal.visionipc.visionipc cimport cl_mem -from cereal.visionipc.visionipc_pyx cimport VisionBuf, CLContext as BaseCLContext +from msgq.visionipc.visionipc cimport cl_mem +from msgq.visionipc.visionipc_pyx cimport VisionBuf, CLContext as BaseCLContext from .commonmodel cimport CL_DEVICE_TYPE_DEFAULT, cl_get_device_id, cl_create_context from .commonmodel cimport mat3, sigmoid as cppSigmoid, ModelFrame as cppModelFrame diff --git a/selfdrive/modeld/runners/snpemodel.pxd b/selfdrive/modeld/runners/snpemodel.pxd index 1f928da3325efe..a911b43584d300 100644 --- a/selfdrive/modeld/runners/snpemodel.pxd +++ b/selfdrive/modeld/runners/snpemodel.pxd @@ -2,7 +2,7 @@ from libcpp.string cimport string -from cereal.visionipc.visionipc cimport cl_context +from msgq.visionipc.visionipc cimport cl_context cdef extern from "selfdrive/modeld/runners/snpemodel.h": cdef cppclass SNPEModel: diff --git a/selfdrive/modeld/runners/thneedmodel.pxd b/selfdrive/modeld/runners/thneedmodel.pxd index 90af972865d0e0..79e24dbdd62518 100644 --- a/selfdrive/modeld/runners/thneedmodel.pxd +++ b/selfdrive/modeld/runners/thneedmodel.pxd @@ -2,7 +2,7 @@ from libcpp.string cimport string -from cereal.visionipc.visionipc cimport cl_context +from msgq.visionipc.visionipc cimport cl_context cdef extern from "selfdrive/modeld/runners/thneedmodel.h": cdef cppclass ThneedModel: diff --git a/selfdrive/modeld/tests/test_modeld.py b/selfdrive/modeld/tests/test_modeld.py index 413057931359b4..1e19a591bf7564 100644 --- a/selfdrive/modeld/tests/test_modeld.py +++ b/selfdrive/modeld/tests/test_modeld.py @@ -2,7 +2,7 @@ import random import cereal.messaging as messaging -from cereal.visionipc import VisionIpcServer, VisionStreamType +from msgq.visionipc import VisionIpcServer, VisionStreamType from openpilot.common.transformations.camera import DEVICE_CAMERAS from openpilot.common.realtime import DT_MDL from openpilot.selfdrive.car.car_helpers import write_car_param diff --git a/selfdrive/navd/SConscript b/selfdrive/navd/SConscript index 5a049f18bad520..7f5d948b77557d 100644 --- a/selfdrive/navd/SConscript +++ b/selfdrive/navd/SConscript @@ -1,7 +1,7 @@ -Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'transformations') +Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'transformations', 'socketmaster') map_env = qt_env.Clone() -libs = ['qt_widgets', 'qt_util', 'QMapLibre', common, messaging, cereal, visionipc, transformations, +libs = ['qt_widgets', 'qt_util', 'QMapLibre', common, socketmaster, messaging, cereal, visionipc, transformations, 'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread', 'json11'] + map_env["LIBS"] if arch == 'larch64': libs.append(':libEGL_mesa.so.0') diff --git a/selfdrive/navd/map_renderer.h b/selfdrive/navd/map_renderer.h index 7a3d2c316b325d..956c1d54bc6a43 100644 --- a/selfdrive/navd/map_renderer.h +++ b/selfdrive/navd/map_renderer.h @@ -12,7 +12,7 @@ #include #include -#include "cereal/visionipc/visionipc_server.h" +#include "msgq/visionipc/visionipc_server.h" #include "cereal/messaging/messaging.h" diff --git a/selfdrive/navd/tests/test_map_renderer.py b/selfdrive/navd/tests/test_map_renderer.py index 6d66528ae3bb6a..04363883b2c0f4 100644 --- a/selfdrive/navd/tests/test_map_renderer.py +++ b/selfdrive/navd/tests/test_map_renderer.py @@ -8,7 +8,7 @@ import cereal.messaging as messaging from typing import Any -from cereal.visionipc import VisionIpcClient, VisionStreamType +from msgq.visionipc import VisionIpcClient, VisionStreamType from openpilot.common.mock.generators import LLK_DECIMATION, LOCATION1, LOCATION2, generate_liveLocationKalman from openpilot.selfdrive.test.helpers import with_processes diff --git a/selfdrive/pandad/SConscript b/selfdrive/pandad/SConscript index 5e70736515b67a..48187c43250583 100644 --- a/selfdrive/pandad/SConscript +++ b/selfdrive/pandad/SConscript @@ -1,6 +1,6 @@ -Import('env', 'envCython', 'common', 'cereal', 'messaging') +Import('env', 'envCython', 'common', 'cereal', 'messaging', 'socketmaster') -libs = ['usb-1.0', common, cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj'] +libs = ['usb-1.0', common, cereal, messaging, socketmaster, 'pthread', 'zmq', 'capnp', 'kj'] panda = env.Library('panda', ['panda.cc', 'panda_comms.cc', 'spi.cc']) env.Program('pandad', ['main.cc', 'pandad.cc'], LIBS=[panda] + libs) diff --git a/selfdrive/test/process_replay/process_replay.py b/selfdrive/test/process_replay/process_replay.py index b325e6e69353bd..739efcb985a75b 100755 --- a/selfdrive/test/process_replay/process_replay.py +++ b/selfdrive/test/process_replay/process_replay.py @@ -16,7 +16,7 @@ import cereal.messaging as messaging from cereal import car from cereal.services import SERVICE_LIST -from cereal.visionipc import VisionIpcServer, get_endpoint_name as vipc_get_endpoint_name +from msgq.visionipc import VisionIpcServer, get_endpoint_name as vipc_get_endpoint_name from openpilot.common.params import Params from openpilot.common.prefix import OpenpilotPrefix from openpilot.common.timeout import Timeout diff --git a/selfdrive/test/process_replay/vision_meta.py b/selfdrive/test/process_replay/vision_meta.py index 9bfe214c1e19c9..12deb587244134 100644 --- a/selfdrive/test/process_replay/vision_meta.py +++ b/selfdrive/test/process_replay/vision_meta.py @@ -1,5 +1,5 @@ from collections import namedtuple -from cereal.visionipc import VisionStreamType +from msgq.visionipc import VisionStreamType from openpilot.common.realtime import DT_MDL, DT_DMON from openpilot.common.transformations.camera import DEVICE_CAMERAS diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index 13b6998de2595a..5bd33f35f034e6 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -1,9 +1,9 @@ import os import json Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', - 'cereal', 'transformations') + 'cereal', 'transformations', 'socketmaster') -base_libs = [common, messaging, cereal, visionipc, transformations, 'zmq', +base_libs = [common, socketmaster, messaging, cereal, visionipc, transformations, 'zmq', 'capnp', 'kj', 'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + qt_env["LIBS"] if arch == 'larch64': diff --git a/selfdrive/ui/qt/widgets/cameraview.h b/selfdrive/ui/qt/widgets/cameraview.h index c97038cf436e6d..85ec49873579dc 100644 --- a/selfdrive/ui/qt/widgets/cameraview.h +++ b/selfdrive/ui/qt/widgets/cameraview.h @@ -22,7 +22,7 @@ #include #endif -#include "cereal/visionipc/visionipc_client.h" +#include "msgq/visionipc/visionipc_client.h" #include "system/camerad/cameras/camera_common.h" #include "selfdrive/ui/ui.h" diff --git a/selfdrive/ui/tests/test_ui/run.py b/selfdrive/ui/tests/test_ui/run.py index 871333563d4379..1d33c103a4d02b 100644 --- a/selfdrive/ui/tests/test_ui/run.py +++ b/selfdrive/ui/tests/test_ui/run.py @@ -10,7 +10,7 @@ import time from cereal import messaging, car, log -from cereal.visionipc import VisionIpcServer, VisionStreamType +from msgq.visionipc import VisionIpcServer, VisionStreamType from cereal.messaging import SubMaster, PubMaster from openpilot.common.mock import mock_messages diff --git a/system/camerad/SConscript b/system/camerad/SConscript index 8f19e7ee19f879..67530c57e400c5 100644 --- a/system/camerad/SConscript +++ b/system/camerad/SConscript @@ -1,6 +1,6 @@ -Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc') +Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'socketmaster') -libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', 'yuv', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon, 'atomic'] +libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', 'yuv', cereal, socketmaster, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon, 'atomic'] camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/camera_util.cc', 'sensors/ar0231.cc', 'sensors/ox03c10.cc', 'sensors/os04c10.cc']) diff --git a/system/camerad/cameras/camera_common.h b/system/camerad/cameras/camera_common.h index 587968fccbaa53..555362ab8bc7ab 100644 --- a/system/camerad/cameras/camera_common.h +++ b/system/camerad/cameras/camera_common.h @@ -5,7 +5,7 @@ #include #include "cereal/messaging/messaging.h" -#include "cereal/visionipc/visionipc_server.h" +#include "msgq/visionipc/visionipc_server.h" #include "common/queue.h" #include "common/util.h" diff --git a/system/camerad/snapshot/snapshot.py b/system/camerad/snapshot/snapshot.py index 381240212b7ebe..4ba38c1df4b19f 100755 --- a/system/camerad/snapshot/snapshot.py +++ b/system/camerad/snapshot/snapshot.py @@ -6,7 +6,7 @@ from PIL import Image import cereal.messaging as messaging -from cereal.visionipc import VisionIpcClient, VisionStreamType +from msgq.visionipc import VisionIpcClient, VisionStreamType from openpilot.common.params import Params from openpilot.common.realtime import DT_MDL from openpilot.system.hardware import PC diff --git a/system/hardware/base.h b/system/hardware/base.h index dc2282a93ae0b8..ca24633a182a91 100644 --- a/system/hardware/base.h +++ b/system/hardware/base.h @@ -5,7 +5,7 @@ #include #include -#include "cereal/messaging/messaging.h" +#include "cereal/gen/cpp/log.capnp.h" // no-op base hw class class HardwareNone { diff --git a/system/logcatd/SConscript b/system/logcatd/SConscript index 6bd7c6ff3e8561..cd34e395664179 100644 --- a/system/logcatd/SConscript +++ b/system/logcatd/SConscript @@ -1,3 +1,3 @@ -Import('env', 'cereal', 'messaging', 'common') +Import('env', 'cereal', 'messaging', 'common', 'socketmaster') -env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, messaging, common, 'zmq', 'capnp', 'kj', 'systemd', 'json11']) +env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, socketmaster, messaging, common, 'zmq', 'capnp', 'kj', 'systemd', 'json11']) diff --git a/system/loggerd/SConscript b/system/loggerd/SConscript index d4f52fb5f1a394..bcf941655aa17a 100644 --- a/system/loggerd/SConscript +++ b/system/loggerd/SConscript @@ -1,6 +1,6 @@ -Import('env', 'arch', 'cereal', 'messaging', 'common', 'visionipc') +Import('env', 'arch', 'cereal', 'messaging', 'common', 'visionipc', 'socketmaster') -libs = [common, cereal, messaging, visionipc, +libs = [common, cereal, socketmaster, messaging, visionipc, 'zmq', 'capnp', 'kj', 'z', 'avformat', 'avcodec', 'swscale', 'avutil', 'yuv', 'OpenCL', 'pthread'] diff --git a/system/loggerd/encoder/encoder.h b/system/loggerd/encoder/encoder.h index 75183611b32f3e..72848609efbf4b 100644 --- a/system/loggerd/encoder/encoder.h +++ b/system/loggerd/encoder/encoder.h @@ -7,7 +7,7 @@ #include #include "cereal/messaging/messaging.h" -#include "cereal/visionipc/visionipc.h" +#include "msgq/visionipc/visionipc.h" #include "common/queue.h" #include "system/camerad/cameras/camera_common.h" #include "system/loggerd/loggerd.h" diff --git a/system/loggerd/loggerd.cc b/system/loggerd/loggerd.cc index 3c0ffc166796a0..a324479fb5de8f 100644 --- a/system/loggerd/loggerd.cc +++ b/system/loggerd/loggerd.cc @@ -215,7 +215,7 @@ void loggerd_thread() { const bool encoder = util::ends_with(it.name, "EncodeData"); const bool livestream_encoder = util::starts_with(it.name, "livestream"); if (!it.should_log && (!encoder || livestream_encoder)) continue; - LOGD("logging %s (on port %d)", it.name.c_str(), it.port); + LOGD("logging %s", it.name.c_str()); SubSocket * sock = SubSocket::create(ctx.get(), it.name); assert(sock != NULL); diff --git a/system/loggerd/loggerd.h b/system/loggerd/loggerd.h index 5eef1618342a78..7c80ba51a214cd 100644 --- a/system/loggerd/loggerd.h +++ b/system/loggerd/loggerd.h @@ -4,7 +4,7 @@ #include "cereal/messaging/messaging.h" #include "cereal/services.h" -#include "cereal/visionipc/visionipc_client.h" +#include "msgq/visionipc/visionipc_client.h" #include "system/camerad/cameras/camera_common.h" #include "system/hardware/hw.h" #include "common/params.h" diff --git a/system/loggerd/tests/test_loggerd.py b/system/loggerd/tests/test_loggerd.py index 882ce99ed7a878..6a24540acf7510 100644 --- a/system/loggerd/tests/test_loggerd.py +++ b/system/loggerd/tests/test_loggerd.py @@ -22,7 +22,7 @@ from openpilot.system.version import get_version from openpilot.tools.lib.helpers import RE from openpilot.tools.lib.logreader import LogReader -from cereal.visionipc import VisionIpcServer, VisionStreamType +from msgq.visionipc import VisionIpcServer, VisionStreamType from openpilot.common.transformations.camera import DEVICE_CAMERAS SentinelType = log.Sentinel.SentinelType diff --git a/system/proclogd/SConscript b/system/proclogd/SConscript index 1f4b7670112260..05cdb73c65934e 100644 --- a/system/proclogd/SConscript +++ b/system/proclogd/SConscript @@ -1,5 +1,5 @@ -Import('env', 'cereal', 'messaging', 'common') -libs = [cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj', 'common', 'zmq', 'json11'] +Import('env', 'cereal', 'messaging', 'common', 'socketmaster') +libs = [cereal, socketmaster, messaging, 'pthread', 'zmq', 'capnp', 'kj', 'common', 'zmq', 'json11'] env.Program('proclogd', ['main.cc', 'proclog.cc'], LIBS=libs) if GetOption('extras'): diff --git a/system/sensord/SConscript b/system/sensord/SConscript index 7974eb07aec162..1808a315415278 100644 --- a/system/sensord/SConscript +++ b/system/sensord/SConscript @@ -1,4 +1,4 @@ -Import('env', 'arch', 'common', 'cereal', 'messaging') +Import('env', 'arch', 'common', 'cereal', 'messaging', 'socketmaster') sensors = [ 'sensors/i2c_sensor.cc', @@ -11,7 +11,7 @@ sensors = [ 'sensors/lsm6ds3_temp.cc', 'sensors/mmc5603nj_magn.cc', ] -libs = [common, cereal, messaging, 'capnp', 'zmq', 'kj', 'pthread'] +libs = [common, socketmaster, cereal, messaging, 'capnp', 'zmq', 'kj', 'pthread'] if arch == "larch64": libs.append('i2c') env.Program('sensord', ['sensors_qcom2.cc'] + sensors, LIBS=libs) diff --git a/system/ubloxd/SConscript b/system/ubloxd/SConscript index 67d9856dad8a11..d6bea572386492 100644 --- a/system/ubloxd/SConscript +++ b/system/ubloxd/SConscript @@ -1,6 +1,6 @@ -Import('env', 'common', 'cereal', 'messaging') +Import('env', 'common', 'cereal', 'messaging', 'socketmaster') -loc_libs = [cereal, messaging, 'zmq', common, 'capnp', 'kj', 'kaitai', 'pthread'] +loc_libs = [cereal, messaging, socketmaster, 'zmq', common, 'capnp', 'kj', 'kaitai', 'pthread'] if GetOption('kaitai'): generated = Dir('generated').srcnode().abspath diff --git a/tools/cabana/SConscript b/tools/cabana/SConscript index 5bcfdb175aa6b4..4ce4c27a05ecf3 100644 --- a/tools/cabana/SConscript +++ b/tools/cabana/SConscript @@ -1,7 +1,7 @@ -Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'replay_lib', 'cereal', 'widgets') +Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'replay_lib', 'cereal', 'widgets', 'socketmaster') base_frameworks = qt_env['FRAMEWORKS'] -base_libs = [common, messaging, cereal, visionipc, 'qt_util', 'zmq', 'capnp', 'kj', 'm', 'ssl', 'crypto', 'pthread'] + qt_env["LIBS"] +base_libs = [common, messaging, cereal, visionipc, socketmaster, 'qt_util', 'zmq', 'capnp', 'kj', 'm', 'ssl', 'crypto', 'pthread'] + qt_env["LIBS"] if arch == "Darwin": base_frameworks.append('OpenCL') diff --git a/tools/camerastream/compressed_vipc.py b/tools/camerastream/compressed_vipc.py index df9de096f0accb..ea427816ef764d 100755 --- a/tools/camerastream/compressed_vipc.py +++ b/tools/camerastream/compressed_vipc.py @@ -8,7 +8,7 @@ import time import cereal.messaging as messaging -from cereal.visionipc import VisionIpcServer, VisionStreamType +from msgq.visionipc import VisionIpcServer, VisionStreamType V4L2_BUF_FLAG_KEYFRAME = 8 diff --git a/tools/replay/SConscript b/tools/replay/SConscript index 5d88f560be39f5..813f7808f36d61 100644 --- a/tools/replay/SConscript +++ b/tools/replay/SConscript @@ -1,7 +1,7 @@ -Import('env', 'qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal') +Import('env', 'qt_env', 'arch', 'common', 'messaging', 'visionipc', 'cereal', 'socketmaster') base_frameworks = qt_env['FRAMEWORKS'] -base_libs = [common, messaging, cereal, visionipc, 'zmq', +base_libs = [common, socketmaster, messaging, cereal, visionipc, 'zmq', 'capnp', 'kj', 'm', 'ssl', 'crypto', 'pthread', 'qt_util'] + qt_env["LIBS"] if arch == "Darwin": diff --git a/tools/replay/camera.h b/tools/replay/camera.h index 77a6293ec64775..21c3d98dcf2550 100644 --- a/tools/replay/camera.h +++ b/tools/replay/camera.h @@ -5,7 +5,7 @@ #include #include -#include "cereal/visionipc/visionipc_server.h" +#include "msgq/visionipc/visionipc_server.h" #include "common/queue.h" #include "tools/replay/framereader.h" #include "tools/replay/logreader.h" diff --git a/tools/replay/framereader.h b/tools/replay/framereader.h index b9abefb7c3a9e4..7a4d024aa20f23 100644 --- a/tools/replay/framereader.h +++ b/tools/replay/framereader.h @@ -3,7 +3,7 @@ #include #include -#include "cereal/visionipc/visionbuf.h" +#include "msgq/visionipc/visionbuf.h" #include "system/camerad/cameras/camera_common.h" #include "tools/replay/filereader.h" diff --git a/tools/replay/ui.py b/tools/replay/ui.py index 31e4fff74d25f1..a790f14ff0ef6e 100755 --- a/tools/replay/ui.py +++ b/tools/replay/ui.py @@ -18,7 +18,7 @@ maybe_update_radar_points, plot_lead, plot_model, pygame_modules_have_loaded) -from cereal.visionipc import VisionIpcClient, VisionStreamType +from msgq.visionipc import VisionIpcClient, VisionStreamType os.environ['BASEDIR'] = BASEDIR diff --git a/tools/sim/Dockerfile.sim b/tools/sim/Dockerfile.sim index cb573b33f2c757..bd83c119b0ae59 100644 --- a/tools/sim/Dockerfile.sim +++ b/tools/sim/Dockerfile.sim @@ -26,6 +26,7 @@ COPY ./rednose_repo/site_scons ${OPENPILOT_PATH}/rednose_repo/site_scons COPY ./common ${OPENPILOT_PATH}/common COPY ./opendbc ${OPENPILOT_PATH}/opendbc COPY ./cereal ${OPENPILOT_PATH}/cereal +COPY ./msgq ${OPENPILOT_PATH}/msgq COPY ./panda ${OPENPILOT_PATH}/panda COPY ./selfdrive ${OPENPILOT_PATH}/selfdrive COPY ./system ${OPENPILOT_PATH}/system diff --git a/tools/sim/lib/camerad.py b/tools/sim/lib/camerad.py index 95769dd44e5dba..b58cbeaaf26334 100644 --- a/tools/sim/lib/camerad.py +++ b/tools/sim/lib/camerad.py @@ -3,7 +3,7 @@ import pyopencl as cl import pyopencl.array as cl_array -from cereal.visionipc import VisionIpcServer, VisionStreamType +from msgq.visionipc import VisionIpcServer, VisionStreamType from cereal import messaging from openpilot.common.basedir import BASEDIR diff --git a/tools/webcam/camerad.py b/tools/webcam/camerad.py index ce33473f9ace72..d0b879a0860a1f 100755 --- a/tools/webcam/camerad.py +++ b/tools/webcam/camerad.py @@ -3,7 +3,7 @@ import os from collections import namedtuple -from cereal.visionipc import VisionIpcServer, VisionStreamType +from msgq.visionipc import VisionIpcServer, VisionStreamType from cereal import messaging from openpilot.tools.webcam.camera import Camera From 83209e0860f61aa383fbe1cc569ace996edcd13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Thu, 6 Jun 2024 15:03:19 -0700 Subject: [PATCH 04/12] Msgq cleanups (#32643) * Dead cereal stuff * Dead code * also dead * More cleanup --- cereal/.pre-commit-config.yaml | 47 ---- cereal/Dockerfile | 54 ----- cereal/LICENSE | 7 - cereal/codecov.yml | 8 - cereal/messaging/demo.cc | 50 ---- cereal/messaging/demo.py | 29 --- cereal/messaging/msgq.md | 54 ----- cereal/messaging/msgq_tests.cc | 394 -------------------------------- cereal/messaging/test_runner.cc | 2 - msgq | 2 +- 10 files changed, 1 insertion(+), 646 deletions(-) delete mode 100644 cereal/.pre-commit-config.yaml delete mode 100644 cereal/Dockerfile delete mode 100644 cereal/LICENSE delete mode 100644 cereal/codecov.yml delete mode 100644 cereal/messaging/demo.cc delete mode 100644 cereal/messaging/demo.py delete mode 100644 cereal/messaging/msgq.md delete mode 100644 cereal/messaging/msgq_tests.cc delete mode 100644 cereal/messaging/test_runner.cc diff --git a/cereal/.pre-commit-config.yaml b/cereal/.pre-commit-config.yaml deleted file mode 100644 index 03360e0f9fa677..00000000000000 --- a/cereal/.pre-commit-config.yaml +++ /dev/null @@ -1,47 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 - hooks: - - id: check-ast - - id: check-yaml - - id: check-executables-have-shebangs - - id: check-shebang-scripts-are-executable -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 - hooks: - - id: mypy -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.5 - hooks: - - id: ruff -- repo: local - hooks: - - id: cppcheck - name: cppcheck - entry: cppcheck - language: system - types: [c++] - exclude: '^(messaging/msgq_tests.cc|messaging/test_runner.cc)' - args: - - --error-exitcode=1 - - --inline-suppr - - --language=c++ - - --force - - --quiet - - -j4 -- repo: https://github.com/cpplint/cpplint - rev: 1.6.1 - hooks: - - id: cpplint - args: - - --quiet - - --counting=detailed - - --linelength=240 - - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces -- repo: https://github.com/codespell-project/codespell - rev: v2.2.6 - hooks: - - id: codespell - args: - - -L ned - - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US diff --git a/cereal/Dockerfile b/cereal/Dockerfile deleted file mode 100644 index d2e3ab591be74c..00000000000000 --- a/cereal/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -FROM ubuntu:24.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ - build-essential \ - ca-certificates \ - capnproto \ - clang \ - cppcheck \ - curl \ - git \ - libbz2-dev \ - libcapnp-dev \ - libclang-rt-dev \ - libffi-dev \ - liblzma-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libzmq3-dev \ - llvm \ - make \ - cmake \ - ocl-icd-opencl-dev \ - opencl-headers \ - python3-dev \ - python3-pip \ - tk-dev \ - wget \ - xz-utils \ - zlib1g-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy - -WORKDIR /project/ -RUN cd /tmp/ && \ - git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \ - cd Catch2 && \ - mv single_include/catch2/ /project/ && \ - cd .. \ - rm -rf Catch2 - -WORKDIR /project/cereal - -ENV PYTHONPATH=/project - -COPY . . -RUN rm -rf .git && \ - scons -c && scons -j$(nproc) diff --git a/cereal/LICENSE b/cereal/LICENSE deleted file mode 100644 index f1fd199c632b14..00000000000000 --- a/cereal/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright (c) 2020, Comma.ai, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/cereal/codecov.yml b/cereal/codecov.yml deleted file mode 100644 index 83427c3ee8b6e5..00000000000000 --- a/cereal/codecov.yml +++ /dev/null @@ -1,8 +0,0 @@ -comment: false -coverage: - status: - project: - default: - informational: true - patch: off - diff --git a/cereal/messaging/demo.cc b/cereal/messaging/demo.cc deleted file mode 100644 index 5b4d24468279d3..00000000000000 --- a/cereal/messaging/demo.cc +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include -#include -#include - -#include "cereal/messaging/messaging.h" -#include "cereal/messaging/impl_zmq.h" - -#define MSGS 1e5 - -int main() { - Context * c = Context::create(); - SubSocket * sub_sock = SubSocket::create(c, "controlsState"); - PubSocket * pub_sock = PubSocket::create(c, "controlsState"); - - char data[8]; - - Poller * poller = Poller::create({sub_sock}); - - auto start = std::chrono::steady_clock::now(); - - for (uint64_t i = 0; i < MSGS; i++){ - *(uint64_t*)data = i; - pub_sock->send(data, 8); - - auto r = poller->poll(100); - - for (auto p : r){ - Message * m = p->receive(); - uint64_t ii = *(uint64_t*)m->getData(); - assert(i == ii); - delete m; - } - } - - - auto end = std::chrono::steady_clock::now(); - double elapsed = std::chrono::duration_cast(end - start).count() / 1e9; - double throughput = ((double) MSGS / (double) elapsed); - std::cout << throughput << " msg/s" << std::endl; - - delete poller; - delete sub_sock; - delete pub_sock; - delete c; - - - return 0; -} diff --git a/cereal/messaging/demo.py b/cereal/messaging/demo.py deleted file mode 100644 index e4850e32b9cb9c..00000000000000 --- a/cereal/messaging/demo.py +++ /dev/null @@ -1,29 +0,0 @@ -import time - -from messaging_pyx import Context, Poller, SubSocket, PubSocket - -MSGS = 1e5 - -if __name__ == "__main__": - c = Context() - sub_sock = SubSocket() - pub_sock = PubSocket() - - sub_sock.connect(c, "controlsState") - pub_sock.connect(c, "controlsState") - - poller = Poller() - poller.registerSocket(sub_sock) - - t = time.time() - for i in range(int(MSGS)): - bts = i.to_bytes(4, 'little') - pub_sock.send(bts) - - for s in poller.poll(100): - dat = s.receive() - ii = int.from_bytes(dat, 'little') - assert(i == ii) - - dt = time.time() - t - print("%.1f msg/s" % (MSGS / dt)) diff --git a/cereal/messaging/msgq.md b/cereal/messaging/msgq.md deleted file mode 100644 index 34fe35682fb542..00000000000000 --- a/cereal/messaging/msgq.md +++ /dev/null @@ -1,54 +0,0 @@ -# MSGQ: A lock free single producer multi consumer message queue - -## What is MSGQ? -MSGQ is a system to pass messages from a single producer to multiple consumers. All the consumers need to be able to receive all the messages. It is designed to be a high performance replacement for ZMQ-like SUB/PUB patterns. It uses a ring buffer in shared memory to efficiently read and write data. Each read requires a copy. Writing can be done without a copy, as long as the size of the data is known in advance. - -## Storage -The storage for the queue consists of an area of metadata, and the actual buffer. The metadata contains: - -1. A counter to the number of readers that are active -2. A pointer to the head of the queue for writing. From now on referred to as *write pointer* -3. A cycle counter for the writer. This counter is incremented when the writer wraps around -4. N pointers, pointing to the current read position for all the readers. From now on referred to as *read pointer* -5. N counters, counting the number of cycles for all the readers -6. N booleans, indicating validity for all the readers. From now on referred to as *validity flag* - -The counter and the pointer are both 32 bit values, packed into 64 bit so they can be read and written atomically. - -The data buffer is a ring buffer. All messages are prefixed by an 8 byte size field, followed by the data. A size of -1 indicates a wrap-around, and means the next message is stored at the beginning of the buffer. - - -## Writing -Writing involves the following steps: - -1. Check if the area that is to be written overlaps with any of the read pointers, mark those readers as invalid by clearing the validity flag. -2. Write the message -3. Increase the write pointer by the size of the message - -In case there is not enough space at the end of the buffer, a special empty message with a prefix of -1 is written. The cycle counter is incremented by one. In this case step 1 will check there are no read pointers pointing to the remainder of the buffer. Then another write cycle will start with the actual message. - -There always needs to be 8 bytes of empty space at the end of the buffer. By doing this there is always space to write the -1. - -## Reset reader -When the reader is lagging too much behind the read pointer becomes invalid and no longer points to the beginning of a valid message. To reset a reader to the current write pointer, the following steps are performed: - -1. Set valid flag -2. Set read cycle counter to that of the writer -3. Set read pointer to write pointer - -## Reading -Reading involves the following steps: - -1. Read the size field at the current read pointer -2. Read the validity flag -3. Copy the data out of the buffer -4. Increase the read pointer by the size of the message -5. Check the validity flag again - -Before starting the copy, the valid flag is checked. This is to prevent a race condition where the size prefix was invalid, and the read could read outside of the buffer. Make sure that step 1 and 2 are not reordered by your compiler or CPU. - -If a writer overwrites the data while it's being copied out, the data will be invalid. Therefore the validity flag is also checked after reading it. The order of step 4 and 5 does not matter. - -If at steps 2 or 5 the validity flag is not set, the reader is reset. Any data that was already read is discarded. After the reader is reset, the reading starts from the beginning. - -If a message with size -1 is encountered, step 3 and 4 are replaced by increasing the cycle counter and setting the read pointer to the beginning of the buffer. After that another read is performed. diff --git a/cereal/messaging/msgq_tests.cc b/cereal/messaging/msgq_tests.cc deleted file mode 100644 index d33facbc047531..00000000000000 --- a/cereal/messaging/msgq_tests.cc +++ /dev/null @@ -1,394 +0,0 @@ -#include "catch2/catch.hpp" -#include "cereal/messaging/msgq.h" - -TEST_CASE("ALIGN"){ - REQUIRE(ALIGN(0) == 0); - REQUIRE(ALIGN(1) == 8); - REQUIRE(ALIGN(7) == 8); - REQUIRE(ALIGN(8) == 8); - REQUIRE(ALIGN(99999) == 100000); -} - -TEST_CASE("msgq_msg_init_size"){ - const size_t msg_size = 30; - msgq_msg_t msg; - - msgq_msg_init_size(&msg, msg_size); - REQUIRE(msg.size == msg_size); - - msgq_msg_close(&msg); -} - -TEST_CASE("msgq_msg_init_data"){ - const size_t msg_size = 30; - char * data = new char[msg_size]; - - for (size_t i = 0; i < msg_size; i++){ - data[i] = i; - } - - msgq_msg_t msg; - msgq_msg_init_data(&msg, data, msg_size); - - REQUIRE(msg.size == msg_size); - REQUIRE(memcmp(msg.data, data, msg_size) == 0); - - delete[] data; - msgq_msg_close(&msg); -} - - -TEST_CASE("msgq_init_subscriber"){ - remove("/dev/shm/test_queue"); - msgq_queue_t q; - msgq_new_queue(&q, "test_queue", 1024); - REQUIRE(*q.num_readers == 0); - - q.reader_id = 1; - *q.read_valids[0] = false; - *q.read_pointers[0] = ((uint64_t)1 << 32); - - *q.write_pointer = 255; - - msgq_init_subscriber(&q); - REQUIRE(q.read_conflate == false); - REQUIRE(*q.read_valids[0] == true); - REQUIRE((*q.read_pointers[0] >> 32) == 0); - REQUIRE((*q.read_pointers[0] & 0xFFFFFFFF) == 255); -} - -TEST_CASE("msgq_msg_send first message"){ - remove("/dev/shm/test_queue"); - msgq_queue_t q; - msgq_new_queue(&q, "test_queue", 1024); - msgq_init_publisher(&q); - - REQUIRE(*q.write_pointer == 0); - - size_t msg_size = 128; - - SECTION("Aligned message size"){ - } - SECTION("Unaligned message size"){ - msg_size--; - } - - char * data = new char[msg_size]; - - for (size_t i = 0; i < msg_size; i++){ - data[i] = i; - } - - msgq_msg_t msg; - msgq_msg_init_data(&msg, data, msg_size); - - - msgq_msg_send(&msg, &q); - REQUIRE(*(int64_t*)q.data == msg_size); // Check size tag - REQUIRE(*q.write_pointer == 128 + sizeof(int64_t)); - REQUIRE(memcmp(q.data + sizeof(int64_t), data, msg_size) == 0); - - delete[] data; - msgq_msg_close(&msg); -} - -TEST_CASE("msgq_msg_send test wraparound"){ - remove("/dev/shm/test_queue"); - msgq_queue_t q; - msgq_new_queue(&q, "test_queue", 1024); - msgq_init_publisher(&q); - - REQUIRE((*q.write_pointer & 0xFFFFFFFF) == 0); - REQUIRE((*q.write_pointer >> 32) == 0); - - const size_t msg_size = 120; - msgq_msg_t msg; - msgq_msg_init_size(&msg, msg_size); - - for (int i = 0; i < 8; i++) { - msgq_msg_send(&msg, &q); - } - // Check 8th message was written at the beginning - REQUIRE((*q.write_pointer & 0xFFFFFFFF) == msg_size + sizeof(int64_t)); - - // Check cycle count - REQUIRE((*q.write_pointer >> 32) == 1); - - // Check wraparound tag - char * tag_location = q.data; - tag_location += 7 * (msg_size + sizeof(int64_t)); - REQUIRE(*(int64_t*)tag_location == -1); - - msgq_msg_close(&msg); -} - -TEST_CASE("msgq_msg_recv test wraparound"){ - remove("/dev/shm/test_queue"); - msgq_queue_t q_pub, q_sub; - msgq_new_queue(&q_pub, "test_queue", 1024); - msgq_new_queue(&q_sub, "test_queue", 1024); - - msgq_init_publisher(&q_pub); - msgq_init_subscriber(&q_sub); - - REQUIRE((*q_pub.write_pointer >> 32) == 0); - REQUIRE((*q_sub.read_pointers[0] >> 32) == 0); - - const size_t msg_size = 120; - msgq_msg_t msg1; - msgq_msg_init_size(&msg1, msg_size); - - - SECTION("Check cycle counter after reset") { - for (int i = 0; i < 8; i++) { - msgq_msg_send(&msg1, &q_pub); - } - - msgq_msg_t msg2; - msgq_msg_recv(&msg2, &q_sub); - REQUIRE(msg2.size == 0); // Reader had to reset - msgq_msg_close(&msg2); - } - SECTION("Check cycle counter while keeping up with writer") { - for (int i = 0; i < 8; i++) { - msgq_msg_send(&msg1, &q_pub); - - msgq_msg_t msg2; - msgq_msg_recv(&msg2, &q_sub); - REQUIRE(msg2.size > 0); - msgq_msg_close(&msg2); - } - } - - REQUIRE((*q_sub.read_pointers[0] >> 32) == 1); - msgq_msg_close(&msg1); -} - -TEST_CASE("msgq_msg_send test invalidation"){ - remove("/dev/shm/test_queue"); - msgq_queue_t q_pub, q_sub; - msgq_new_queue(&q_pub, "test_queue", 1024); - msgq_new_queue(&q_sub, "test_queue", 1024); - - msgq_init_publisher(&q_pub); - msgq_init_subscriber(&q_sub); - *q_sub.write_pointer = (uint64_t)1 << 32; - - REQUIRE(*q_sub.read_valids[0] == true); - - SECTION("read pointer in tag"){ - *q_sub.read_pointers[0] = 0; - } - SECTION("read pointer in data section"){ - *q_sub.read_pointers[0] = 64; - } - SECTION("read pointer in wraparound section"){ - *q_pub.write_pointer = ((uint64_t)1 << 32) | 1000; // Writer is one cycle ahead - *q_sub.read_pointers[0] = 1020; - } - - msgq_msg_t msg; - msgq_msg_init_size(&msg, 128); - msgq_msg_send(&msg, &q_pub); - - REQUIRE(*q_sub.read_valids[0] == false); - - msgq_msg_close(&msg); -} - -TEST_CASE("msgq_init_subscriber init 2 subscribers"){ - remove("/dev/shm/test_queue"); - msgq_queue_t q1, q2; - msgq_new_queue(&q1, "test_queue", 1024); - msgq_new_queue(&q2, "test_queue", 1024); - - *q1.num_readers = 0; - - REQUIRE(*q1.num_readers == 0); - REQUIRE(*q2.num_readers == 0); - - msgq_init_subscriber(&q1); - REQUIRE(*q1.num_readers == 1); - REQUIRE(*q2.num_readers == 1); - REQUIRE(q1.reader_id == 0); - - msgq_init_subscriber(&q2); - REQUIRE(*q1.num_readers == 2); - REQUIRE(*q2.num_readers == 2); - REQUIRE(q2.reader_id == 1); -} - - -TEST_CASE("Write 1 msg, read 1 msg", "[integration]"){ - remove("/dev/shm/test_queue"); - const size_t msg_size = 128; - msgq_queue_t writer, reader; - - msgq_new_queue(&writer, "test_queue", 1024); - msgq_new_queue(&reader, "test_queue", 1024); - - msgq_init_publisher(&writer); - msgq_init_subscriber(&reader); - - // Build 128 byte message - msgq_msg_t outgoing_msg; - msgq_msg_init_size(&outgoing_msg, msg_size); - - for (size_t i = 0; i < msg_size; i++){ - outgoing_msg.data[i] = i; - } - - REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); - - msgq_msg_t incoming_msg1; - REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size); - REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0); - - // Verify that there are no more messages - msgq_msg_t incoming_msg2; - REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == 0); - - msgq_msg_close(&outgoing_msg); - msgq_msg_close(&incoming_msg1); - msgq_msg_close(&incoming_msg2); -} - -TEST_CASE("Write 2 msg, read 2 msg - conflate = false", "[integration]"){ - remove("/dev/shm/test_queue"); - const size_t msg_size = 128; - msgq_queue_t writer, reader; - - msgq_new_queue(&writer, "test_queue", 1024); - msgq_new_queue(&reader, "test_queue", 1024); - - msgq_init_publisher(&writer); - msgq_init_subscriber(&reader); - - // Build 128 byte message - msgq_msg_t outgoing_msg; - msgq_msg_init_size(&outgoing_msg, msg_size); - - for (size_t i = 0; i < msg_size; i++){ - outgoing_msg.data[i] = i; - } - - REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); - REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); - - msgq_msg_t incoming_msg1; - REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size); - REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0); - - msgq_msg_t incoming_msg2; - REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == msg_size); - REQUIRE(memcmp(incoming_msg2.data, outgoing_msg.data, msg_size) == 0); - - msgq_msg_close(&outgoing_msg); - msgq_msg_close(&incoming_msg1); - msgq_msg_close(&incoming_msg2); -} - -TEST_CASE("Write 2 msg, read 2 msg - conflate = true", "[integration]"){ - remove("/dev/shm/test_queue"); - const size_t msg_size = 128; - msgq_queue_t writer, reader; - - msgq_new_queue(&writer, "test_queue", 1024); - msgq_new_queue(&reader, "test_queue", 1024); - - msgq_init_publisher(&writer); - msgq_init_subscriber(&reader); - reader.read_conflate = true; - - // Build 128 byte message - msgq_msg_t outgoing_msg; - msgq_msg_init_size(&outgoing_msg, msg_size); - - for (size_t i = 0; i < msg_size; i++){ - outgoing_msg.data[i] = i; - } - - REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); - REQUIRE(msgq_msg_send(&outgoing_msg, &writer) == msg_size); - - msgq_msg_t incoming_msg1; - REQUIRE(msgq_msg_recv(&incoming_msg1, &reader) == msg_size); - REQUIRE(memcmp(incoming_msg1.data, outgoing_msg.data, msg_size) == 0); - - // Verify that there are no more messages - msgq_msg_t incoming_msg2; - REQUIRE(msgq_msg_recv(&incoming_msg2, &reader) == 0); - - msgq_msg_close(&outgoing_msg); - msgq_msg_close(&incoming_msg1); - msgq_msg_close(&incoming_msg2); -} - -TEST_CASE("1 publisher, 1 slow subscriber", "[integration]"){ - remove("/dev/shm/test_queue"); - msgq_queue_t writer, reader; - - msgq_new_queue(&writer, "test_queue", 1024); - msgq_new_queue(&reader, "test_queue", 1024); - - msgq_init_publisher(&writer); - msgq_init_subscriber(&reader); - - int n_received = 0; - int n_skipped = 0; - - for (uint64_t i = 0; i < 1e5; i++) { - msgq_msg_t outgoing_msg; - msgq_msg_init_data(&outgoing_msg, (char*)&i, sizeof(uint64_t)); - msgq_msg_send(&outgoing_msg, &writer); - msgq_msg_close(&outgoing_msg); - - if (i % 10 == 0){ - msgq_msg_t msg1; - msgq_msg_recv(&msg1, &reader); - - if (msg1.size == 0){ - n_skipped++; - } else { - n_received++; - } - msgq_msg_close(&msg1); - } - } - - // TODO: verify these numbers by hand - REQUIRE(n_received == 8572); - REQUIRE(n_skipped == 1428); -} - -TEST_CASE("1 publisher, 2 subscribers", "[integration]"){ - remove("/dev/shm/test_queue"); - msgq_queue_t writer, reader1, reader2; - - msgq_new_queue(&writer, "test_queue", 1024); - msgq_new_queue(&reader1, "test_queue", 1024); - msgq_new_queue(&reader2, "test_queue", 1024); - - msgq_init_publisher(&writer); - msgq_init_subscriber(&reader1); - msgq_init_subscriber(&reader2); - - for (uint64_t i = 0; i < 1024 * 3; i++) { - msgq_msg_t outgoing_msg; - msgq_msg_init_data(&outgoing_msg, (char*)&i, sizeof(uint64_t)); - msgq_msg_send(&outgoing_msg, &writer); - - msgq_msg_t msg1, msg2; - msgq_msg_recv(&msg1, &reader1); - msgq_msg_recv(&msg2, &reader2); - - REQUIRE(msg1.size == sizeof(uint64_t)); - REQUIRE(msg2.size == sizeof(uint64_t)); - REQUIRE(*(uint64_t*)msg1.data == i); - REQUIRE(*(uint64_t*)msg2.data == i); - - msgq_msg_close(&outgoing_msg); - msgq_msg_close(&msg1); - msgq_msg_close(&msg2); - } -} diff --git a/cereal/messaging/test_runner.cc b/cereal/messaging/test_runner.cc deleted file mode 100644 index 62bf7476a18996..00000000000000 --- a/cereal/messaging/test_runner.cc +++ /dev/null @@ -1,2 +0,0 @@ -#define CATCH_CONFIG_MAIN -#include "catch2/catch.hpp" diff --git a/msgq b/msgq index 6d2cc6e22229a9..615aea9b5519d2 160000 --- a/msgq +++ b/msgq @@ -1 +1 @@ -Subproject commit 6d2cc6e22229a9c855d8474e5643b26fbf2b5976 +Subproject commit 615aea9b5519d2a3631fce4753bed29287fc4f9b From 280a284b760c2a0519ffedcdb7228be23c90edf9 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 6 Jun 2024 16:24:09 -0700 Subject: [PATCH 05/12] update TOTAL_SCONS_NODES from https://github.com/commaai/openpilot/commit/fe7d3429abd85c120f0205093149eb9eabc81cb1 --- system/manager/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/manager/build.py b/system/manager/build.py index ac7822d51b3d90..956336a604fdff 100755 --- a/system/manager/build.py +++ b/system/manager/build.py @@ -14,7 +14,7 @@ MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9 CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache") -TOTAL_SCONS_NODES = 2410 +TOTAL_SCONS_NODES = 2820 MAX_BUILD_PROGRESS = 100 def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None: From a7ed190df4665a7f972862c992455bfd7daa6eab Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 6 Jun 2024 16:46:27 -0700 Subject: [PATCH 06/12] controlsd: remove last mention of CAN (#32646) --- selfdrive/controls/controlsd.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index bc79c1fb41808c..4f5bd194056ba7 100755 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -317,19 +317,18 @@ def update_events(self, CS): # generic catch-all. ideally, a more specific event should be added above instead has_disable_events = self.events.contains(ET.NO_ENTRY) and (self.events.contains(ET.SOFT_DISABLE) or self.events.contains(ET.IMMEDIATE_DISABLE)) no_system_errors = (not has_disable_events) or (len(self.events) == num_events) - if (not self.sm.all_checks() or CS.canRcvTimeout) and no_system_errors: + if not self.sm.all_checks() and no_system_errors: if not self.sm.all_alive(): self.events.add(EventName.commIssue) elif not self.sm.all_freq_ok(): self.events.add(EventName.commIssueAvgFreq) - else: # invalid or can_rcv_timeout. + else: self.events.add(EventName.commIssue) logs = { 'invalid': [s for s, valid in self.sm.valid.items() if not valid], 'not_alive': [s for s, alive in self.sm.alive.items() if not alive], 'not_freq_ok': [s for s, freq_ok in self.sm.freq_ok.items() if not freq_ok], - 'can_rcv_timeout': CS.canRcvTimeout, } if logs != self.logged_comm_issue: cloudlog.event("commIssue", error=True, **logs) From ccd26de097dbdbc82af799d22eb2f4b6c92defcc Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 6 Jun 2024 21:12:21 -0700 Subject: [PATCH 07/12] update msgq repo url (#32647) --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index ee65939b6b6eda..4952fcbeae168d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,7 +6,7 @@ url = ../../commaai/opendbc.git [submodule "msgq"] path = msgq - url = ../../commaai/cereal.git + url = ../../commaai/msgq.git [submodule "rednose_repo"] path = rednose_repo url = ../../commaai/rednose.git From e6caaf40fb16e7c27221e1353d0e903d5c805918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Thu, 6 Jun 2024 23:35:13 -0700 Subject: [PATCH 08/12] More cereal cleanups (#32644) * rm generate_javascript.sh * Unused --- cereal/generate_javascript.sh | 26 -------------------------- cereal/messaging/stress.py | 14 -------------- 2 files changed, 40 deletions(-) delete mode 100755 cereal/generate_javascript.sh delete mode 100644 cereal/messaging/stress.py diff --git a/cereal/generate_javascript.sh b/cereal/generate_javascript.sh deleted file mode 100755 index d6525a64d95eed..00000000000000 --- a/cereal/generate_javascript.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -rm -r gen/ts -rm -r gen/js - -mkdir gen/ts -mkdir gen/js - -echo "Installing needed npm modules" -npm i capnpc-ts capnp-ts - -capnpc -o node_modules/.bin/capnpc-ts:gen/ts log.capnp car.capnp -capnpc -o node_modules/.bin/capnpc-ts:gen/ts car.capnp - -cat log.capnp | egrep '\([a-zA-Z]*\.[^\s]+\.[^s]+\)' | sed 's/^.*([a-zA-Z]*\.\([a-zA-Z.]*\)).*/\1/' | while read line -do - TOKEN=`echo $line | sed 's/\./_/g'` - ROOT=`echo $line | sed 's/\..*$//g'` - cat gen/ts/log.capnp.ts | grep '^import.*'${TOKEN} - if [[ "$?" == "1" ]] - then - sed -i 's/^\(import {.*\)'${ROOT}'\(,*\) \(.*\)$/\1'${ROOT}', '${TOKEN}'\2 \3/' ./gen/ts/log.capnp.ts - fi -done - -tsc ./gen/ts/* --lib es2015 --outDir ./gen/js diff --git a/cereal/messaging/stress.py b/cereal/messaging/stress.py deleted file mode 100644 index 1a27e520edd01e..00000000000000 --- a/cereal/messaging/stress.py +++ /dev/null @@ -1,14 +0,0 @@ -from messaging_pyx import Context, SubSocket, PubSocket - -if __name__ == "__main__": - c = Context() - pub_sock = PubSocket() - pub_sock.connect(c, "controlsState") - - for i in range(int(1e10)): - print(i) - sub_sock = SubSocket() - sub_sock.connect(c, "controlsState") - - pub_sock.send(b'a') - print(sub_sock.receive()) From 5c3c06dfdf087c74214a6cfc350cbcc02346b41e Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 7 Jun 2024 15:26:55 -0700 Subject: [PATCH 09/12] blacklist more files from release (#32616) * blacklist more files from release * dbcs * fix * just use the lfs * rm that too * cleanup * cleanup --- SConstruct | 2 +- release/release_files.py | 68 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/SConstruct b/SConstruct index ba49d926aa340e..e827d6641364ed 100644 --- a/SConstruct +++ b/SConstruct @@ -67,7 +67,7 @@ AddOption('--pc-thneed', AddOption('--minimal', action='store_false', dest='extras', - default=os.path.islink(Dir('#rednose/').abspath), # minimal by default on release branch (where rednose is not a link) + default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS) help='the minimum build to run openpilot. no tests, tools, etc.') ## Architecture name breakdown (arch) diff --git a/release/release_files.py b/release/release_files.py index a6fb7d6481aaa6..2d279e97a4a439 100755 --- a/release/release_files.py +++ b/release/release_files.py @@ -10,13 +10,25 @@ # - minimizing release download size # - keeping the diff readable blacklist = [ - "^scripts/", "body/STL/", - "tools/cabana/", + + "panda/drivers/", "panda/examples/", + "panda/tests/safety/", + + "opendbc/.*.dbc$", "opendbc/generator/", + "cereal/.*test.*", + "^common/tests/", + + # particularly large text files + "poetry.lock", + "third_party/catch2", + "selfdrive/car/tests/test_models.*", + "^tools/", + "^scripts/", "^tinygrad_repo/", "matlab.*.md", @@ -56,8 +68,60 @@ "tinygrad_repo/tinygrad/runtime/ops_gpu.py", "tinygrad_repo/tinygrad/shape/*", "tinygrad_repo/tinygrad/.*.py", + + # TODO: do this automatically + "opendbc/comma_body.dbc", + "opendbc/chrysler_ram_hd_generated.dbc", + "opendbc/chrysler_ram_dt_generated.dbc", + "opendbc/chrysler_pacifica_2017_hybrid_generated.dbc", + "opendbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc", + "opendbc/gm_global_a_powertrain_generated.dbc", + "opendbc/gm_global_a_object.dbc", + "opendbc/gm_global_a_chassis.dbc", + "opendbc/FORD_CADS.dbc", + "opendbc/ford_fusion_2018_adas.dbc", + "opendbc/ford_lincoln_base_pt.dbc", + "opendbc/honda_accord_2018_can_generated.dbc", + "opendbc/acura_ilx_2016_can_generated.dbc", + "opendbc/acura_rdx_2018_can_generated.dbc", + "opendbc/acura_rdx_2020_can_generated.dbc", + "opendbc/honda_civic_touring_2016_can_generated.dbc", + "opendbc/honda_civic_hatchback_ex_2017_can_generated.dbc", + "opendbc/honda_crv_touring_2016_can_generated.dbc", + "opendbc/honda_crv_ex_2017_can_generated.dbc", + "opendbc/honda_crv_ex_2017_body_generated.dbc", + "opendbc/honda_crv_executive_2016_can_generated.dbc", + "opendbc/honda_fit_ex_2018_can_generated.dbc", + "opendbc/honda_odyssey_exl_2018_generated.dbc", + "opendbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc", + "opendbc/honda_insight_ex_2019_can_generated.dbc", + "opendbc/acura_ilx_2016_nidec.dbc", + "opendbc/honda_civic_ex_2022_can_generated.dbc", + "opendbc/hyundai_canfd.dbc", + "opendbc/hyundai_kia_generic.dbc", + "opendbc/hyundai_kia_mando_front_radar_generated.dbc", + "opendbc/mazda_2017.dbc", + "opendbc/nissan_x_trail_2017_generated.dbc", + "opendbc/nissan_leaf_2018_generated.dbc", + "opendbc/subaru_global_2017_generated.dbc", + "opendbc/subaru_global_2020_hybrid_generated.dbc", + "opendbc/subaru_outback_2015_generated.dbc", + "opendbc/subaru_outback_2019_generated.dbc", + "opendbc/subaru_forester_2017_generated.dbc", + "opendbc/toyota_tnga_k_pt_generated.dbc", + "opendbc/toyota_new_mc_pt_generated.dbc", + "opendbc/toyota_nodsu_pt_generated.dbc", + "opendbc/toyota_adas.dbc", + "opendbc/toyota_tss2_adas.dbc", + "opendbc/vw_golf_mk4.dbc", + "opendbc/vw_mqb_2010.dbc", + "opendbc/tesla_can.dbc", + "opendbc/tesla_radar_bosch_generated.dbc", + "opendbc/tesla_radar_continental_generated.dbc", + "opendbc/tesla_powertrain.dbc", ] + if __name__ == "__main__": for f in Path(ROOT).rglob("**/*"): if not (f.is_file() or f.is_symlink()): From 2bad3c29d99ed6f037e40489f450d72e1e28dd27 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 7 Jun 2024 15:46:32 -0700 Subject: [PATCH 10/12] build_release: run tests with pytest --- release/build_release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/build_release.sh b/release/build_release.sh index b68ae455c35d87..510df75e514772 100755 --- a/release/build_release.sh +++ b/release/build_release.sh @@ -90,9 +90,9 @@ TEST_FILES="tools/" cd $SOURCE_DIR cp -pR -n --parents $TEST_FILES $BUILD_DIR/ cd $BUILD_DIR -RELEASE=1 selfdrive/test/test_onroad.py +RELEASE=1 pytest -n0 -s selfdrive/test/test_onroad.py #system/manager/test/test_manager.py -selfdrive/car/tests/test_car_interfaces.py +pytest selfdrive/car/tests/test_car_interfaces.py rm -rf $TEST_FILES if [ ! -z "$RELEASE_BRANCH" ]; then From b42b156fcd8d98adee9e6b5384558cfabd4f2165 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 7 Jun 2024 17:03:17 -0700 Subject: [PATCH 11/12] card: remove old canRcvtimeout field (#32650) * card: remove old canRcvtimeout field * deprecate --- cereal/car.capnp | 2 +- selfdrive/car/card.py | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/cereal/car.capnp b/cereal/car.capnp index d7ec1b2484a390..8d76f0ead8de51 100644 --- a/cereal/car.capnp +++ b/cereal/car.capnp @@ -156,7 +156,6 @@ struct CarState { canValid @26 :Bool; # invalid counter/checksums canTimeout @40 :Bool; # CAN bus dropped out canErrorCounter @48 :UInt32; - canRcvTimeout @49 :Bool; # car speed vEgo @1 :Float32; # best estimate of speed @@ -282,6 +281,7 @@ struct CarState { brakeLightsDEPRECATED @19 :Bool; steeringRateLimitedDEPRECATED @29 :Bool; canMonoTimesDEPRECATED @12: List(UInt64); + canRcvTimeoutDEPRECATED @49 :Bool; } # ******* radar state @ 20hz ******* diff --git a/selfdrive/car/card.py b/selfdrive/car/card.py index 330ad40e0d4adf..d9ee020ba493b1 100755 --- a/selfdrive/car/card.py +++ b/selfdrive/car/card.py @@ -29,8 +29,7 @@ def __init__(self, CI=None): self.sm = messaging.SubMaster(['pandaStates', 'carControl', 'onroadEvents']) self.pm = messaging.PubMaster(['sendcan', 'carState', 'carParams', 'carOutput']) - self.can_rcv_timeout_counter = 0 # consecutive timeout count - self.can_rcv_cum_timeout_counter = 0 # cumulative timeout count + self.can_rcv_cum_timeout_counter = 0 self.CC_prev = car.CarControl.new_message() self.CS_prev = car.CarState.new_message() @@ -96,12 +95,7 @@ def state_update(self) -> car.CarState: # Check for CAN timeout if not can_rcv_valid: - self.can_rcv_timeout_counter += 1 self.can_rcv_cum_timeout_counter += 1 - else: - self.can_rcv_timeout_counter = 0 - - self.can_rcv_timeout = self.can_rcv_timeout_counter >= 5 if can_rcv_valid and REPLAY: self.can_log_mono_time = messaging.log_from_bytes(can_strs[0]).logMonoTime @@ -141,7 +135,6 @@ def state_publish(self, CS: car.CarState): cs_send = messaging.new_message('carState') cs_send.valid = CS.canValid cs_send.carState = CS - cs_send.carState.canRcvTimeout = self.can_rcv_timeout cs_send.carState.canErrorCounter = self.can_rcv_cum_timeout_counter cs_send.carState.cumLagMs = -self.rk.remaining * 1000. self.pm.send('carState', cs_send) From 9ce380d3ce515f7065e27df3e7b155164bcdb56a Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 7 Jun 2024 21:12:10 -0700 Subject: [PATCH 12/12] bump opendbc (#32651) * bump * bump2 * bump3 --- opendbc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendbc b/opendbc index f58bc3392cfffd..776bca184bc997 160000 --- a/opendbc +++ b/opendbc @@ -1 +1 @@ -Subproject commit f58bc3392cfffd25b311e093aa72f0ef3a89f6e8 +Subproject commit 776bca184bc997b587afb20df71bc56a4890c4d8