From 4e32d97896bc59995f5f9db0feb46a4a97fb892f Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 7 Mar 2026 10:15:28 -0800 Subject: [PATCH 1/7] add HITL tests for CAN error -> SPI lockup Tests that the panda stays responsive over SPI when CAN error conditions occur (speed mismatch, bus-off, sustained error storms). Validates recovery after errors and checks that interrupt rate faults aren't triggered. Co-Authored-By: Claude Opus 4.6 --- Jenkinsfile | 4 +- tests/hitl/10_can_errors.py | 212 ++++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 tests/hitl/10_can_errors.py diff --git a/Jenkinsfile b/Jenkinsfile index a064bdc17f8..b1516dc2613 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -114,7 +114,7 @@ pipeline { ["build", "scons"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && pytest --durations=0 2*.py [5-9]*.py"], + ["test", "cd tests/hitl && pytest --durations=0 2*.py [5-9]*.py 10*.py"], ]) } } @@ -126,7 +126,7 @@ pipeline { ["build", "scons"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && pytest --durations=0 2*.py [5-9]*.py"], + ["test", "cd tests/hitl && pytest --durations=0 2*.py [5-9]*.py 10*.py"], ]) } } diff --git a/tests/hitl/10_can_errors.py b/tests/hitl/10_can_errors.py new file mode 100644 index 00000000000..d4dd56c2c7c --- /dev/null +++ b/tests/hitl/10_can_errors.py @@ -0,0 +1,212 @@ +import time +import pytest + +from opendbc.car.structs import CarParams +from panda.tests.hitl.helpers import clear_can_buffers + +SPEED_NORMAL = 500 +SPEED_MISMATCH = 250 + + +def _assert_spi_responsive(p, duration=5.0, interval=0.05): + """Poll health over SPI for `duration` seconds, assert we get responses.""" + start = time.monotonic() + count = 0 + max_gap = 0.0 + last_response = start + while time.monotonic() - start < duration: + h = p.health() + assert h['uptime'] > 0 + now = time.monotonic() + gap = now - last_response + if gap > max_gap: + max_gap = gap + last_response = now + count += 1 + time.sleep(interval) + expected = int(duration / interval) + return count, max_gap, expected + + +def _flood_jungle_can(panda_jungle, duration=3.0, buses=(0, 1, 2)): + """Send CAN messages from jungle continuously for `duration` seconds.""" + msg = b"\xaa" * 8 + start = time.monotonic() + sent = 0 + while time.monotonic() - start < duration: + for bus in buses: + panda_jungle.can_send(0x123, msg, bus) + sent += 1 + return sent + + +@pytest.mark.panda_expect_can_error +@pytest.mark.timeout(120) +class TestCanErrorResilience: + """Verify panda stays responsive over SPI during CAN error conditions.""" + + def test_spi_responsive_during_can_errors(self, p, panda_jungle): + """Speed mismatch causes CAN error interrupts; SPI must stay responsive.""" + p.set_safety_mode(CarParams.SafetyModel.allOutput) + + # Set panda to 500kbps, jungle to 250kbps -> every frame is a CAN error + for bus in range(3): + p.set_can_speed_kbps(bus, SPEED_NORMAL) + panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + + # Flood from jungle while polling panda health + msg = b"\xaa" * 8 + start = time.monotonic() + health_count = 0 + max_gap = 0.0 + last_response = start + + while time.monotonic() - start < 8.0: + # send burst from jungle + for bus in range(3): + panda_jungle.can_send(0x123, msg, bus) + + # check panda responsiveness + h = p.health() + assert h['uptime'] > 0 + now = time.monotonic() + gap = now - last_response + if gap > max_gap: + max_gap = gap + last_response = now + health_count += 1 + + print(f"health responses: {health_count}, max gap: {max_gap*1000:.1f}ms") + + # verify errors actually occurred + for bus in range(3): + ch = p.can_health(bus) + print(f" bus {bus}: total_error_cnt={ch['total_error_cnt']}, " + f"irq0_rate={ch['irq0_call_rate']}, " + f"bus_off_cnt={ch['bus_off_cnt']}, " + f"core_resets={ch['can_core_reset_count']}") + assert ch['total_error_cnt'] > 0, f"Bus {bus}: expected CAN errors" + + # SPI should not have had long gaps (>250ms would indicate lockup) + assert max_gap < 0.250, f"SPI gap too large: {max_gap*1000:.1f}ms (lockup?)" + + def test_spi_responsive_during_bus_off(self, p, panda_jungle): + """TX with no ACK -> bus-off -> must not block SPI.""" + p.set_safety_mode(CarParams.SafetyModel.allOutput) + + # Mismatch speeds so jungle can't ACK panda's TX + for bus in range(3): + p.set_can_speed_kbps(bus, SPEED_NORMAL) + panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + + # Send from panda into the void -- TEC rises, eventually bus-off + msg = b"\xbb" * 8 + for _ in range(300): + for bus in range(3): + p.can_send(0x456, msg, bus) + time.sleep(0.001) + + # Panda must still respond + count, max_gap, expected = _assert_spi_responsive(p, duration=5.0) + print(f"health responses: {count}/{expected}, max gap: {max_gap*1000:.1f}ms") + + for bus in range(3): + ch = p.can_health(bus) + print(f" bus {bus}: bus_off_cnt={ch['bus_off_cnt']}, " + f"tec={ch['transmit_error_cnt']}, " + f"core_resets={ch['can_core_reset_count']}") + + assert max_gap < 0.250, f"SPI gap too large: {max_gap*1000:.1f}ms (lockup?)" + + def test_sustained_error_storm(self, p, panda_jungle): + """Sustained CAN errors for 15s must not degrade SPI responsiveness.""" + p.set_safety_mode(CarParams.SafetyModel.allOutput) + + for bus in range(3): + p.set_can_speed_kbps(bus, SPEED_NORMAL) + panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + + msg = b"\xcc" * 8 + start = time.monotonic() + health_count = 0 + gaps = [] + last_response = start + + while time.monotonic() - start < 15.0: + for bus in range(3): + panda_jungle.can_send(0x789, msg, bus) + + h = p.health() + assert h['uptime'] > 0 + now = time.monotonic() + gaps.append(now - last_response) + last_response = now + health_count += 1 + + max_gap = max(gaps) + avg_gap = sum(gaps) / len(gaps) + p95_gap = sorted(gaps)[int(len(gaps) * 0.95)] + print(f"health responses: {health_count}, " + f"avg gap: {avg_gap*1000:.1f}ms, " + f"p95 gap: {p95_gap*1000:.1f}ms, " + f"max gap: {max_gap*1000:.1f}ms") + + assert max_gap < 0.250, f"SPI max gap: {max_gap*1000:.1f}ms (lockup?)" + assert health_count > 100, f"Too few responses: {health_count}" + + def test_can_recovery_after_errors(self, p, panda_jungle): + """After CAN errors, normal communication must resume.""" + p.set_safety_mode(CarParams.SafetyModel.allOutput) + + # Phase 1: induce errors + for bus in range(3): + p.set_can_speed_kbps(bus, SPEED_NORMAL) + panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + + msg = b"\xdd" * 8 + for _ in range(100): + for bus in range(3): + panda_jungle.can_send(0xabc, msg, bus) + time.sleep(1.0) + + # Phase 2: fix speeds, verify normal CAN works + clear_can_buffers(p, speed=SPEED_NORMAL) + clear_can_buffers(panda_jungle, speed=SPEED_NORMAL) + time.sleep(0.5) + + test_msg = b"\xee" * 8 + for bus in range(3): + panda_jungle.can_send(0x100, test_msg, bus) + time.sleep(0.5) + + msgs = p.can_recv() + buses_received = {m[2] for m in msgs if m[0] == 0x100} + print(f"Received on buses: {buses_received}") + assert len(buses_received) == 3, \ + f"CAN didn't recover on all buses, only got: {buses_received}" + + def test_no_faults_during_errors(self, p, panda_jungle): + """CAN errors should not trigger interrupt rate faults.""" + p.set_safety_mode(CarParams.SafetyModel.allOutput) + + for bus in range(3): + p.set_can_speed_kbps(bus, SPEED_NORMAL) + panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + + msg = b"\xff" * 8 + for _ in range(200): + for bus in range(3): + panda_jungle.can_send(0x555, msg, bus) + time.sleep(0.001) + + # wait for rate counters to update (1s timer) + time.sleep(2.0) + + h = p.health() + print(f"faults: 0x{h['faults']:x}, interrupt_load: {h['interrupt_load']}") + for bus in range(3): + ch = p.can_health(bus) + print(f" bus {bus}: irq0_rate={ch['irq0_call_rate']}, " + f"irq1_rate={ch['irq1_call_rate']}") + + assert h['faults'] == 0, f"Faults during CAN errors: 0x{h['faults']:x}" From a38df4fe46d8e87e9119028545fd859de9800211 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 7 Mar 2026 10:15:39 -0800 Subject: [PATCH 2/7] CI: only run CAN error tests for faster iteration Co-Authored-By: Claude Opus 4.6 --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b1516dc2613..ecde4976d5c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -114,7 +114,7 @@ pipeline { ["build", "scons"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && pytest --durations=0 2*.py [5-9]*.py 10*.py"], + ["test", "cd tests/hitl && pytest -v --durations=0 10*.py"], ]) } } @@ -126,7 +126,7 @@ pipeline { ["build", "scons"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && pytest --durations=0 2*.py [5-9]*.py 10*.py"], + ["test", "cd tests/hitl && pytest -v --durations=0 10*.py"], ]) } } From 1d488aa2c3c8b2cd3642fa93e8102c4187847c78 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 7 Mar 2026 10:18:53 -0800 Subject: [PATCH 3/7] fix linter ISC002: avoid implicit string concatenation Co-Authored-By: Claude Opus 4.6 --- tests/hitl/10_can_errors.py | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/tests/hitl/10_can_errors.py b/tests/hitl/10_can_errors.py index d4dd56c2c7c..9b93461703d 100644 --- a/tests/hitl/10_can_errors.py +++ b/tests/hitl/10_can_errors.py @@ -28,18 +28,6 @@ def _assert_spi_responsive(p, duration=5.0, interval=0.05): return count, max_gap, expected -def _flood_jungle_can(panda_jungle, duration=3.0, buses=(0, 1, 2)): - """Send CAN messages from jungle continuously for `duration` seconds.""" - msg = b"\xaa" * 8 - start = time.monotonic() - sent = 0 - while time.monotonic() - start < duration: - for bus in buses: - panda_jungle.can_send(0x123, msg, bus) - sent += 1 - return sent - - @pytest.mark.panda_expect_can_error @pytest.mark.timeout(120) class TestCanErrorResilience: @@ -81,10 +69,7 @@ def test_spi_responsive_during_can_errors(self, p, panda_jungle): # verify errors actually occurred for bus in range(3): ch = p.can_health(bus) - print(f" bus {bus}: total_error_cnt={ch['total_error_cnt']}, " - f"irq0_rate={ch['irq0_call_rate']}, " - f"bus_off_cnt={ch['bus_off_cnt']}, " - f"core_resets={ch['can_core_reset_count']}") + print(f" bus {bus}: total_error_cnt={ch['total_error_cnt']}, irq0_rate={ch['irq0_call_rate']}, bus_off_cnt={ch['bus_off_cnt']}, core_resets={ch['can_core_reset_count']}") assert ch['total_error_cnt'] > 0, f"Bus {bus}: expected CAN errors" # SPI should not have had long gaps (>250ms would indicate lockup) @@ -112,9 +97,7 @@ def test_spi_responsive_during_bus_off(self, p, panda_jungle): for bus in range(3): ch = p.can_health(bus) - print(f" bus {bus}: bus_off_cnt={ch['bus_off_cnt']}, " - f"tec={ch['transmit_error_cnt']}, " - f"core_resets={ch['can_core_reset_count']}") + print(f" bus {bus}: bus_off_cnt={ch['bus_off_cnt']}, tec={ch['transmit_error_cnt']}, core_resets={ch['can_core_reset_count']}") assert max_gap < 0.250, f"SPI gap too large: {max_gap*1000:.1f}ms (lockup?)" @@ -146,10 +129,7 @@ def test_sustained_error_storm(self, p, panda_jungle): max_gap = max(gaps) avg_gap = sum(gaps) / len(gaps) p95_gap = sorted(gaps)[int(len(gaps) * 0.95)] - print(f"health responses: {health_count}, " - f"avg gap: {avg_gap*1000:.1f}ms, " - f"p95 gap: {p95_gap*1000:.1f}ms, " - f"max gap: {max_gap*1000:.1f}ms") + print(f"health responses: {health_count}, avg gap: {avg_gap*1000:.1f}ms, p95 gap: {p95_gap*1000:.1f}ms, max gap: {max_gap*1000:.1f}ms") assert max_gap < 0.250, f"SPI max gap: {max_gap*1000:.1f}ms (lockup?)" assert health_count > 100, f"Too few responses: {health_count}" @@ -182,8 +162,7 @@ def test_can_recovery_after_errors(self, p, panda_jungle): msgs = p.can_recv() buses_received = {m[2] for m in msgs if m[0] == 0x100} print(f"Received on buses: {buses_received}") - assert len(buses_received) == 3, \ - f"CAN didn't recover on all buses, only got: {buses_received}" + assert len(buses_received) == 3, f"CAN didn't recover on all buses, only got: {buses_received}" def test_no_faults_during_errors(self, p, panda_jungle): """CAN errors should not trigger interrupt rate faults.""" @@ -206,7 +185,6 @@ def test_no_faults_during_errors(self, p, panda_jungle): print(f"faults: 0x{h['faults']:x}, interrupt_load: {h['interrupt_load']}") for bus in range(3): ch = p.can_health(bus) - print(f" bus {bus}: irq0_rate={ch['irq0_call_rate']}, " - f"irq1_rate={ch['irq1_call_rate']}") + print(f" bus {bus}: irq0_rate={ch['irq0_call_rate']}, irq1_rate={ch['irq1_call_rate']}") assert h['faults'] == 0, f"Faults during CAN errors: 0x{h['faults']:x}" From bbce182d34b27b38b05f24ed7fb53f299ba0d1fa Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 7 Mar 2026 10:22:38 -0800 Subject: [PATCH 4/7] fix E501 line too long Co-Authored-By: Claude Opus 4.6 --- tests/hitl/10_can_errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/hitl/10_can_errors.py b/tests/hitl/10_can_errors.py index 9b93461703d..348a62e5f7c 100644 --- a/tests/hitl/10_can_errors.py +++ b/tests/hitl/10_can_errors.py @@ -69,7 +69,7 @@ def test_spi_responsive_during_can_errors(self, p, panda_jungle): # verify errors actually occurred for bus in range(3): ch = p.can_health(bus) - print(f" bus {bus}: total_error_cnt={ch['total_error_cnt']}, irq0_rate={ch['irq0_call_rate']}, bus_off_cnt={ch['bus_off_cnt']}, core_resets={ch['can_core_reset_count']}") + print(f" bus {bus}: errs={ch['total_error_cnt']} irq0={ch['irq0_call_rate']} busoff={ch['bus_off_cnt']} resets={ch['can_core_reset_count']}") assert ch['total_error_cnt'] > 0, f"Bus {bus}: expected CAN errors" # SPI should not have had long gaps (>250ms would indicate lockup) From 591a5f2b4d998a4da66357836990cd2574b917cc Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 7 Mar 2026 10:32:43 -0800 Subject: [PATCH 5/7] handle SPI lockup gracefully, disable xdist for HITL - Catch PandaSpiException instead of crashing test runner - Disable pytest-xdist (tests share one panda) - Report lockup as assertion failure with stats Co-Authored-By: Claude Opus 4.6 --- Jenkinsfile | 4 +- tests/hitl/10_can_errors.py | 179 +++++++++++++++++++----------------- 2 files changed, 99 insertions(+), 84 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ecde4976d5c..9386851d047 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -114,7 +114,7 @@ pipeline { ["build", "scons"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && pytest -v --durations=0 10*.py"], + ["test", "cd tests/hitl && pytest -v --durations=0 -p no:xdist 10*.py"], ]) } } @@ -126,7 +126,7 @@ pipeline { ["build", "scons"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && pytest -v --durations=0 10*.py"], + ["test", "cd tests/hitl && pytest -v --durations=0 -p no:xdist 10*.py"], ]) } } diff --git a/tests/hitl/10_can_errors.py b/tests/hitl/10_can_errors.py index 348a62e5f7c..8d40f325d58 100644 --- a/tests/hitl/10_can_errors.py +++ b/tests/hitl/10_can_errors.py @@ -2,34 +2,87 @@ import pytest from opendbc.car.structs import CarParams +from panda import Panda +from panda.python.spi import PandaSpiException from panda.tests.hitl.helpers import clear_can_buffers SPEED_NORMAL = 500 SPEED_MISMATCH = 250 -def _assert_spi_responsive(p, duration=5.0, interval=0.05): - """Poll health over SPI for `duration` seconds, assert we get responses.""" +def _reset_speeds(p, panda_jungle): + """Restore matching CAN speeds and clear buffers.""" + for bus in range(3): + panda_jungle.set_can_speed_kbps(bus, SPEED_NORMAL) + # panda might be locked up, try to reset it + try: + p.reset(reconnect=True) + for bus in range(3): + p.set_can_speed_kbps(bus, SPEED_NORMAL) + clear_can_buffers(p) + except Exception: + # if reset fails, reconnect + p.reconnect() + + +def _health_check(p): + """Get health, return None if SPI fails (lockup detected).""" + try: + return p.health() + except PandaSpiException: + return None + + +def _poll_health_during_errors(p, panda_jungle, duration, send_from_panda=False): + """Send CAN at mismatched speed while polling health. Returns stats.""" + msg = b"\xaa" * 8 start = time.monotonic() - count = 0 - max_gap = 0.0 + health_count = 0 + spi_failures = 0 + gaps = [] last_response = start + while time.monotonic() - start < duration: - h = p.health() - assert h['uptime'] > 0 + # generate CAN errors + for bus in range(3): + if send_from_panda: + try: + p.can_send(0x456, msg, bus) + except PandaSpiException: + spi_failures += 1 + continue + else: + panda_jungle.can_send(0x123, msg, bus) + + # check panda responsiveness + h = _health_check(p) now = time.monotonic() gap = now - last_response - if gap > max_gap: - max_gap = gap + gaps.append(gap) last_response = now - count += 1 - time.sleep(interval) - expected = int(duration / interval) - return count, max_gap, expected + + if h is not None: + health_count += 1 + else: + spi_failures += 1 + + max_gap = max(gaps) if gaps else 0 + avg_gap = (sum(gaps) / len(gaps)) if gaps else 0 + p95_idx = int(len(gaps) * 0.95) if gaps else 0 + p95_gap = sorted(gaps)[p95_idx] if gaps else 0 + + return { + 'health_count': health_count, + 'spi_failures': spi_failures, + 'max_gap': max_gap, + 'avg_gap': avg_gap, + 'p95_gap': p95_gap, + 'total_polls': health_count + spi_failures, + } @pytest.mark.panda_expect_can_error -@pytest.mark.timeout(120) +@pytest.mark.timeout(60) class TestCanErrorResilience: """Verify panda stays responsive over SPI during CAN error conditions.""" @@ -37,69 +90,46 @@ def test_spi_responsive_during_can_errors(self, p, panda_jungle): """Speed mismatch causes CAN error interrupts; SPI must stay responsive.""" p.set_safety_mode(CarParams.SafetyModel.allOutput) - # Set panda to 500kbps, jungle to 250kbps -> every frame is a CAN error for bus in range(3): p.set_can_speed_kbps(bus, SPEED_NORMAL) panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) - # Flood from jungle while polling panda health - msg = b"\xaa" * 8 - start = time.monotonic() - health_count = 0 - max_gap = 0.0 - last_response = start + stats = _poll_health_during_errors(p, panda_jungle, duration=8.0) - while time.monotonic() - start < 8.0: - # send burst from jungle - for bus in range(3): - panda_jungle.can_send(0x123, msg, bus) + print(f"health: {stats['health_count']}, spi_fail: {stats['spi_failures']}, " + f"max_gap: {stats['max_gap']*1000:.1f}ms") - # check panda responsiveness - h = p.health() - assert h['uptime'] > 0 - now = time.monotonic() - gap = now - last_response - if gap > max_gap: - max_gap = gap - last_response = now - health_count += 1 - - print(f"health responses: {health_count}, max gap: {max_gap*1000:.1f}ms") - - # verify errors actually occurred - for bus in range(3): - ch = p.can_health(bus) - print(f" bus {bus}: errs={ch['total_error_cnt']} irq0={ch['irq0_call_rate']} busoff={ch['bus_off_cnt']} resets={ch['can_core_reset_count']}") - assert ch['total_error_cnt'] > 0, f"Bus {bus}: expected CAN errors" + # verify errors actually occurred (if panda is still alive) + if stats['spi_failures'] == 0: + for bus in range(3): + ch = p.can_health(bus) + print(f" bus {bus}: errs={ch['total_error_cnt']} irq0={ch['irq0_call_rate']} " + f"busoff={ch['bus_off_cnt']} resets={ch['can_core_reset_count']}") - # SPI should not have had long gaps (>250ms would indicate lockup) - assert max_gap < 0.250, f"SPI gap too large: {max_gap*1000:.1f}ms (lockup?)" + assert stats['spi_failures'] == 0, f"SPI failed {stats['spi_failures']} times (panda locked up)" + assert stats['max_gap'] < 0.250, f"SPI gap too large: {stats['max_gap']*1000:.1f}ms" def test_spi_responsive_during_bus_off(self, p, panda_jungle): """TX with no ACK -> bus-off -> must not block SPI.""" p.set_safety_mode(CarParams.SafetyModel.allOutput) - # Mismatch speeds so jungle can't ACK panda's TX for bus in range(3): p.set_can_speed_kbps(bus, SPEED_NORMAL) panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) - # Send from panda into the void -- TEC rises, eventually bus-off - msg = b"\xbb" * 8 - for _ in range(300): - for bus in range(3): - p.can_send(0x456, msg, bus) - time.sleep(0.001) + stats = _poll_health_during_errors(p, panda_jungle, duration=5.0, send_from_panda=True) - # Panda must still respond - count, max_gap, expected = _assert_spi_responsive(p, duration=5.0) - print(f"health responses: {count}/{expected}, max gap: {max_gap*1000:.1f}ms") + print(f"health: {stats['health_count']}, spi_fail: {stats['spi_failures']}, " + f"max_gap: {stats['max_gap']*1000:.1f}ms") - for bus in range(3): - ch = p.can_health(bus) - print(f" bus {bus}: bus_off_cnt={ch['bus_off_cnt']}, tec={ch['transmit_error_cnt']}, core_resets={ch['can_core_reset_count']}") + if stats['spi_failures'] == 0: + for bus in range(3): + ch = p.can_health(bus) + print(f" bus {bus}: bus_off_cnt={ch['bus_off_cnt']}, " + f"tec={ch['transmit_error_cnt']}, resets={ch['can_core_reset_count']}") - assert max_gap < 0.250, f"SPI gap too large: {max_gap*1000:.1f}ms (lockup?)" + assert stats['spi_failures'] == 0, f"SPI failed {stats['spi_failures']} times (panda locked up)" + assert stats['max_gap'] < 0.250, f"SPI gap too large: {stats['max_gap']*1000:.1f}ms" def test_sustained_error_storm(self, p, panda_jungle): """Sustained CAN errors for 15s must not degrade SPI responsiveness.""" @@ -109,30 +139,15 @@ def test_sustained_error_storm(self, p, panda_jungle): p.set_can_speed_kbps(bus, SPEED_NORMAL) panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) - msg = b"\xcc" * 8 - start = time.monotonic() - health_count = 0 - gaps = [] - last_response = start - - while time.monotonic() - start < 15.0: - for bus in range(3): - panda_jungle.can_send(0x789, msg, bus) - - h = p.health() - assert h['uptime'] > 0 - now = time.monotonic() - gaps.append(now - last_response) - last_response = now - health_count += 1 + stats = _poll_health_during_errors(p, panda_jungle, duration=15.0) - max_gap = max(gaps) - avg_gap = sum(gaps) / len(gaps) - p95_gap = sorted(gaps)[int(len(gaps) * 0.95)] - print(f"health responses: {health_count}, avg gap: {avg_gap*1000:.1f}ms, p95 gap: {p95_gap*1000:.1f}ms, max gap: {max_gap*1000:.1f}ms") + print(f"health: {stats['health_count']}, spi_fail: {stats['spi_failures']}, " + f"avg_gap: {stats['avg_gap']*1000:.1f}ms, p95_gap: {stats['p95_gap']*1000:.1f}ms, " + f"max_gap: {stats['max_gap']*1000:.1f}ms") - assert max_gap < 0.250, f"SPI max gap: {max_gap*1000:.1f}ms (lockup?)" - assert health_count > 100, f"Too few responses: {health_count}" + assert stats['spi_failures'] == 0, f"SPI failed {stats['spi_failures']} times" + assert stats['max_gap'] < 0.250, f"SPI max gap: {stats['max_gap']*1000:.1f}ms" + assert stats['health_count'] > 100, f"Too few responses: {stats['health_count']}" def test_can_recovery_after_errors(self, p, panda_jungle): """After CAN errors, normal communication must resume.""" @@ -150,8 +165,7 @@ def test_can_recovery_after_errors(self, p, panda_jungle): time.sleep(1.0) # Phase 2: fix speeds, verify normal CAN works - clear_can_buffers(p, speed=SPEED_NORMAL) - clear_can_buffers(panda_jungle, speed=SPEED_NORMAL) + _reset_speeds(p, panda_jungle) time.sleep(0.5) test_msg = b"\xee" * 8 @@ -181,7 +195,8 @@ def test_no_faults_during_errors(self, p, panda_jungle): # wait for rate counters to update (1s timer) time.sleep(2.0) - h = p.health() + h = _health_check(p) + assert h is not None, "Panda unresponsive after CAN errors" print(f"faults: 0x{h['faults']:x}, interrupt_load: {h['interrupt_load']}") for bus in range(3): ch = p.can_health(bus) From 8c8de7d6c0ce516dbd4f6c24068e149047a6cf98 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 7 Mar 2026 10:39:37 -0800 Subject: [PATCH 6/7] fix linter ISC002, use -o addopts= to disable xdist Co-Authored-By: Claude Opus 4.6 --- Jenkinsfile | 4 ++-- tests/hitl/10_can_errors.py | 44 +++++++++++++++---------------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9386851d047..db495967056 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -114,7 +114,7 @@ pipeline { ["build", "scons"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && pytest -v --durations=0 -p no:xdist 10*.py"], + ["test", "cd tests/hitl && pytest -v --durations=0 -o 'addopts=' 10*.py"], ]) } } @@ -126,7 +126,7 @@ pipeline { ["build", "scons"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && pytest -v --durations=0 -p no:xdist 10*.py"], + ["test", "cd tests/hitl && pytest -v --durations=0 -o 'addopts=' 10*.py"], ]) } } diff --git a/tests/hitl/10_can_errors.py b/tests/hitl/10_can_errors.py index 8d40f325d58..624caca9ce3 100644 --- a/tests/hitl/10_can_errors.py +++ b/tests/hitl/10_can_errors.py @@ -14,14 +14,12 @@ def _reset_speeds(p, panda_jungle): """Restore matching CAN speeds and clear buffers.""" for bus in range(3): panda_jungle.set_can_speed_kbps(bus, SPEED_NORMAL) - # panda might be locked up, try to reset it try: p.reset(reconnect=True) for bus in range(3): p.set_can_speed_kbps(bus, SPEED_NORMAL) clear_can_buffers(p) except Exception: - # if reset fails, reconnect p.reconnect() @@ -43,7 +41,6 @@ def _poll_health_during_errors(p, panda_jungle, duration, send_from_panda=False) last_response = start while time.monotonic() - start < duration: - # generate CAN errors for bus in range(3): if send_from_panda: try: @@ -54,7 +51,6 @@ def _poll_health_during_errors(p, panda_jungle, duration, send_from_panda=False) else: panda_jungle.can_send(0x123, msg, bus) - # check panda responsiveness h = _health_check(p) now = time.monotonic() gap = now - last_response @@ -81,6 +77,19 @@ def _poll_health_during_errors(p, panda_jungle, duration, send_from_panda=False) } +def _print_stats(stats): + mg = stats['max_gap'] * 1000 + ag = stats['avg_gap'] * 1000 + p95 = stats['p95_gap'] * 1000 + print(f"health={stats['health_count']} spi_fail={stats['spi_failures']} max_gap={mg:.1f}ms avg={ag:.1f}ms p95={p95:.1f}ms") + + +def _print_can_health(p): + for bus in range(3): + ch = p.can_health(bus) + print(f" bus {bus}: errs={ch['total_error_cnt']} irq0={ch['irq0_call_rate']} busoff={ch['bus_off_cnt']} resets={ch['can_core_reset_count']}") + + @pytest.mark.panda_expect_can_error @pytest.mark.timeout(60) class TestCanErrorResilience: @@ -95,16 +104,10 @@ def test_spi_responsive_during_can_errors(self, p, panda_jungle): panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) stats = _poll_health_during_errors(p, panda_jungle, duration=8.0) + _print_stats(stats) - print(f"health: {stats['health_count']}, spi_fail: {stats['spi_failures']}, " - f"max_gap: {stats['max_gap']*1000:.1f}ms") - - # verify errors actually occurred (if panda is still alive) if stats['spi_failures'] == 0: - for bus in range(3): - ch = p.can_health(bus) - print(f" bus {bus}: errs={ch['total_error_cnt']} irq0={ch['irq0_call_rate']} " - f"busoff={ch['bus_off_cnt']} resets={ch['can_core_reset_count']}") + _print_can_health(p) assert stats['spi_failures'] == 0, f"SPI failed {stats['spi_failures']} times (panda locked up)" assert stats['max_gap'] < 0.250, f"SPI gap too large: {stats['max_gap']*1000:.1f}ms" @@ -118,15 +121,10 @@ def test_spi_responsive_during_bus_off(self, p, panda_jungle): panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) stats = _poll_health_during_errors(p, panda_jungle, duration=5.0, send_from_panda=True) - - print(f"health: {stats['health_count']}, spi_fail: {stats['spi_failures']}, " - f"max_gap: {stats['max_gap']*1000:.1f}ms") + _print_stats(stats) if stats['spi_failures'] == 0: - for bus in range(3): - ch = p.can_health(bus) - print(f" bus {bus}: bus_off_cnt={ch['bus_off_cnt']}, " - f"tec={ch['transmit_error_cnt']}, resets={ch['can_core_reset_count']}") + _print_can_health(p) assert stats['spi_failures'] == 0, f"SPI failed {stats['spi_failures']} times (panda locked up)" assert stats['max_gap'] < 0.250, f"SPI gap too large: {stats['max_gap']*1000:.1f}ms" @@ -140,10 +138,7 @@ def test_sustained_error_storm(self, p, panda_jungle): panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) stats = _poll_health_during_errors(p, panda_jungle, duration=15.0) - - print(f"health: {stats['health_count']}, spi_fail: {stats['spi_failures']}, " - f"avg_gap: {stats['avg_gap']*1000:.1f}ms, p95_gap: {stats['p95_gap']*1000:.1f}ms, " - f"max_gap: {stats['max_gap']*1000:.1f}ms") + _print_stats(stats) assert stats['spi_failures'] == 0, f"SPI failed {stats['spi_failures']} times" assert stats['max_gap'] < 0.250, f"SPI max gap: {stats['max_gap']*1000:.1f}ms" @@ -153,7 +148,6 @@ def test_can_recovery_after_errors(self, p, panda_jungle): """After CAN errors, normal communication must resume.""" p.set_safety_mode(CarParams.SafetyModel.allOutput) - # Phase 1: induce errors for bus in range(3): p.set_can_speed_kbps(bus, SPEED_NORMAL) panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) @@ -164,7 +158,6 @@ def test_can_recovery_after_errors(self, p, panda_jungle): panda_jungle.can_send(0xabc, msg, bus) time.sleep(1.0) - # Phase 2: fix speeds, verify normal CAN works _reset_speeds(p, panda_jungle) time.sleep(0.5) @@ -192,7 +185,6 @@ def test_no_faults_during_errors(self, p, panda_jungle): panda_jungle.can_send(0x555, msg, bus) time.sleep(0.001) - # wait for rate counters to update (1s timer) time.sleep(2.0) h = _health_check(p) From 0d068c6367beeb4653bf3a7af49b0bf34bfe3396 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 7 Mar 2026 11:21:28 -0800 Subject: [PATCH 7/7] cleanup --- tests/hitl/10_can_errors.py | 56 +++++++++++++++---------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/tests/hitl/10_can_errors.py b/tests/hitl/10_can_errors.py index 624caca9ce3..013c24d6221 100644 --- a/tests/hitl/10_can_errors.py +++ b/tests/hitl/10_can_errors.py @@ -2,14 +2,20 @@ import pytest from opendbc.car.structs import CarParams -from panda import Panda from panda.python.spi import PandaSpiException +from panda.tests.hitl.conftest import SPEED_NORMAL from panda.tests.hitl.helpers import clear_can_buffers -SPEED_NORMAL = 500 SPEED_MISMATCH = 250 +def _set_speed_mismatch(p, panda_jungle): + """Set panda and jungle to different CAN speeds to induce errors.""" + for bus in range(3): + p.set_can_speed_kbps(bus, SPEED_NORMAL) + panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + + def _reset_speeds(p, panda_jungle): """Restore matching CAN speeds and clear buffers.""" for bus in range(3): @@ -19,7 +25,7 @@ def _reset_speeds(p, panda_jungle): for bus in range(3): p.set_can_speed_kbps(bus, SPEED_NORMAL) clear_can_buffers(p) - except Exception: + except PandaSpiException: p.reconnect() @@ -37,7 +43,9 @@ def _poll_health_during_errors(p, panda_jungle, duration, send_from_panda=False) start = time.monotonic() health_count = 0 spi_failures = 0 - gaps = [] + max_gap = 0.0 + gap_sum = 0.0 + gap_count = 0 last_response = start while time.monotonic() - start < duration: @@ -54,7 +62,10 @@ def _poll_health_during_errors(p, panda_jungle, duration, send_from_panda=False) h = _health_check(p) now = time.monotonic() gap = now - last_response - gaps.append(gap) + if gap > max_gap: + max_gap = gap + gap_sum += gap + gap_count += 1 last_response = now if h is not None: @@ -62,26 +73,20 @@ def _poll_health_during_errors(p, panda_jungle, duration, send_from_panda=False) else: spi_failures += 1 - max_gap = max(gaps) if gaps else 0 - avg_gap = (sum(gaps) / len(gaps)) if gaps else 0 - p95_idx = int(len(gaps) * 0.95) if gaps else 0 - p95_gap = sorted(gaps)[p95_idx] if gaps else 0 + avg_gap = (gap_sum / gap_count) if gap_count else 0 return { 'health_count': health_count, 'spi_failures': spi_failures, 'max_gap': max_gap, 'avg_gap': avg_gap, - 'p95_gap': p95_gap, - 'total_polls': health_count + spi_failures, } def _print_stats(stats): mg = stats['max_gap'] * 1000 ag = stats['avg_gap'] * 1000 - p95 = stats['p95_gap'] * 1000 - print(f"health={stats['health_count']} spi_fail={stats['spi_failures']} max_gap={mg:.1f}ms avg={ag:.1f}ms p95={p95:.1f}ms") + print(f"health={stats['health_count']} spi_fail={stats['spi_failures']} max_gap={mg:.1f}ms avg={ag:.1f}ms") def _print_can_health(p): @@ -98,10 +103,7 @@ class TestCanErrorResilience: def test_spi_responsive_during_can_errors(self, p, panda_jungle): """Speed mismatch causes CAN error interrupts; SPI must stay responsive.""" p.set_safety_mode(CarParams.SafetyModel.allOutput) - - for bus in range(3): - p.set_can_speed_kbps(bus, SPEED_NORMAL) - panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + _set_speed_mismatch(p, panda_jungle) stats = _poll_health_during_errors(p, panda_jungle, duration=8.0) _print_stats(stats) @@ -115,10 +117,7 @@ def test_spi_responsive_during_can_errors(self, p, panda_jungle): def test_spi_responsive_during_bus_off(self, p, panda_jungle): """TX with no ACK -> bus-off -> must not block SPI.""" p.set_safety_mode(CarParams.SafetyModel.allOutput) - - for bus in range(3): - p.set_can_speed_kbps(bus, SPEED_NORMAL) - panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + _set_speed_mismatch(p, panda_jungle) stats = _poll_health_during_errors(p, panda_jungle, duration=5.0, send_from_panda=True) _print_stats(stats) @@ -132,10 +131,7 @@ def test_spi_responsive_during_bus_off(self, p, panda_jungle): def test_sustained_error_storm(self, p, panda_jungle): """Sustained CAN errors for 15s must not degrade SPI responsiveness.""" p.set_safety_mode(CarParams.SafetyModel.allOutput) - - for bus in range(3): - p.set_can_speed_kbps(bus, SPEED_NORMAL) - panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + _set_speed_mismatch(p, panda_jungle) stats = _poll_health_during_errors(p, panda_jungle, duration=15.0) _print_stats(stats) @@ -147,10 +143,7 @@ def test_sustained_error_storm(self, p, panda_jungle): def test_can_recovery_after_errors(self, p, panda_jungle): """After CAN errors, normal communication must resume.""" p.set_safety_mode(CarParams.SafetyModel.allOutput) - - for bus in range(3): - p.set_can_speed_kbps(bus, SPEED_NORMAL) - panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + _set_speed_mismatch(p, panda_jungle) msg = b"\xdd" * 8 for _ in range(100): @@ -174,10 +167,7 @@ def test_can_recovery_after_errors(self, p, panda_jungle): def test_no_faults_during_errors(self, p, panda_jungle): """CAN errors should not trigger interrupt rate faults.""" p.set_safety_mode(CarParams.SafetyModel.allOutput) - - for bus in range(3): - p.set_can_speed_kbps(bus, SPEED_NORMAL) - panda_jungle.set_can_speed_kbps(bus, SPEED_MISMATCH) + _set_speed_mismatch(p, panda_jungle) msg = b"\xff" * 8 for _ in range(200):