nros_node/executor/monitor.rs
1//! RFC-0052 / phase-296 W3b.4/.5 — on-target contract monitors.
2//!
3//! The baked shape mirrors Phase 211.H's `qos_overrides`: codegen emits a
4//! `&'static [MonitorSpec]` table (plus one `static PubMonitorCell` per
5//! contracted publisher) from the SystemModel's contract layer; the entry
6//! installs it on the executor before entity creation. An uncontracted
7//! image bakes an empty table — every path below dead-code-eliminates.
8//!
9//! Publish counting is an atomic bump on the publisher handle (no clock,
10//! no lock on the hot path); the rate check runs on spin ticks over a
11//! ~[`RATE_CHECK_INTERVAL_US`] window and pushes violations into a small
12//! ring the entry glue drains into the `nros-diagnostics` reporter.
13//!
14//! W3b.5 adds three more rules on the same drain:
15//! - `max-age-runtime` — subscriber take-age (`epoch_now - header.stamp`
16//! peeked from the raw CDR buffer at `RosMessage::STAMP_OFFSET`,
17//! recorded into a [`SubMonitorCell`] on the take path).
18//! - `max-latency-runtime` — node-path (take → publish) latency: the
19//! dispatch elapsed time is attributed to every monitored publisher
20//! whose counter advanced during that dispatch (an upper bound on
21//! take → publish, measured on the executor's monotonic clock). The span
22//! is callback entry to exit, for every dispatch kind: a timer callback
23//! is timed whether it fires in the drains or in the trigger-miss sweep
24//! (phase-474 I6), and a service call it makes is inside the span. The
25//! timer's release jitter (a late start) is not part of it; that is
26//! `release-jitter-runtime`.
27//! - `deadline-miss-runtime` — a dispatched callback ran past its bound
28//! SchedContext's `deadline_us`; what ELSE happens is the tier's
29//! [`DeadlineAction`](super::sched_context::DeadlineAction).
30//!
31//! phase10-B3 adds the route rule beside it:
32//! - `route-latency-runtime` -- a node path whose trigger is an INPUT is
33//! budgeted from the TAKE of that input to its output, and on a node that
34//! reads its input on a timer tick that span includes the wait for the next
35//! tick. `max-latency-runtime` times one dispatch and cannot see the wait,
36//! so a route made of a long tick wait and a slow dispatch could exceed its
37//! budget while every callback passed. The executor stamps the dispatch that
38//! takes the trigger input (the earliest take not yet answered) and, at the
39//! end of a dispatch that published one of the path's outputs, judges
40//! take-to-dispatch-end against the path's `max_latency_ms`
41//! (`route_take`, `route_judge`). Record only: the rule has no action.
42//!
43//! phase-462 W2 adds one more on the same drain, and it is the other half of
44//! what the contract's `on_violation` lowers to:
45//! - `silence-runtime` -- a contracted subscription that took NOTHING for a
46//! whole `max_age_ms` window (`check_silence`). A violation is either "the
47//! callback ran too long", which is the deadline action, or "the input
48//! stopped coming", which is this; the four rules above can only see the
49//! first kind.
50
51use core::sync::atomic::Ordering;
52// portable-atomic: RMW ops (fetch_add/fetch_max/swap) exist even on
53// riscv32imc / Cortex-M0+ that lack native CAS (same choice as
54// `SporadicState` / `AtomicSporadicState` in sched_context.rs).
55use portable_atomic::AtomicU32;
56
57/// One contracted publisher's counters. Baked as a `static` by codegen
58/// (or declared by the fixture); the publisher handle bumps `count` on
59/// every publish, the executor reads deltas on spin ticks.
60#[derive(Debug, Default)]
61pub struct PubMonitorCell {
62 pub count: AtomicU32,
63 /// W3b.5 — max observed take→publish latency (µs) in the current
64 /// check window. Written by the dispatch loop (fetch_max), drained
65 /// (swap 0) by the latency check.
66 pub max_latency_us: AtomicU32,
67 /// Age of the stamp this publisher last put ON THE WIRE, in
68 /// microseconds: `epoch_now - outgoing header.stamp`.
69 ///
70 /// Distinct from `max_latency_us`, which times this node's own
71 /// take→publish work. This says how old the DATA is that the node just
72 /// published, which is the quantity a chain is made of.
73 ///
74 /// It exists to answer a question `max-age-runtime` cannot. That rule
75 /// measures `epoch_now - stamp` on the TAKE path, so if every node in a
76 /// chain propagates the original stamp -- the usual ROS convention, each
77 /// node copying its input's stamp to its output -- the age at the final
78 /// consumer already IS the end-to-end latency. If any node re-stamps
79 /// with `now`, the clock silently resets and the same number becomes
80 /// single-hop age instead. Same units, same magnitude, no warning.
81 ///
82 /// A publish age near zero on a node that consumes input is the
83 /// signature of re-stamping. Recording it here is what lets a chain's
84 /// provenance be checked at all, rather than assumed.
85 ///
86 /// `0` = never observed, matching the other cells: the type has no
87 /// `STAMP_OFFSET`, or no epoch source is installed.
88 pub last_publish_stamp_age_us: AtomicU32,
89 /// phase10-B3 -- the route rule's state for this output
90 /// ([`MonitorSpec::route_trigger`]): the trigger subscription's executor
91 /// slot + 1 in the low 16 bits (0 = no subscription bound, set when the
92 /// subscription registers), [`ROUTE_VIOLATED`] and [`ROUTE_PENDING`] above.
93 pub route: AtomicU32,
94 /// phase10-B3 -- when the earliest take not yet answered by a publish of
95 /// this output happened, executor clock in microseconds (wrapping). Valid
96 /// while [`ROUTE_PENDING`] is set in `route`.
97 pub route_take_us: AtomicU32,
98}
99
100/// phase10-B3 -- `PubMonitorCell::route`: a take of the trigger input is
101/// waiting for this output.
102pub const ROUTE_PENDING: u32 = 1 << 31;
103/// phase10-B3 -- `PubMonitorCell::route`: the last judged route was over its
104/// budget (report once until one is within it, as the latency rule does).
105pub const ROUTE_VIOLATED: u32 = 1 << 30;
106/// phase10-B3 -- `PubMonitorCell::route`: the bound slot + 1.
107pub const ROUTE_SLOT_MASK: u32 = 0xffff;
108
109impl PubMonitorCell {
110 pub const fn new() -> Self {
111 Self {
112 count: AtomicU32::new(0),
113 max_latency_us: AtomicU32::new(0),
114 last_publish_stamp_age_us: AtomicU32::new(0),
115 route: AtomicU32::new(0),
116 route_take_us: AtomicU32::new(0),
117 }
118 }
119}
120
121/// One contracted subscriber's take-age accumulator (W3b.5). The take
122/// path records `epoch_now - header.stamp` per message (fetch_max); the
123/// age check drains it (swap 0) per window.
124#[derive(Debug, Default)]
125pub struct SubMonitorCell {
126 /// Max observed take-age (ms) in the current check window.
127 pub max_age_ms: AtomicU32,
128 /// phase-462 W2 -- observations recorded since the last check, drained
129 /// (swap 0) by `check_age`. The age itself cannot answer "did anything
130 /// arrive": a window with no take and a window whose only take was
131 /// perfectly fresh both leave `max_age_ms` at 0.
132 pub takes: AtomicU32,
133}
134
135impl SubMonitorCell {
136 pub const fn new() -> Self {
137 Self {
138 max_age_ms: AtomicU32::new(0),
139 takes: AtomicU32::new(0),
140 }
141 }
142
143 /// Take-path hook: record one message's age. `stamp_us` is the
144 /// peeked `header.stamp` as µs since the UNIX epoch, `epoch_now_us`
145 /// the receive-side wall clock. A stamp from the future clamps to 0.
146 ///
147 /// phase-462 W2 -- also the silence rule's only evidence that this
148 /// endpoint is being fed. It counts what the AGE rule can see, which is
149 /// stamped takes on a type that carries a stamp: a contracted endpoint
150 /// whose messages carry no readable stamp records nothing here and is
151 /// reported silent. That is the intended verdict rather than a gap --
152 /// its `max_age_ms` cannot be judged either, and an unjudgeable promise
153 /// reported as met is the silent class this phase exists to remove.
154 pub fn observe(&self, stamp_us: u64, epoch_now_us: u64) {
155 let age_ms = (epoch_now_us.saturating_sub(stamp_us) / 1_000).min(u32::MAX as u64) as u32;
156 self.max_age_ms.fetch_max(age_ms, Ordering::Relaxed);
157 self.takes.fetch_add(1, Ordering::Relaxed);
158 }
159}
160
161/// Peek `Time { i32 sec; u32 nanosec }` little-endian at `offset` in a
162/// raw CDR receive buffer (encapsulation header included) and return µs
163/// since the UNIX epoch. `None` when the buffer is too short or the
164/// stamp is pre-epoch/zero (unstamped messages never fire age monitors).
165/// Record the age of the stamp a publisher just put on the wire.
166///
167/// Called from the publish path with the encoded CDR still in hand, using the
168/// same `STAMP_OFFSET` peek the take path uses. A no-op when the type carries
169/// no stamp, when no epoch source is installed, or when the publisher is
170/// uncontracted -- the same three ways `observe_age` folds away.
171///
172/// Stores rather than accumulates: this is "how old was the last thing
173/// published", a state, not a window maximum. A chain check wants the current
174/// value, and a max would be pinned forever by one stale message at startup.
175#[inline]
176pub fn observe_publish_stamp(cell: &PubMonitorCell, raw: &[u8], offset: usize, now_us: u64) {
177 if let Some(stamp_us) = peek_stamp_us(raw, offset) {
178 let age = now_us.saturating_sub(stamp_us).min(u32::MAX as u64) as u32;
179 cell.last_publish_stamp_age_us.store(age, Ordering::Relaxed);
180 }
181}
182
183pub fn peek_stamp_us(raw: &[u8], offset: usize) -> Option<u64> {
184 let sec_b = raw.get(offset..offset + 4)?;
185 let nsec_b = raw.get(offset + 4..offset + 8)?;
186 let sec = i32::from_le_bytes([sec_b[0], sec_b[1], sec_b[2], sec_b[3]]);
187 let nsec = u32::from_le_bytes([nsec_b[0], nsec_b[1], nsec_b[2], nsec_b[3]]);
188 if sec <= 0 {
189 return None;
190 }
191 Some(sec as u64 * 1_000_000 + nsec as u64 / 1_000)
192}
193
194/// One monitored publisher endpoint.
195#[derive(Debug, Clone, Copy)]
196pub struct MonitorSpec {
197 /// Topic name EXACTLY as the node passes it to `create_publisher`
198 /// (the SystemModel's wiring carries the same resolved name).
199 pub topic: &'static str,
200 /// Endpoint ref for violation reports (`<node FQN>/<endpoint>` — the
201 /// SystemModel contract key).
202 pub fqn: &'static str,
203 /// Declared publisher guarantee, in milli-Hz (fixed point: Hz × 1000).
204 /// 0 = no rate contract on this endpoint.
205 pub min_rate_hz_milli: u32,
206 /// W3b.5 — node-path budget (ms) for paths whose OUTPUT is this
207 /// endpoint (`contracts.node_paths[..].max_latency_ms`). 0 = no
208 /// latency contract.
209 pub max_latency_ms: u32,
210 /// phase10-B3 -- the topic of the INPUT that triggers a node path whose
211 /// output is this endpoint (`contracts.node_paths[..].input`, resolved to
212 /// its wiring topic EXACTLY as the node passes it to
213 /// `create_subscription`), or `""` for none. A subscription on this topic
214 /// registered on the same executor is the route's trigger.
215 pub route_trigger: &'static str,
216 /// phase10-B3 -- that path's budget (ms), from the take of the trigger to
217 /// this output (`route-latency-runtime`). 0 = no route contract.
218 pub route_max_latency_ms: u32,
219 /// phase-503 -- the route's SUPERVISED deadline (ms) from the take of the
220 /// trigger: a take still unanswered at `take + route_deadline_ms` is
221 /// reported (`route-deadline-runtime`) and the executor's reaction hook is
222 /// called, without waiting for an output that may never come. 0 = the
223 /// route is not supervised (only the completion verdict above judges it).
224 /// Needs `route_trigger`; ignored without it.
225 pub route_deadline_ms: u32,
226 /// phase-503 -- the reaction the hook is asked to start when the deadline
227 /// passes: an id codegen assigns to the contract's `reaction` path
228 /// (1-based; [`REACTION_NONE`] = none named). Opaque to the executor.
229 pub reaction_id: u16,
230 /// phase-503 -- which mechanism the contract chose to supervise this
231 /// route: [`MECHANISM_EXECUTOR`] (this check, in the spin loop) or
232 /// [`MECHANISM_TIMER`] (an independent timer the port arms from
233 /// `Executor::route_next_due_us`). The executor checks the deadline either
234 /// way and passes the mechanism to the hook.
235 pub mechanism: u8,
236 /// The endpoint's counter cell.
237 pub cell: &'static PubMonitorCell,
238}
239
240/// phase-503 -- [`MonitorSpec::reaction_id`]: no reaction named.
241pub const REACTION_NONE: u16 = 0;
242/// phase-503 -- [`MonitorSpec::mechanism`]: checked in the executor's spin
243/// loop. Catches a slow route, not a callback that never returns.
244pub const MECHANISM_EXECUTOR: u8 = 0;
245/// phase-503 -- [`MonitorSpec::mechanism`]: an independent timer outside the
246/// executor (the port's), which also catches a stuck callback.
247pub const MECHANISM_TIMER: u8 = 1;
248
249/// phase-503 -- what the reaction hook receives when a supervised route
250/// passes its deadline. `repr(C)`: the C ABI hands the same struct through.
251#[repr(C)]
252#[derive(Debug, Clone, Copy, PartialEq, Eq)]
253pub struct ReactionEvent {
254 /// [`MonitorSpec::reaction_id`] of the row.
255 pub reaction_id: u32,
256 /// Index of the row in the installed monitor table.
257 pub row: u32,
258 /// [`rule_code`] of the verdict (`route-deadline-runtime`).
259 pub rule_code: u32,
260 /// Time from the take to the check, ms.
261 pub measured_ms: u32,
262 /// The deadline, ms.
263 pub declared_ms: u32,
264 /// [`MonitorSpec::mechanism`] of the row.
265 pub mechanism: u8,
266}
267
268/// phase-503 -- the reaction hook: called on the spin thread, inside
269/// `spin_once`, once per missed route deadline. It must not block; it starts
270/// the fallback (or hands it to another thread) and returns. `ctx` is what
271/// was registered with the hook.
272pub type ReactionHookFn =
273 unsafe extern "C" fn(event: *const ReactionEvent, ctx: *mut core::ffi::c_void);
274
275/// One monitored subscriber endpoint (W3b.5 age contracts). Separate
276/// table from [`MonitorSpec`] — sub contracts key different endpoints
277/// and need no publish counter.
278#[derive(Debug, Clone, Copy)]
279pub struct AgeMonitorSpec {
280 /// Topic name EXACTLY as the node passes it to `create_subscription`.
281 pub topic: &'static str,
282 /// Endpoint ref for violation reports (the SystemModel contract key).
283 pub fqn: &'static str,
284 /// Declared max take-age (ms). 0 = no age contract.
285 pub max_age_ms: u32,
286 /// The endpoint's age accumulator.
287 pub cell: &'static SubMonitorCell,
288}
289
290/// phase-436 E4 — how often the stack-headroom rule may query the port, in µs.
291///
292/// On a painted Zephyr stack the query walks every UNUSED byte from the stack
293/// base (`z_stack_space_get`), so querying on every spin put that scan inside
294/// the control loop. The high-water mark only grows, so a throttled check sees
295/// the same low mark and only sees it later. Since E5 the query is also capped
296/// at the declared minimum (`stack_unused_bytes_upto`), so it reads at most
297/// that many bytes.
298pub(crate) const STACK_HEADROOM_CHECK_INTERVAL_US: u64 = 1_000_000;
299
300/// The same limit for a build with no clock: once per this many spins.
301pub(crate) const STACK_HEADROOM_CHECK_SPIN_STRIDE: u32 = 1_000;
302
303/// Whether the stack-headroom query is due.
304///
305/// `last_check_us` is `None` until the first check, which is immediate, so a
306/// bound set at boot is checked on the first spin. After that a clocked build
307/// waits one full interval and a clockless build one full stride of spins.
308pub(crate) fn stack_headroom_check_due(
309 now_us: Option<u64>,
310 last_check_us: Option<u64>,
311 spins_since_check: u32,
312) -> bool {
313 match (last_check_us, now_us) {
314 (None, _) => true,
315 (Some(last), Some(now)) => now.saturating_sub(last) >= STACK_HEADROOM_CHECK_INTERVAL_US,
316 (Some(_), None) => spins_since_check >= STACK_HEADROOM_CHECK_SPIN_STRIDE,
317 }
318}
319
320/// Rate-check window (µs). Matches play_launch's ~5 s time-based trigger
321/// so both runtimes converge on comparable cadence.
322pub const RATE_CHECK_INTERVAL_US: u64 = 5_000_000;
323
324/// Max RATE/LATENCY-monitored endpoints per executor (inline table, no_std).
325///
326/// phase-496 W1 (issue 1471) -- generated, not a literal: the
327/// `NROS_EXECUTOR_MAX_MONITORS` knob, whose derived rung is the image's own
328/// `monitor_rows` count (the sizing descriptor's `[image] monitor_rows`), 8 when nothing
329/// states or derives it. A table longer than this is REFUSED at install
330/// ([`check_table_capacity`]), never truncated: the spin loop inspects only the
331/// first `MAX_MONITORS` specs, so a truncating install would boot with the rest
332/// of the contract silently unwatched.
333pub const MAX_MONITORS: usize = crate::config::MAX_MONITORS;
334/// Max AGE-monitored subscriptions per executor -- the second table, on its
335/// own knob (`NROS_EXECUTOR_MAX_AGE_MONITORS`, derived from `age_rows`) so an
336/// image with rate contracts and no age contracts pays for one table only.
337pub const MAX_AGE_MONITORS: usize = crate::config::MAX_AGE_MONITORS;
338
339/// A monitor table longer than the executor can watch -- phase-496 W1.
340///
341/// Its `Display` NAMES THE KNOB to raise (RFC-0065 D2, refuse and name the
342/// remedy), so every installer that refuses reports the same words.
343#[derive(Debug, Clone, Copy, PartialEq, Eq)]
344pub struct MonitorTableFull {
345 /// `"NROS_EXECUTOR_MAX_MONITORS"` or `"NROS_EXECUTOR_MAX_AGE_MONITORS"`.
346 pub knob: &'static str,
347 /// Rows the table carries.
348 pub rows: usize,
349 /// Rows this build can watch (the knob's resolved value).
350 pub capacity: usize,
351}
352
353impl core::fmt::Display for MonitorTableFull {
354 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
355 write!(
356 f,
357 "monitor table has {} rows but this executor watches {}; raise {} \
358 (it derives from the contract's row count when nothing states it)",
359 self.rows, self.capacity, self.knob
360 )
361 }
362}
363
364/// Whether a table of `n_rows` rate/latency rows and `n_ages` age rows fits
365/// this build's executor. The rate table is judged first.
366pub fn check_table_capacity(n_rows: usize, n_ages: usize) -> Result<(), MonitorTableFull> {
367 check_capacity_against(n_rows, n_ages, MAX_MONITORS, MAX_AGE_MONITORS)
368}
369
370/// [`check_table_capacity`] against explicit capacities, so the refusal is
371/// testable at any knob value without rebuilding the crate.
372pub fn check_capacity_against(
373 n_rows: usize,
374 n_ages: usize,
375 max_rows: usize,
376 max_ages: usize,
377) -> Result<(), MonitorTableFull> {
378 if n_rows > max_rows {
379 return Err(MonitorTableFull {
380 knob: "NROS_EXECUTOR_MAX_MONITORS",
381 rows: n_rows,
382 capacity: max_rows,
383 });
384 }
385 if n_ages > max_ages {
386 return Err(MonitorTableFull {
387 knob: "NROS_EXECUTOR_MAX_AGE_MONITORS",
388 rows: n_ages,
389 capacity: max_ages,
390 });
391 }
392 Ok(())
393}
394/// Violation ring depth, per executor, and the slot count of the image-wide
395/// SWD record ([`ViolationRecord`]).
396///
397/// phase-474 I1 -- `NROS_EXECUTOR_MAX_VIOLATIONS` (Zephyr:
398/// `CONFIG_NROS_EXECUTOR_MAX_VIOLATIONS`), default 8. The ring keeps the
399/// LATEST this-many verdicts, so the depth bounds how much recent history a
400/// reader sees and never whether a late violation is kept at all.
401pub const MAX_VIOLATIONS: usize = crate::config::MAX_VIOLATIONS;
402
403/// A detected contract violation, in the play_launch rule-id vocabulary.
404#[derive(Debug, Clone)]
405pub struct Violation {
406 /// `"rate-hierarchy-runtime"` | `"max-age-runtime"` |
407 /// `"max-latency-runtime"` | `"deadline-miss-runtime"` |
408 /// `"stack-headroom-runtime"` | `"alive-supervision-runtime"` |
409 /// `"silence-runtime"` (phase-462 W2) | `"route-latency-runtime"`
410 /// (phase10-B3) | `"route-deadline-runtime"` (phase-503).
411 pub rule: &'static str,
412 /// Violating endpoint ref (from the spec's `fqn`; the SC name for
413 /// deadline misses).
414 pub fqn: &'static str,
415 /// Measured value. Unit is per-rule: milli-Hz for the rate rule, ms
416 /// for age/latency, µs for deadline misses, dropped activations for
417 /// the timer-overrun rule.
418 pub measured: u32,
419 /// Declared bound, same unit as `measured`.
420 pub declared: u32,
421}
422
423/// phase-474 I1 -- every rule id the executor reports, in WIRE ORDER: a rule's
424/// numeric code is its index here plus one, and 0 means "not a rule this table
425/// knows".
426///
427/// The code is what a trace marker and the SWD record carry, because neither
428/// can carry a string. Append only: a capture or a RAM dump taken before a
429/// reorder would silently decode as the wrong rule, which is the class the
430/// callback-trace marker ids are frozen against (`callback_trace.rs`).
431pub const RULE_IDS: [&str; 14] = [
432 "rate-hierarchy-runtime",
433 "max-age-runtime",
434 "max-latency-runtime",
435 "deadline-miss-runtime",
436 "stack-headroom-runtime",
437 "alive-supervision-runtime",
438 "silence-runtime",
439 "timer-overrun-runtime",
440 "release-jitter-runtime",
441 // phase10-B3.
442 "route-latency-runtime",
443 // phase-503.
444 "route-deadline-runtime",
445 // phase-462 W3b -- the hazard observer, in play_launch's spelling (no
446 // `-runtime` suffix) so one log reader serves both sides of a bridge.
447 "hazard-detected",
448 "hazard-reaction",
449 "hazard-recovered",
450];
451
452/// The wire code of `rule` ([`RULE_IDS`] index + 1), or 0 for a rule the
453/// table does not know.
454pub fn rule_code(rule: &str) -> u32 {
455 RULE_IDS
456 .iter()
457 .position(|r| *r == rule)
458 .map(|i| i as u32 + 1)
459 .unwrap_or(0)
460}
461
462/// The rule a wire code names, or `None` for 0 / an unknown code.
463pub fn rule_name(code: u32) -> Option<&'static str> {
464 RULE_IDS.get((code as usize).checked_sub(1)?).copied()
465}
466
467/// 32-bit FNV-1a of an endpoint ref -- the identity a trace marker and the SWD
468/// record carry for `Violation::fqn` (a string neither can hold).
469///
470/// FNV-1a because a decoder can recompute it in one line from the model's
471/// endpoint list (`scripts/read-violation-record.py` does), with no table
472/// baked into the image.
473pub const fn fqn_hash(fqn: &str) -> u32 {
474 let b = fqn.as_bytes();
475 let mut h: u32 = 0x811c_9dc5;
476 let mut i = 0;
477 while i < b.len() {
478 h ^= b[i] as u32;
479 h = h.wrapping_mul(0x0100_0193);
480 i += 1;
481 }
482 h
483}
484
485/// phase-474 I1 -- trace marker: a stored violation, `seq << 8 | rule code`.
486/// The id block continues `callback_trace`'s 16-20; ids 1-7 are the
487/// application's and are never used here.
488pub const MARKER_VIOLATION: u32 = 21;
489/// phase-474 I1 -- trace marker: the endpoint ref's [`fqn_hash`].
490pub const MARKER_VIOLATION_FQN: u32 = 22;
491/// phase-474 I1 -- trace marker: `measured`.
492pub const MARKER_VIOLATION_MEASURED: u32 = 23;
493/// phase-474 I1 -- trace marker: `declared`.
494pub const MARKER_VIOLATION_DECLARED: u32 = 24;
495
496/// phase-474 I1 -- the four `(marker_id, arg)` events one stored violation
497/// becomes in a trace (`callback_trace` emits them, in this order, when the
498/// `trace-callbacks` feature is on and a sink is installed). `seq` is the
499/// executor's sequence number; its low 24 bits ride in the first event.
500pub fn violation_marker_words(seq: u32, v: &Violation) -> [(u32, u32); 4] {
501 [
502 (
503 MARKER_VIOLATION,
504 ((seq & 0x00ff_ffff) << 8) | (rule_code(v.rule) & 0xff),
505 ),
506 (MARKER_VIOLATION_FQN, fqn_hash(v.fqn)),
507 (MARKER_VIOLATION_MEASURED, v.measured),
508 (MARKER_VIOLATION_DECLARED, v.declared),
509 ]
510}
511
512/// `"NRVR"` -- nano-ros violation record.
513pub const RECORD_MAGIC: u32 = 0x4e52_5652;
514/// [`ViolationRecord`] layout version. Bump on any field change; a reader
515/// refuses a version it does not know (`scripts/read-violation-record.py`).
516pub const RECORD_VERSION: u32 = 1;
517/// `u32` words in one [`ViolationSlot`].
518pub const RECORD_SLOT_WORDS: u32 = 7;
519/// `u32` words in the [`ViolationRecord`] header, before the first slot.
520pub const RECORD_HEADER_WORDS: u32 = 10;
521
522/// One slot of the SWD record. Every field is a `u32` on every target, so the
523/// layout is the same on a 32-bit board and a 64-bit host.
524#[repr(C)]
525#[derive(Debug)]
526pub struct ViolationSlot {
527 /// 1-based sequence number of the violation in this slot, 0 = empty.
528 /// Written LAST (zeroed first), so a reader that sees a non-zero `seq`
529 /// twice around a read of the other words has a whole entry.
530 pub seq: AtomicU32,
531 /// [`rule_code`].
532 pub rule: AtomicU32,
533 /// [`fqn_hash`] of the endpoint ref.
534 pub fqn_hash: AtomicU32,
535 pub measured: AtomicU32,
536 pub declared: AtomicU32,
537 /// Address of the endpoint ref's bytes (exact on a 32-bit target, the low
538 /// half of it on a 64-bit host) and its length: the text is in the
539 /// image's rodata, so a reader with the ELF can print the name.
540 pub fqn_addr: AtomicU32,
541 pub fqn_len: AtomicU32,
542}
543
544impl ViolationSlot {
545 pub const fn new() -> Self {
546 Self {
547 seq: AtomicU32::new(0),
548 rule: AtomicU32::new(0),
549 fqn_hash: AtomicU32::new(0),
550 measured: AtomicU32::new(0),
551 declared: AtomicU32::new(0),
552 fqn_addr: AtomicU32::new(0),
553 fqn_len: AtomicU32::new(0),
554 }
555 }
556}
557
558impl Default for ViolationSlot {
559 fn default() -> Self {
560 Self::new()
561 }
562}
563
564/// phase-474 I1 -- the image-wide violation record a debugger reads by NAME,
565/// for a board whose console reaches nobody.
566///
567/// The executor's ring is carved out of executor storage (no symbol, and a
568/// drain empties it), so on the Autoware Safety Island it could only be found
569/// by scanning RAM for rule-id string pointers, and it held the FIRST eight
570/// verdicts since boot. This record is the black box beside it: a `#[repr(C)]`
571/// static, every word a `u32`, never drained, keeping the LATEST
572/// [`MAX_VIOLATIONS`] stored violations of every executor in the image, with
573/// the running total.
574///
575/// Layout (version [`RECORD_VERSION`], little-endian `u32` words):
576///
577/// | word | field | meaning |
578/// | --- | --- | --- |
579/// | 0 | `magic` | [`RECORD_MAGIC`] (`"NRVR"`) |
580/// | 1 | `version` | [`RECORD_VERSION`] |
581/// | 2 | `capacity` | slots (`NROS_EXECUTOR_MAX_VIOLATIONS`) |
582/// | 3 | `slot_words` | [`RECORD_SLOT_WORDS`] |
583/// | 4 | `total` | violations stored since boot (the last `seq`) |
584/// | 5 | `head` | slot the NEXT violation takes (`total % capacity`) |
585/// | 6 | `dropped` | entries overwritten (`total - capacity`, saturating) |
586/// | 7 | `suppressed_before_arm` | verdicts before the monitors armed, all executors (phase-474 I2) |
587/// | 8 | `armed` | executors whose monitors armed (counted as each arms; at its first spin by default) (phase-474 I2) |
588/// | 9 | `reserved` | 0 |
589/// | 10.. | `slots` | `capacity` x [`ViolationSlot`] |
590///
591/// The newest entry is in slot `(head + capacity - 1) % capacity`; reading
592/// `capacity` slots backwards from there gives newest to oldest, and a slot
593/// whose `seq` is 0 was never written.
594///
595/// Present only when the image asked for the boot report
596/// (`NROS_BOOT_REPORT=1`, Zephyr `CONFIG_NROS_BOOT_REPORT=y`) -- the same
597/// opt-in as the SWD boot record it sits beside, so an image that does not
598/// opt in is unchanged. Read it with `scripts/read-violation-record.py`.
599#[repr(C)]
600#[derive(Debug)]
601pub struct ViolationRecord<const N: usize> {
602 pub magic: AtomicU32,
603 pub version: AtomicU32,
604 pub capacity: AtomicU32,
605 pub slot_words: AtomicU32,
606 pub total: AtomicU32,
607 pub head: AtomicU32,
608 pub dropped: AtomicU32,
609 pub suppressed_before_arm: AtomicU32,
610 pub armed: AtomicU32,
611 pub reserved: AtomicU32,
612 pub slots: [ViolationSlot; N],
613}
614
615impl<const N: usize> ViolationRecord<N> {
616 /// A valid, empty record: the header is written at compile time, so a
617 /// record with `total == 0` reads as "no violation since boot" rather than
618 /// as uninitialised RAM.
619 pub const fn new() -> Self {
620 Self {
621 magic: AtomicU32::new(RECORD_MAGIC),
622 version: AtomicU32::new(RECORD_VERSION),
623 capacity: AtomicU32::new(N as u32),
624 slot_words: AtomicU32::new(RECORD_SLOT_WORDS),
625 total: AtomicU32::new(0),
626 head: AtomicU32::new(0),
627 dropped: AtomicU32::new(0),
628 suppressed_before_arm: AtomicU32::new(0),
629 armed: AtomicU32::new(0),
630 reserved: AtomicU32::new(0),
631 slots: [const { ViolationSlot::new() }; N],
632 }
633 }
634
635 /// Store one violation, overwriting the oldest when full. Returns its
636 /// 1-based sequence number.
637 ///
638 /// Lock-free and callable from any executor thread: the slot is claimed by
639 /// a `fetch_add` on `total`, so two executors never write one slot unless
640 /// `N` more violations land during one write.
641 pub fn store(&self, v: &Violation) -> u32 {
642 let seq = self.total.fetch_add(1, Ordering::AcqRel).wrapping_add(1);
643 if N == 0 {
644 return seq;
645 }
646 let slot = &self.slots[(seq as usize - 1) % N];
647 slot.seq.store(0, Ordering::Release);
648 slot.rule.store(rule_code(v.rule), Ordering::Relaxed);
649 slot.fqn_hash.store(fqn_hash(v.fqn), Ordering::Relaxed);
650 slot.measured.store(v.measured, Ordering::Relaxed);
651 slot.declared.store(v.declared, Ordering::Relaxed);
652 slot.fqn_addr
653 .store(v.fqn.as_ptr() as usize as u32, Ordering::Relaxed);
654 slot.fqn_len.store(v.fqn.len() as u32, Ordering::Relaxed);
655 slot.seq.store(seq, Ordering::Release);
656 self.head.store(seq % N as u32, Ordering::Relaxed);
657 self.dropped
658 .store(seq.saturating_sub(N as u32), Ordering::Relaxed);
659 seq
660 }
661
662 /// The stored entries, newest first, as `(seq, rule code, fqn hash,
663 /// measured, declared)`. For tests and on-target self-checks; a debugger
664 /// reads the words directly.
665 pub fn newest_first(&self, mut f: impl FnMut(u32, u32, u32, u32, u32)) {
666 let total = self.total.load(Ordering::Acquire);
667 let n = (total as usize).min(N);
668 for k in 0..n {
669 let seq = total - k as u32;
670 let slot = &self.slots[(seq as usize - 1) % N];
671 if slot.seq.load(Ordering::Acquire) != seq {
672 continue; // overwritten or mid-write
673 }
674 f(
675 seq,
676 slot.rule.load(Ordering::Relaxed),
677 slot.fqn_hash.load(Ordering::Relaxed),
678 slot.measured.load(Ordering::Relaxed),
679 slot.declared.load(Ordering::Relaxed),
680 );
681 }
682 }
683}
684
685impl<const N: usize> Default for ViolationRecord<N> {
686 fn default() -> Self {
687 Self::new()
688 }
689}
690
691/// phase-474 I1 -- THE record, by name: `nm zephyr.elf | grep
692/// NROS_VIOLATION_RECORD`. See [`ViolationRecord`] for the layout.
693#[cfg(nros_boot_report)]
694#[unsafe(no_mangle)]
695pub static NROS_VIOLATION_RECORD: ViolationRecord<MAX_VIOLATIONS> = ViolationRecord::new();
696
697/// phase-474 I2 -- application arm requests, image-wide. Each call of
698/// [`request_monitor_arming`] bumps it; an executor waiting for the
699/// application arms when it sees a value other than the one it was opened
700/// with, so a request made before an executor existed does not arm it.
701static ARM_REQUESTS: AtomicU32 = AtomicU32::new(0);
702
703/// phase-474 I2 -- the application says its start-up is over: arm the
704/// contract monitors of every executor in the image that waits for this
705/// (`NROS_MONITOR_ARM_ON_CALL`, or [`Executor::set_monitor_arming`]).
706///
707/// Callable from anywhere -- a node's callback, another thread -- because it
708/// only bumps an atomic; each executor arms at its next spin. On an executor
709/// that is already armed (the default: armed at the first spin) it does
710/// nothing. C: `rclc_monitors_arm()`; C++: `nros::arm_monitors()`.
711///
712/// [`Executor::set_monitor_arming`]: super::Executor::set_monitor_arming
713pub fn request_monitor_arming() {
714 ARM_REQUESTS.fetch_add(1, Ordering::AcqRel);
715}
716
717/// The current arm-request count (for an executor to remember at open).
718pub(crate) fn arm_requests() -> u32 {
719 ARM_REQUESTS.load(Ordering::Acquire)
720}
721
722/// phase-474 I2 -- when an executor's contract monitors start judging.
723///
724/// A verdict reached before arming is COUNTED
725/// ([`Executor::violations_suppressed_before_arm`], and the SWD record's
726/// `suppressed_before_arm`) but not stored, logged, sunk or traced: start-up
727/// -- registration, the first join, inputs that have not started yet -- looks
728/// like overruns, jitter and silence, and judging it as steady state filled
729/// the island's ring with eight start-up entries before any act ran.
730///
731/// The default is read from Kconfig / the build environment only
732/// (`NROS_MONITOR_ARM_ON_CALL`, `NROS_MONITOR_ARM_GRACE_MS`); the contract
733/// carries no key for it.
734///
735/// [`Executor::violations_suppressed_before_arm`]: super::Executor::violations_suppressed_before_arm
736#[derive(Debug, Clone, Copy, PartialEq, Eq)]
737pub struct MonitorArming {
738 /// Wait for [`request_monitor_arming`] / `Executor::arm_monitors`.
739 /// `false` (the default) arms at the first spin, today's behaviour.
740 pub on_call: bool,
741 /// Arm anyway this many ms after the first spin; 0 = no deadline. With
742 /// `on_call` it is the backstop for an application that never calls; on
743 /// its own it is a grace period for an application with no lifecycle.
744 pub grace_ms: u32,
745}
746
747impl MonitorArming {
748 /// The build's default (`NROS_MONITOR_ARM_ON_CALL`,
749 /// `NROS_MONITOR_ARM_GRACE_MS`).
750 pub const DEFAULT: Self = Self {
751 on_call: crate::config::MONITOR_ARM_ON_CALL,
752 grace_ms: crate::config::MONITOR_ARM_GRACE_MS,
753 };
754
755 /// Armed from the start: no wait for a call and no grace period.
756 pub const fn armed_at_first_spin(self) -> bool {
757 !self.on_call && self.grace_ms == 0
758 }
759}
760
761/// Issue 1635 -- where an executor hands its violations when the image asked
762/// it to (`Executor::set_violation_sink`).
763///
764/// A function and an opaque context rather than a closure, because the
765/// executor is not generic and the reporter it reaches lives in a crate above
766/// this one (`nros-cpp` publishes on `/diagnostics` through
767/// `nros-diagnostics`). Called from inside the executor's spin (never from
768/// inside a user callback) -- so the sink may publish.
769///
770/// phase-474 I7 -- called OFF the judged dispatch, not at detection: a stored
771/// verdict is queued ([`REPORT_QUEUE`] deep) and the queue is handed over at
772/// the end of a spin whose next timer is further away than the last report
773/// cost (or once a verdict has waited [`REPORT_DEFER_MAX_US`]). The sink gets
774/// every pending verdict, oldest first, and returns how many of them it took
775/// -- a PREFIX, so it may coalesce as many as fit one message and leave the
776/// rest for the next slot. Returning 0 is a failure the sink has counted;
777/// the executor then drops the oldest pending verdict so a report that can
778/// never be sent cannot stall the queue (it stays in the ring and the record).
779///
780/// On the safety island the report at detection cost 15-20 ms of spin per
781/// verdict over a 921,600-baud serial link, inside the tick being judged, and
782/// that cost tripped `release-jitter-runtime` and `timer-overrun-runtime`:
783/// one verdict bred more.
784///
785/// # Safety
786/// The sink is called with the `ctx` it was installed with; whoever installs it
787/// guarantees `ctx` is valid for every call until the sink is replaced or the
788/// executor is dropped.
789pub type ViolationSink = unsafe fn(ctx: *mut core::ffi::c_void, pending: &[Violation]) -> usize;
790
791/// phase-474 I7 -- how many stored verdicts wait for the sink. A verdict
792/// stored while the queue is full evicts the oldest pending one, which is
793/// counted (`ReportQueue::overflowed`); both stay in the ring and the SWD
794/// record. Four is the `DiagnosticArray` status capacity of the generated
795/// type, so one coalesced report never needs more.
796pub const REPORT_QUEUE: usize = 4;
797
798/// phase-474 I7 -- the longest a pending verdict waits for a slot with slack
799/// before it is reported anyway, in microseconds. Bounds the deferral on an
800/// executor whose timers never leave a gap as long as the report costs.
801pub const REPORT_DEFER_MAX_US: u64 = 1_000_000;
802
803/// phase-474 I7 -- verdicts stored but not yet handed to the sink, and what
804/// the hand-over has cost. One struct so the executor size test can name it.
805#[derive(Debug, Default)]
806pub(crate) struct ReportQueue {
807 /// Oldest first.
808 pub(crate) pending: heapless::Vec<Violation, REPORT_QUEUE>,
809 /// Monotonic us at which the oldest pending verdict was first seen by a
810 /// flush, for [`REPORT_DEFER_MAX_US`].
811 pub(crate) since_us: Option<u64>,
812 /// What the last hand-over took, and the worst one, in us.
813 pub(crate) last_cost_us: u32,
814 pub(crate) max_cost_us: u32,
815 /// Pending verdicts evicted by a newer one before any slot came (saturating).
816 pub(crate) overflowed: u32,
817 /// Verdicts the sink was handed and could not send (saturating).
818 pub(crate) failed: u32,
819 /// The reporter's own spin time since the last release-jitter sample: it
820 /// is the monitor's cost, stated by `max_cost_us`, not lateness of the
821 /// application's release, so the jitter rule does not charge it.
822 pub(crate) debt_us: u64,
823}
824
825/// phase-474 I1 -- everything an executor does with a detected violation, in
826/// one place: the switches, the sink, the ring and its counters.
827///
828/// The ring keeps the LATEST [`MAX_VIOLATIONS`] verdicts: a push into a full
829/// ring evicts the oldest and counts it in `dropped`. It used to refuse the
830/// push instead, so a board whose start-up filled the ring stored nothing
831/// after it (the Autoware Safety Island's W31 bring-up: 8 of 8 slots, all
832/// start-up). `total` counts every verdict stored since boot, so
833/// `total - dropped - ring.len()` is how many a drain has already taken.
834pub(crate) struct ViolationChannel<'s> {
835 /// Issue 0514 -- log each violation at detection.
836 pub(crate) report: bool,
837 /// phase-474 I1 -- drain the ring at the end of every spin and report each
838 /// entry with its sequence number and the counters
839 /// ([`Executor::set_violation_drain_report`]). Replaces the log at
840 /// detection while on, so one verdict is one line.
841 ///
842 /// [`Executor::set_violation_drain_report`]: super::Executor::set_violation_drain_report
843 pub(crate) drain_report: bool,
844 /// Issue 1635 -- the image's reporter. The context is a `usize` so the
845 /// executor keeps its auto traits; it is the installer's pointer.
846 /// phase-474 I7 -- fed from `reports`, off the judged dispatch.
847 pub(crate) sink: Option<(ViolationSink, usize)>,
848 /// phase-474 I7 -- what waits for the sink.
849 pub(crate) reports: ReportQueue,
850 /// phase-409 -- CARVED, at `MAX_VIOLATIONS` (no `ExecutorSizing` knob:
851 /// the build-time depth is the capability).
852 pub(crate) ring: super::storage::CarvedVec<'s, Violation>,
853 /// phase-474 I1/I2 -- the counters and the arming state.
854 pub(crate) counts: ViolationCounts,
855}
856
857/// phase-474 I1/I2 -- the scalars of a [`ViolationChannel`]: its counters and
858/// its arming state. One struct so `the_executor_value_does_not_scale_with_the_knobs`
859/// can name what they cost; none of them scales with a knob.
860#[derive(Debug, Clone, Copy)]
861pub(crate) struct ViolationCounts {
862 /// Violations this executor stored since boot (wrapping). It is also the
863 /// sequence number of the newest entry in `ring`: every stored verdict is
864 /// pushed, so the ring holds `total - len + 1 ..= total`.
865 pub(crate) total: u32,
866 /// Issue 0514 / phase-474 I1 -- entries evicted from the full ring before
867 /// any drain took them (saturating).
868 pub(crate) dropped: u32,
869 /// phase-474 I2 -- whether verdicts are stored. Until then they are only
870 /// counted in `suppressed_before_arm`.
871 pub(crate) armed: bool,
872 /// phase-474 I2 -- verdicts reached before arming (saturating).
873 pub(crate) suppressed_before_arm: u32,
874 /// phase-474 I2 -- the arming policy ([`MonitorArming`]), kept as its two
875 /// fields so the flags pack beside the ones above.
876 pub(crate) arm_on_call: bool,
877 pub(crate) arm_grace_ms: u32,
878 /// phase-474 I2 -- whether the executor has spun, and the monotonic ms
879 /// (wrapping `u32`) of its first spin, which the grace deadline counts
880 /// from.
881 pub(crate) spun: bool,
882 pub(crate) first_spin_ms: u32,
883 /// phase-474 I2 -- [`arm_requests`] when the executor opened; a different
884 /// value later is the application's call.
885 pub(crate) arm_requests_seen: u32,
886}
887
888impl<'s> ViolationChannel<'s> {
889 pub(crate) fn new(ring: super::storage::CarvedVec<'s, Violation>) -> Self {
890 Self {
891 report: true,
892 drain_report: crate::config::VIOLATION_DRAIN_REPORT,
893 sink: None,
894 reports: ReportQueue::default(),
895 ring,
896 counts: ViolationCounts {
897 total: 0,
898 dropped: 0,
899 armed: MonitorArming::DEFAULT.armed_at_first_spin(),
900 suppressed_before_arm: 0,
901 arm_on_call: MonitorArming::DEFAULT.on_call,
902 arm_grace_ms: MonitorArming::DEFAULT.grace_ms,
903 spun: false,
904 first_spin_ms: 0,
905 arm_requests_seen: arm_requests(),
906 },
907 }
908 }
909
910 /// Issue 1635 / phase-474 I1 -- THE one place a detected violation goes:
911 /// the log floor (issue 0514) unless the drain hook reports instead, the
912 /// image's sink's queue when one is installed (sent later, phase-474 I7),
913 /// the trace marker when callback
914 /// tracing is compiled in, the SWD record when the image keeps one, and
915 /// the ring [`drain_violations`] reads.
916 ///
917 /// [`drain_violations`]: super::Executor::drain_violations
918 pub(crate) fn record(&mut self, v: Violation) {
919 if !self.counts.armed {
920 // phase-474 I2 -- start-up: counted, never stored.
921 self.counts.suppressed_before_arm = self.counts.suppressed_before_arm.saturating_add(1);
922 #[cfg(nros_boot_report)]
923 NROS_VIOLATION_RECORD
924 .suppressed_before_arm
925 .fetch_add(1, Ordering::Relaxed);
926 return;
927 }
928 if self.report && !self.drain_report {
929 log_violation(&v);
930 }
931 if self.sink.is_some() {
932 // phase-474 I7 -- queued, not sent: the report leaves at the end of
933 // a spin with slack (`Executor::flush_violation_reports`), so it
934 // never costs the dispatch being judged.
935 if self.reports.pending.is_full() {
936 self.reports.pending.remove(0);
937 self.reports.overflowed = self.reports.overflowed.saturating_add(1);
938 }
939 let _ = self.reports.pending.push(v.clone());
940 }
941 // `total` is this executor's sequence number for the verdict: the log
942 // line and the trace marker carry it, so the two name one verdict by
943 // one number. The SWD record numbers across every executor in the
944 // image; on a single-executor image the two coincide.
945 self.counts.total = self.counts.total.wrapping_add(1);
946 #[cfg(nros_boot_report)]
947 NROS_VIOLATION_RECORD.store(&v);
948 #[cfg(feature = "trace-callbacks")]
949 super::callback_trace::violation(self.counts.total, &v);
950 if self.ring.capacity() == 0 {
951 self.counts.dropped = self.counts.dropped.saturating_add(1);
952 return;
953 }
954 if self.ring.len() == self.ring.capacity() {
955 self.ring.pop_front();
956 self.counts.dropped = self.counts.dropped.saturating_add(1);
957 }
958 // Cannot fail: a slot was just made if the ring was full.
959 let _ = self.ring.push(v);
960 }
961
962 /// Hand every entry still in the ring to `f`, oldest first, with its
963 /// sequence number, and empty the ring.
964 pub(crate) fn drain(&mut self, mut f: impl FnMut(u32, &Violation)) {
965 let n = self.ring.len() as u32;
966 let first = self.counts.total.wrapping_sub(n).wrapping_add(1);
967 for (k, v) in self.ring.iter().enumerate() {
968 f(first.wrapping_add(k as u32), v);
969 }
970 self.ring.clear();
971 }
972
973 /// phase-474 I1 -- the drain-and-report hook: one log line per entry with
974 /// its sequence number and the counters, so a reader of the log knows how
975 /// many it missed. Called at the end of a spin when `drain_report` is on.
976 pub(crate) fn drain_and_report(&mut self) {
977 if !self.drain_report || self.ring.is_empty() {
978 return;
979 }
980 let (total, dropped) = (self.counts.total, self.counts.dropped);
981 self.drain(|seq, v| log_drained_violation(seq, total, dropped, v));
982 }
983}
984
985/// Per-spec accounting state (parallel to the spec table).
986#[derive(Debug, Clone, Copy, Default)]
987pub(crate) struct MonitorState {
988 /// Window opened (a plain bool, not a 0-sentinel on the timestamp —
989 /// `now_us == 0` is a legitimate first sample on freshly-started
990 /// monotonic clocks).
991 pub(crate) opened: bool,
992 pub(crate) window_start_us: u64,
993 pub(crate) count_at_window_start: u32,
994 /// Suppress duplicate reports: only re-report after a clean window.
995 pub(crate) violated_last_window: bool,
996 /// W3b.5 — separate dedup for the latency rule on the same spec row.
997 pub(crate) latency_violated_last_window: bool,
998 /// phase-474 I8 -- the open window has been moved onto the stream's
999 /// phase (or judged as opened). See [`check_rate`].
1000 pub(crate) aligned: bool,
1001}
1002
1003/// Pure rate check over one window boundary. Returns `Some(violation)`
1004/// when the window elapsed AND the measured rate is below the declared
1005/// minimum (and we didn't already report last window).
1006///
1007/// Extracted from the executor so the math is unit-testable without a
1008/// session: publish counting is injected via the cell, time via `now_us`.
1009pub(crate) fn check_rate(
1010 spec: &MonitorSpec,
1011 state: &mut MonitorState,
1012 now_us: u64,
1013) -> Option<Violation> {
1014 if spec.min_rate_hz_milli == 0 {
1015 return None;
1016 }
1017 let count = spec.cell.count.load(Ordering::Relaxed);
1018 if !state.opened {
1019 // First observation: open the window, no verdict yet.
1020 state.opened = true;
1021 state.aligned = false;
1022 state.window_start_us = now_us;
1023 state.count_at_window_start = count;
1024 return None;
1025 }
1026 let window_us = now_us.saturating_sub(state.window_start_us);
1027 if !state.aligned && window_us < RATE_CHECK_INTERVAL_US {
1028 // phase-474 I8 -- the first window starts at the first tick that sees
1029 // a new sample, not where it was opened. The tick runs before the
1030 // spin's dispatch, so in steady state every boundary sits one tick
1031 // after a publish and the count matches the span. The window opened
1032 // at boot or by arming sits wherever that spin fell: off the
1033 // stream's phase it spans N samples over N periods plus the offset,
1034 // and a stream at exactly its declared rate read short (the island:
1035 // 9984 of 10000 mHz, once, after arming). Re-anchoring here puts the
1036 // first window on the same phase the later ones have. A stream that
1037 // stays silent is not held open: once a whole window passes with no
1038 // sample it is judged as opened (below), at 0.
1039 if count != state.count_at_window_start {
1040 state.aligned = true;
1041 state.window_start_us = now_us;
1042 state.count_at_window_start = count;
1043 }
1044 return None;
1045 }
1046 state.aligned = true;
1047 if window_us < RATE_CHECK_INTERVAL_US {
1048 return None;
1049 }
1050 let published = count.wrapping_sub(state.count_at_window_start) as u64;
1051 // milli-Hz = published * 1e3 / window_s = published * 1e9 / window_us
1052 let measured_milli_hz =
1053 (published.saturating_mul(1_000_000_000) / window_us.max(1)).min(u32::MAX as u64) as u32;
1054 // phase-474 I9 -- judge the count, not the floored quotient. The window
1055 // closes at the first tick that finds it elapsed, so `window_us` is the
1056 // declared interval plus that tick's lateness, while a stream at exactly
1057 // its declared minimum has published one sample per period: 50 over
1058 // 5.0005 s reads 9999 of 10000 mHz and was judged slow (the island, every
1059 // window, reported once thanks to `violated_last_window`). A stream at
1060 // the declared minimum delivers at least floor(min_rate * window) samples
1061 // in any window, whatever its phase; fewer than that is slow.
1062 let required = (spec.min_rate_hz_milli as u64).saturating_mul(window_us) / 1_000_000_000;
1063
1064 // Roll the window.
1065 state.window_start_us = now_us;
1066 state.count_at_window_start = count;
1067
1068 if published < required {
1069 if state.violated_last_window {
1070 return None; // still violated — already reported
1071 }
1072 state.violated_last_window = true;
1073 Some(Violation {
1074 rule: "rate-hierarchy-runtime",
1075 fqn: spec.fqn,
1076 measured: measured_milli_hz,
1077 declared: spec.min_rate_hz_milli,
1078 })
1079 } else {
1080 state.violated_last_window = false;
1081 None
1082 }
1083}
1084
1085/// Pure latency check: drains the spec cell's window-max take→publish
1086/// latency and fires when it exceeds the declared node-path budget.
1087/// Same report-once-until-recovery semantics as the rate rule; runs on
1088/// every monitor tick (the cell accumulates between ticks, so no window
1089/// bookkeeping is needed — draining IS the window roll).
1090pub(crate) fn check_latency(spec: &MonitorSpec, state: &mut MonitorState) -> Option<Violation> {
1091 if spec.max_latency_ms == 0 {
1092 return None;
1093 }
1094 let max_us = spec.cell.max_latency_us.swap(0, Ordering::Relaxed);
1095 let max_ms = max_us / 1_000;
1096 if max_ms > spec.max_latency_ms {
1097 if state.latency_violated_last_window {
1098 return None;
1099 }
1100 state.latency_violated_last_window = true;
1101 Some(Violation {
1102 rule: "max-latency-runtime",
1103 fqn: spec.fqn,
1104 measured: max_ms,
1105 declared: spec.max_latency_ms,
1106 })
1107 } else {
1108 // A quiet window (no dispatch attributed) also counts as clean —
1109 // recovery resets the dedup like the rate rule's clean window.
1110 state.latency_violated_last_window = false;
1111 None
1112 }
1113}
1114
1115/// phase10-B3 -- bind a subscription slot as the route trigger of every row
1116/// whose [`MonitorSpec::route_trigger`] is `topic`. Called once per
1117/// registration; the first subscription on the topic keeps the binding.
1118pub(crate) fn route_bind(table: &[MonitorSpec], topic: &str, slot: usize) {
1119 if slot >= ROUTE_SLOT_MASK as usize {
1120 return;
1121 }
1122 for spec in table.iter().take(MAX_MONITORS) {
1123 if spec.route_max_latency_ms == 0 || spec.route_trigger != topic {
1124 continue;
1125 }
1126 let w = spec.cell.route.load(Ordering::Relaxed);
1127 if w & ROUTE_SLOT_MASK == 0 {
1128 spec.cell.route.store(
1129 (w & !ROUTE_SLOT_MASK) | (slot as u32 + 1),
1130 Ordering::Relaxed,
1131 );
1132 }
1133 }
1134}
1135
1136/// phase10-B3 -- forget a released slot's route bindings and any take it left
1137/// waiting.
1138pub(crate) fn route_unbind(table: &[MonitorSpec], slot: usize) {
1139 for spec in table.iter().take(MAX_MONITORS) {
1140 let w = spec.cell.route.load(Ordering::Relaxed);
1141 if w & ROUTE_SLOT_MASK == slot as u32 + 1 {
1142 spec.cell.route.store(0, Ordering::Relaxed);
1143 }
1144 }
1145}
1146
1147/// phase10-B3 -- the dispatch in `slot` took a sample at `take_us`: every
1148/// route whose trigger is that slot starts its clock, unless an earlier take
1149/// is still waiting for the output (the route is judged from the EARLIEST
1150/// unanswered take: a take that arrives just after a tick waits for the whole
1151/// next period, and that is the wait the budget has to hold).
1152pub(crate) fn route_take(table: &[MonitorSpec], slot: usize, take_us: u64) {
1153 let bound = slot as u32 + 1;
1154 for spec in table.iter().take(MAX_MONITORS) {
1155 let w = spec.cell.route.load(Ordering::Relaxed);
1156 if w & ROUTE_SLOT_MASK != bound || w & ROUTE_PENDING != 0 {
1157 continue;
1158 }
1159 spec.cell
1160 .route_take_us
1161 .store(take_us as u32, Ordering::Relaxed);
1162 spec.cell.route.store(w | ROUTE_PENDING, Ordering::Relaxed);
1163 }
1164}
1165
1166/// phase10-B3 -- a dispatch ended at `end_us`. Every route output whose
1167/// counter advanced during it (`counts_before`, as the latency rule reads it)
1168/// and that has a take waiting is judged: take to dispatch end, an upper bound
1169/// on take to that publish, against the path's budget. Over budget is
1170/// reported once until a route is within it again.
1171///
1172/// phase-503 -- a SUPERVISED route ([`MonitorSpec::route_deadline_ms`])
1173/// answered at or after its deadline is also overdue: it is reported as
1174/// `route-deadline-runtime` and its reaction requested, as `route_overdue`
1175/// would have done had it run in time. Under `MECHANISM_EXECUTOR` this is the
1176/// only detection a single long callback gets: the callback that holds the
1177/// spin thread past the deadline publishes the output before the executor's
1178/// overdue check can run, so the miss is caught here, at completion -- late
1179/// by the callback's overrun. A route whose deadline was already caught
1180/// (`route_overdue` dropped its take) is not pending any more and is not
1181/// judged again, so each take gets at most one deadline verdict.
1182///
1183/// `report(row, violation, record, react)`: `record` is `false` while the
1184/// route is still marked violated from an earlier verdict (report once until
1185/// a route is within its budget again); `react` asks the caller to call the
1186/// reaction hook, on every late supervised take as `route_overdue` does.
1187pub(crate) fn route_judge(
1188 table: &[MonitorSpec],
1189 counts_before: &[u32; MAX_MONITORS],
1190 end_us: u64,
1191 mut report: impl FnMut(usize, Violation, bool, bool),
1192) {
1193 for (k, spec) in table.iter().take(MAX_MONITORS).enumerate() {
1194 if spec.route_max_latency_ms == 0 {
1195 continue;
1196 }
1197 let w = spec.cell.route.load(Ordering::Relaxed);
1198 if w & ROUTE_PENDING == 0 || spec.cell.count.load(Ordering::Relaxed) == counts_before[k] {
1199 continue;
1200 }
1201 let took_us = (end_us as u32).wrapping_sub(spec.cell.route_take_us.load(Ordering::Relaxed));
1202 let measured_ms = took_us / 1_000;
1203 let first = w & ROUTE_VIOLATED == 0;
1204 let late = measured_ms > spec.route_max_latency_ms;
1205 let overdue =
1206 route_supervised(spec) && took_us >= spec.route_deadline_ms.saturating_mul(1_000);
1207 let mut next = w & !ROUTE_PENDING;
1208 if late && first {
1209 report(
1210 k,
1211 Violation {
1212 rule: "route-latency-runtime",
1213 fqn: spec.fqn,
1214 measured: measured_ms,
1215 declared: spec.route_max_latency_ms,
1216 },
1217 true,
1218 false,
1219 );
1220 }
1221 if overdue {
1222 report(
1223 k,
1224 Violation {
1225 rule: "route-deadline-runtime",
1226 fqn: spec.fqn,
1227 measured: measured_ms,
1228 declared: spec.route_deadline_ms,
1229 },
1230 first,
1231 true,
1232 );
1233 }
1234 if late || overdue {
1235 next |= ROUTE_VIOLATED;
1236 } else {
1237 next &= !ROUTE_VIOLATED;
1238 }
1239 spec.cell.route.store(next, Ordering::Relaxed);
1240 }
1241}
1242
1243/// phase-503 -- whether a pending route of `spec` has a supervised deadline.
1244#[inline]
1245fn route_supervised(spec: &MonitorSpec) -> bool {
1246 spec.route_deadline_ms > 0 && spec.route_max_latency_ms > 0
1247}
1248
1249/// phase-503 -- time from now to the earliest deadline of a pending supervised
1250/// route, in microseconds (0 when one is already due), or `None` when no
1251/// supervised route is waiting. This is the executor's wake source for the
1252/// rule, and what a port's independent timer (`MECHANISM_TIMER`) arms on.
1253///
1254/// The due time is `take + route_deadline_ms`, derived from the stored take
1255/// rather than kept in a second cell: both are the clock's low 32 bits, so the
1256/// difference is exact across a wrap as long as a route waits less than ~71
1257/// minutes, and a supervised one is reported long before that.
1258pub fn route_next_due_us(table: &[MonitorSpec], now_us: u64) -> Option<u64> {
1259 let mut best: Option<u64> = None;
1260 for spec in table.iter().take(MAX_MONITORS) {
1261 if !route_supervised(spec) {
1262 continue;
1263 }
1264 let w = spec.cell.route.load(Ordering::Relaxed);
1265 if w & ROUTE_PENDING == 0 {
1266 continue;
1267 }
1268 let waited_us =
1269 (now_us as u32).wrapping_sub(spec.cell.route_take_us.load(Ordering::Relaxed));
1270 let deadline_us = spec.route_deadline_ms.saturating_mul(1_000);
1271 let left = deadline_us.saturating_sub(waited_us) as u64;
1272 best = Some(best.map_or(left, |b| b.min(left)));
1273 if left == 0 {
1274 break;
1275 }
1276 }
1277 best
1278}
1279
1280/// phase-503 -- judge every pending supervised route at `now_us`: a take still
1281/// unanswered at or after `take + route_deadline_ms` is OVERDUE. Its waiting
1282/// take is dropped (the route stops pending, so the next take starts a new
1283/// one) and the route is marked [`ROUTE_VIOLATED`], as `route_judge` marks an
1284/// over-budget route.
1285///
1286/// `react(row, violation, first)` is called for every overdue route: `first`
1287/// is `false` while the route is still marked violated from an earlier verdict
1288/// (report once until a route is within its budget again, the latency rule's
1289/// rule), and the caller records the violation only when it is `true`. The
1290/// reaction is requested every time, since each overdue take is a missed
1291/// deadline the fallback has to cover.
1292pub(crate) fn route_overdue(
1293 table: &[MonitorSpec],
1294 now_us: u64,
1295 mut react: impl FnMut(usize, Violation, bool),
1296) {
1297 for (k, spec) in table.iter().take(MAX_MONITORS).enumerate() {
1298 if !route_supervised(spec) {
1299 continue;
1300 }
1301 let w = spec.cell.route.load(Ordering::Relaxed);
1302 if w & ROUTE_PENDING == 0 {
1303 continue;
1304 }
1305 let waited_us =
1306 (now_us as u32).wrapping_sub(spec.cell.route_take_us.load(Ordering::Relaxed));
1307 if waited_us < spec.route_deadline_ms.saturating_mul(1_000) {
1308 continue;
1309 }
1310 spec.cell
1311 .route
1312 .store((w & !ROUTE_PENDING) | ROUTE_VIOLATED, Ordering::Relaxed);
1313 react(
1314 k,
1315 Violation {
1316 rule: "route-deadline-runtime",
1317 fqn: spec.fqn,
1318 measured: waited_us / 1_000,
1319 declared: spec.route_deadline_ms,
1320 },
1321 w & ROUTE_VIOLATED == 0,
1322 );
1323 }
1324}
1325
1326/// phase-503 -- whether any row carries a supervised route deadline.
1327pub(crate) fn route_deadline_active(table: &[MonitorSpec]) -> bool {
1328 table.iter().any(route_supervised)
1329}
1330
1331/// phase10-B3 -- whether any row carries a route contract.
1332pub(crate) fn route_active(table: &[MonitorSpec]) -> bool {
1333 table.iter().any(|m| m.route_max_latency_ms > 0)
1334}
1335
1336/// phase10-B3 -- drop every waiting take and the report-once state (arming:
1337/// a take from start-up is not a route of RUN's).
1338pub(crate) fn route_reset(table: &[MonitorSpec]) {
1339 for spec in table.iter() {
1340 let w = spec.cell.route.load(Ordering::Relaxed);
1341 spec.cell
1342 .route
1343 .store(w & ROUTE_SLOT_MASK, Ordering::Relaxed);
1344 }
1345}
1346
1347/// Per-age-spec dedup state.
1348#[derive(Debug, Clone, Copy, Default)]
1349pub(crate) struct AgeState {
1350 pub(crate) violated_last_window: bool,
1351 /// phase-462 W2 -- the silence rule's window. `opened` is a plain bool for
1352 /// the same reason [`MonitorState`]'s is: `now_us == 0` is a legitimate
1353 /// first sample.
1354 pub(crate) silence_opened: bool,
1355 /// Report-once-until-recovery, matching every other rule here.
1356 pub(crate) silence_reported: bool,
1357 /// Monotonic MILLISECONDS of the last check that saw a take (or of the
1358 /// first check, which starts the lease rather than judging it).
1359 ///
1360 /// Milliseconds and `u32`, not the uss the clock hands out, because this
1361 /// array is INLINE in the `Executor` value (`[AgeState; MAX_AGE_MONITORS]`) and
1362 /// that value has a byte budget a knob-scaled table must not blow (issue
1363 /// 0961 / `storage.rs`'s `the_executor_value_does_not_scale_with_the_knobs`
1364 /// -- a `u64` here costs 8 rows x 16 B and fails it). The unit is the
1365 /// contract's own (`max_age_ms`), the comparison is a wrapping delta, so
1366 /// the 49-day rollover is not a discontinuity, and a silence window is
1367 /// never shorter than a millisecond.
1368 pub(crate) last_take_ms: u32,
1369}
1370
1371/// phase-462 W2 -- the on-target liveliness lease: a contracted subscription
1372/// that has taken NOTHING for a whole `max_age_ms` window.
1373///
1374/// The bound needs no new declaration and deliberately does not invent one.
1375/// `max_age_ms` already says how old this input's data may be; data that never
1376/// arrives is older than that by the same clock, so the window IS the declared
1377/// bound. (`on_violation` selects what the executor then DOES about it; the
1378/// reaction beyond reporting is the fault hook phase-462 W3 owns.)
1379///
1380/// Why this is not `max-age-runtime` with a longer arm: that rule judges a
1381/// message that arrived. An input that stops entirely produces no message to
1382/// judge, so it leaves the age rule silent forever -- the exact failure a
1383/// `max_age_ms` promise is made against, and the one play_launch's reaction
1384/// engine catches on the Linux side of the same contract.
1385///
1386/// `now_us` is the executor's monotonic clock, `None` on a build with no clock
1387/// at all. Then nothing is judged: a clock that never moves is a window that
1388/// never elapses, and guessing a window from spin counts would put a tolerance
1389/// nobody declared into a safety rule.
1390fn check_silence(spec: &AgeMonitorSpec, state: &mut AgeState, took: u32, now_us: u64) -> bool {
1391 let now_ms = (now_us / 1_000) as u32;
1392 if !state.silence_opened {
1393 state.silence_opened = true;
1394 state.last_take_ms = now_ms;
1395 return false;
1396 }
1397 if took > 0 {
1398 state.last_take_ms = now_ms;
1399 state.silence_reported = false;
1400 return false;
1401 }
1402 if now_ms.wrapping_sub(state.last_take_ms) < spec.max_age_ms {
1403 return false;
1404 }
1405 if state.silence_reported {
1406 return false; // still silent -- already reported
1407 }
1408 state.silence_reported = true;
1409 true
1410}
1411
1412/// Pure age check: drains the sub cell's window-max take-age and fires
1413/// when it exceeds the declared bound. Report-once-until-recovery.
1414///
1415/// phase-462 W2 -- also the silence rule's tick, because the two verdicts read
1416/// ONE observation (what this endpoint took since the last check) and reading
1417/// it twice would let one rule's drain hide the other's evidence. They cannot
1418/// both fire: with no takes the drained age is 0, so `max-age-runtime` has
1419/// nothing to judge.
1420pub(crate) fn check_age(
1421 spec: &AgeMonitorSpec,
1422 state: &mut AgeState,
1423 now_us: Option<u64>,
1424) -> Option<Violation> {
1425 if spec.max_age_ms == 0 {
1426 return None;
1427 }
1428 let took = spec.cell.takes.swap(0, Ordering::Relaxed);
1429 if let Some(now_us) = now_us
1430 && check_silence(spec, state, took, now_us)
1431 {
1432 return Some(Violation {
1433 rule: "silence-runtime",
1434 fqn: spec.fqn,
1435 // Takes in the window, against the window the contract declares.
1436 measured: 0,
1437 declared: spec.max_age_ms,
1438 });
1439 }
1440 let max_ms = spec.cell.max_age_ms.swap(0, Ordering::Relaxed);
1441 if max_ms > spec.max_age_ms {
1442 if state.violated_last_window {
1443 return None;
1444 }
1445 state.violated_last_window = true;
1446 Some(Violation {
1447 rule: "max-age-runtime",
1448 fqn: spec.fqn,
1449 measured: max_ms,
1450 declared: spec.max_age_ms,
1451 })
1452 } else {
1453 state.violated_last_window = false;
1454 None
1455 }
1456}
1457
1458// ---------------------------------------------------------------------------
1459// phase-462 W3b -- the hazard observer.
1460// ---------------------------------------------------------------------------
1461
1462/// One guard member: which monitor row's verdict says this topic has faulted.
1463///
1464/// The observer adds no detector of its own. A guard topic has faulted when
1465/// the row the image already bakes for it says so: an age row whose
1466/// `silence-runtime` lease is reported, or a rate row whose last window was
1467/// short (`min-rate-runtime`). Reading the rows' state rather than their
1468/// violations keeps the two in step by construction -- a hazard cannot be
1469/// detected by a rule that did not fire, nor stay detected after it recovered.
1470#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1471pub enum GuardRef {
1472 /// Row index into the executor's age table.
1473 Age(u16),
1474 /// Row index into the executor's rate/latency table.
1475 Rate(u16),
1476}
1477
1478/// One fault a hazard covers: a topic, or a redundant set of topics.
1479#[derive(Debug, Clone, Copy)]
1480pub struct GuardSpec {
1481 /// `true`: the guard faults only when EVERY member has (a redundant
1482 /// set). `false`: when any member has.
1483 pub all_of: bool,
1484 /// The guard's members.
1485 pub members: &'static [GuardRef],
1486}
1487
1488/// [`HazardSpec::sink`] when the hazard's reaction states no `safe_state`
1489/// publisher this image monitors: the observer reports detection and
1490/// recovery, and no reaction.
1491pub const NO_SINK: u16 = u16::MAX;
1492
1493/// One hazard's observer state. Baked as a `static` beside the spec, the way
1494/// [`PubMonitorCell`] is, so the observer adds a slice to the `Executor`
1495/// value and nothing per hazard (issue 0961).
1496#[derive(Debug, Default)]
1497pub struct HazardCell {
1498 /// 0 clear, 1 detected, 2 detected and reacted.
1499 state: AtomicU32,
1500 /// Monotonic ms the fault began: the faulting age row's last take, or the
1501 /// tick that first saw a rate guard short.
1502 fault_ms: AtomicU32,
1503 /// The sink publisher's count when the fault was detected.
1504 sink_count: AtomicU32,
1505}
1506
1507impl HazardCell {
1508 /// A cleared cell, for a `static`.
1509 pub const fn new() -> Self {
1510 Self {
1511 state: AtomicU32::new(0),
1512 fault_ms: AtomicU32::new(0),
1513 sink_count: AtomicU32::new(0),
1514 }
1515 }
1516}
1517
1518/// One contracted hazard (`contracts.hazards`), lowered by the bake.
1519#[derive(Debug, Clone, Copy)]
1520pub struct HazardSpec {
1521 /// The hazard's contract key (`<scope>/<name>`), the report key.
1522 pub key: &'static str,
1523 /// Declared `ftti_ms`; 0 = none declared.
1524 pub ftti_ms: u32,
1525 /// The reacting path's `safe_state.settle_ms`, added to the reaction
1526 /// time as play_launch adds it; 0 = none.
1527 pub settle_ms: u32,
1528 /// The faults the hazard covers; any one of them is the hazard.
1529 pub guards: &'static [GuardSpec],
1530 /// Rate-table row of the `safe_state.emits` publisher, or [`NO_SINK`].
1531 pub sink: u16,
1532 /// The observer's state.
1533 pub cell: &'static HazardCell,
1534}
1535
1536const HAZARD_CLEAR: u32 = 0;
1537const HAZARD_DETECTED: u32 = 1;
1538const HAZARD_REACTED: u32 = 2;
1539
1540/// Whether `member` has faulted, and since when (ms) when the row knows.
1541fn member_fault(
1542 member: GuardRef,
1543 monitor_states: &[MonitorState],
1544 age_states: &[AgeState],
1545) -> Option<Option<u32>> {
1546 match member {
1547 GuardRef::Age(i) => {
1548 let s = age_states.get(i as usize)?;
1549 s.silence_reported.then_some(Some(s.last_take_ms))
1550 }
1551 GuardRef::Rate(i) => {
1552 let s = monitor_states.get(i as usize)?;
1553 s.violated_last_window.then_some(None)
1554 }
1555 }
1556}
1557
1558/// Whether any guard of `spec` has faulted, and the earliest fault start the
1559/// faulting rows know.
1560fn hazard_fault(
1561 spec: &HazardSpec,
1562 monitor_states: &[MonitorState],
1563 age_states: &[AgeState],
1564) -> Option<Option<u32>> {
1565 let mut faulted = false;
1566 let mut since: Option<u32> = None;
1567 for g in spec.guards {
1568 let mut seen = 0usize;
1569 let mut g_since: Option<u32> = None;
1570 for m in g.members {
1571 if let Some(at) = member_fault(*m, monitor_states, age_states) {
1572 seen += 1;
1573 if let Some(at) = at {
1574 g_since = Some(g_since.map_or(at, |s: u32| s.min(at)));
1575 }
1576 }
1577 }
1578 let guard_faulted = if g.all_of {
1579 !g.members.is_empty() && seen == g.members.len()
1580 } else {
1581 seen > 0
1582 };
1583 if guard_faulted {
1584 faulted = true;
1585 if let Some(at) = g_since {
1586 since = Some(since.map_or(at, |s| s.min(at)));
1587 }
1588 }
1589 }
1590 faulted.then_some(since)
1591}
1592
1593/// phase-462 W3b -- one tick of a hazard's observer, after the monitor rows
1594/// have run. Emits, in play_launch's vocabulary so one log reader serves both
1595/// sides of a bridge:
1596///
1597/// - `hazard-detected` when a guard first faults: measured is the ms since
1598/// the fault began, declared the FTTI;
1599/// - `hazard-reaction` when the `safe_state` publisher publishes after that:
1600/// measured is fault-to-reaction plus `settle_ms`, declared the FTTI;
1601/// - `hazard-recovered` when no guard is faulted any more: measured is the
1602/// ms the hazard lasted.
1603///
1604/// `fqn` is the hazard's key on all three.
1605pub(crate) fn check_hazard(
1606 spec: &HazardSpec,
1607 monitor_table: &[MonitorSpec],
1608 monitor_states: &[MonitorState],
1609 age_states: &[AgeState],
1610 now_ms: u32,
1611 mut emit: impl FnMut(Violation),
1612) {
1613 let cell = spec.cell;
1614 let state = cell.state.load(Ordering::Relaxed);
1615 let sink_count = monitor_table
1616 .get(spec.sink as usize)
1617 .map(|m| m.cell.count.load(Ordering::Relaxed));
1618 let fault = hazard_fault(spec, monitor_states, age_states);
1619 let v = |rule, measured: u32| Violation {
1620 rule,
1621 fqn: spec.key,
1622 measured,
1623 declared: spec.ftti_ms,
1624 };
1625 match (state, fault) {
1626 (HAZARD_CLEAR, Some(since)) => {
1627 let at = since.unwrap_or(now_ms);
1628 cell.fault_ms.store(at, Ordering::Relaxed);
1629 cell.sink_count
1630 .store(sink_count.unwrap_or(0), Ordering::Relaxed);
1631 cell.state.store(HAZARD_DETECTED, Ordering::Relaxed);
1632 emit(v("hazard-detected", now_ms.wrapping_sub(at)));
1633 }
1634 (HAZARD_DETECTED, Some(_)) => {
1635 if let Some(count) = sink_count
1636 && count != cell.sink_count.load(Ordering::Relaxed)
1637 {
1638 cell.state.store(HAZARD_REACTED, Ordering::Relaxed);
1639 let since = now_ms.wrapping_sub(cell.fault_ms.load(Ordering::Relaxed));
1640 emit(v("hazard-reaction", since.saturating_add(spec.settle_ms)));
1641 }
1642 }
1643 (HAZARD_DETECTED | HAZARD_REACTED, None) => {
1644 cell.state.store(HAZARD_CLEAR, Ordering::Relaxed);
1645 let lasted = now_ms.wrapping_sub(cell.fault_ms.load(Ordering::Relaxed));
1646 emit(v("hazard-recovered", lasted));
1647 }
1648 _ => {}
1649 }
1650}
1651
1652#[cfg(test)]
1653mod hazard_tests {
1654 use super::*;
1655
1656 static PUB_CELL: PubMonitorCell = PubMonitorCell::new();
1657 static SUB_CELL: SubMonitorCell = SubMonitorCell::new();
1658 static SINK_CELL: PubMonitorCell = PubMonitorCell::new();
1659
1660 static RATE_ROWS: [MonitorSpec; 2] = [
1661 MonitorSpec {
1662 topic: "/scan",
1663 fqn: "/sensor/scan",
1664 min_rate_hz_milli: 10_000,
1665 max_latency_ms: 0,
1666 route_trigger: "",
1667 route_max_latency_ms: 0,
1668 route_deadline_ms: 0,
1669 reaction_id: 0,
1670 mechanism: 0,
1671 cell: &PUB_CELL,
1672 },
1673 MonitorSpec {
1674 topic: "/stop",
1675 fqn: "/handler/stop",
1676 min_rate_hz_milli: 0,
1677 max_latency_ms: 0,
1678 route_trigger: "",
1679 route_max_latency_ms: 0,
1680 route_deadline_ms: 0,
1681 reaction_id: 0,
1682 mechanism: 0,
1683 cell: &SINK_CELL,
1684 },
1685 ];
1686 #[allow(dead_code)]
1687 static AGE_ROW: AgeMonitorSpec = AgeMonitorSpec {
1688 topic: "/scan",
1689 fqn: "/handler/scan",
1690 max_age_ms: 100,
1691 cell: &SUB_CELL,
1692 };
1693
1694 fn run(
1695 spec: &HazardSpec,
1696 mon: &[MonitorState],
1697 age: &[AgeState],
1698 now: u32,
1699 ) -> std::vec::Vec<Violation> {
1700 let mut out = std::vec::Vec::new();
1701 check_hazard(spec, &RATE_ROWS, mon, age, now, |v| out.push(v));
1702 out
1703 }
1704
1705 /// detected -> reaction -> recovered, each once, on a silence guard, with
1706 /// times measured from the guard's last take.
1707 #[test]
1708 fn a_silence_guard_detects_reacts_and_recovers_once_each() {
1709 static CELL: HazardCell = HazardCell::new();
1710 static GUARDS: [GuardSpec; 1] = [GuardSpec {
1711 all_of: false,
1712 members: &[GuardRef::Age(0)],
1713 }];
1714 let spec = HazardSpec {
1715 key: "demo.launch.xml/scan_lost",
1716 ftti_ms: 500,
1717 settle_ms: 30,
1718 guards: &GUARDS,
1719 sink: 1,
1720 cell: &CELL,
1721 };
1722 let mon = [MonitorState::default(); 2];
1723 let mut age = [AgeState::default(); 1];
1724 assert!(run(&spec, &mon, &age, 1_000).is_empty(), "clear: nothing");
1725
1726 age[0].silence_reported = true;
1727 age[0].last_take_ms = 900;
1728 let v = run(&spec, &mon, &age, 1_000);
1729 assert_eq!(v.len(), 1);
1730 assert_eq!((v[0].rule, v[0].fqn), ("hazard-detected", spec.key));
1731 assert_eq!((v[0].measured, v[0].declared), (100, 500));
1732 assert!(run(&spec, &mon, &age, 1_010).is_empty(), "detected once");
1733
1734 SINK_CELL.count.fetch_add(1, Ordering::Relaxed);
1735 let v = run(&spec, &mon, &age, 1_050);
1736 assert_eq!(v.len(), 1);
1737 assert_eq!(v[0].rule, "hazard-reaction");
1738 assert_eq!(v[0].measured, 150 + 30, "fault-to-reaction plus settle");
1739 SINK_CELL.count.fetch_add(1, Ordering::Relaxed);
1740 assert!(run(&spec, &mon, &age, 1_060).is_empty(), "reacted once");
1741
1742 age[0].silence_reported = false;
1743 let v = run(&spec, &mon, &age, 1_200);
1744 assert_eq!(v.len(), 1);
1745 assert_eq!((v[0].rule, v[0].measured), ("hazard-recovered", 300));
1746 assert!(run(&spec, &mon, &age, 1_210).is_empty(), "recovered once");
1747 }
1748
1749 /// A redundant set faults only when every member has.
1750 #[test]
1751 fn an_all_of_guard_waits_for_every_member() {
1752 static CELL: HazardCell = HazardCell::new();
1753 static GUARDS: [GuardSpec; 1] = [GuardSpec {
1754 all_of: true,
1755 members: &[GuardRef::Age(0), GuardRef::Rate(0)],
1756 }];
1757 let spec = HazardSpec {
1758 key: "demo.launch.xml/both_lost",
1759 ftti_ms: 0,
1760 settle_ms: 0,
1761 guards: &GUARDS,
1762 sink: NO_SINK,
1763 cell: &CELL,
1764 };
1765 let mut mon = [MonitorState::default(); 2];
1766 let mut age = [AgeState::default(); 1];
1767 age[0].silence_reported = true;
1768 assert!(
1769 run(&spec, &mon, &age, 10).is_empty(),
1770 "one of two: no hazard"
1771 );
1772 mon[0].violated_last_window = true;
1773 let v = run(&spec, &mon, &age, 20);
1774 assert_eq!(v[0].rule, "hazard-detected");
1775 assert_eq!(v[0].declared, 0, "no FTTI declared");
1776 }
1777
1778 /// The text `contract_monitors::render_monitor_rs_full` emits for the
1779 /// W3a/W3b fixture (one sensor, one handler, one hazard), verbatim but for
1780 /// the crate root. It is here so the GENERATED shape is compiled against
1781 /// the real types: a renderer change that stops compiling has to change
1782 /// this copy too, in plain sight.
1783 #[allow(dead_code, non_upper_case_globals)]
1784 mod rendered {
1785 use crate::executor::monitor::{
1786 AgeMonitorSpec, MonitorSpec, PubMonitorCell, SubMonitorCell,
1787 };
1788
1789 static NROS_MONITOR_CELL_0: PubMonitorCell = PubMonitorCell::new();
1790 static NROS_MONITOR_CELL_1: PubMonitorCell = PubMonitorCell::new();
1791 static NROS_AGE_CELL_0: SubMonitorCell = SubMonitorCell::new();
1792
1793 pub static NROS_MONITORS: &[MonitorSpec] = &[
1794 MonitorSpec {
1795 topic: "/stop",
1796 fqn: "/handler/stop",
1797 min_rate_hz_milli: 0u32,
1798 max_latency_ms: 20u32,
1799 route_trigger: "/scan",
1800 route_max_latency_ms: 20u32,
1801 route_deadline_ms: 0u32,
1802 reaction_id: 0u16,
1803 mechanism: 0u8,
1804 cell: &NROS_MONITOR_CELL_0,
1805 },
1806 MonitorSpec {
1807 topic: "/scan",
1808 fqn: "/sensor/scan",
1809 min_rate_hz_milli: 10000u32,
1810 max_latency_ms: 0u32,
1811 route_trigger: "",
1812 route_max_latency_ms: 0u32,
1813 route_deadline_ms: 0u32,
1814 reaction_id: 0u16,
1815 mechanism: 0u8,
1816 cell: &NROS_MONITOR_CELL_1,
1817 },
1818 ];
1819
1820 pub static NROS_AGE_MONITORS: &[AgeMonitorSpec] = &[AgeMonitorSpec {
1821 topic: "/scan",
1822 fqn: "/handler/scan",
1823 max_age_ms: 100u32,
1824 cell: &NROS_AGE_CELL_0,
1825 }];
1826
1827 use crate::executor::monitor::{GuardRef, GuardSpec, HazardCell, HazardSpec};
1828
1829 static NROS_HAZARD_CELL_0: HazardCell = HazardCell::new();
1830 const NROS_HAZARD_GUARDS_0: &[GuardSpec] = &[GuardSpec {
1831 all_of: false,
1832 members: &[GuardRef::Age(0u16), GuardRef::Rate(1u16)],
1833 }];
1834
1835 pub static NROS_HAZARDS: &[HazardSpec] = &[HazardSpec {
1836 key: "demo.launch.xml/scan_lost",
1837 ftti_ms: 0u32,
1838 settle_ms: 30u32,
1839 guards: NROS_HAZARD_GUARDS_0,
1840 sink: 0u16,
1841 cell: &NROS_HAZARD_CELL_0,
1842 }];
1843 }
1844
1845 /// The rendered tables drive the observer: the age row's silence is the
1846 /// hazard, and the sink row the renderer chose is the one it watches.
1847 #[test]
1848 fn the_rendered_include_drives_the_observer() {
1849 let spec = &rendered::NROS_HAZARDS[0];
1850 let mon = [MonitorState::default(); 2];
1851 let mut age = [AgeState::default(); 1];
1852 age[0].silence_reported = true;
1853 age[0].last_take_ms = 40;
1854 let mut out = std::vec::Vec::new();
1855 check_hazard(spec, rendered::NROS_MONITORS, &mon, &age, 100, |v| {
1856 out.push(v)
1857 });
1858 assert_eq!((out[0].rule, out[0].measured), ("hazard-detected", 60));
1859 assert_eq!(
1860 rendered::NROS_MONITORS[spec.sink as usize].fqn,
1861 "/handler/stop"
1862 );
1863 }
1864
1865 /// With no sink the observer never claims a reaction it cannot see.
1866 #[test]
1867 fn no_sink_no_reaction() {
1868 static CELL: HazardCell = HazardCell::new();
1869 static GUARDS: [GuardSpec; 1] = [GuardSpec {
1870 all_of: false,
1871 members: &[GuardRef::Rate(0)],
1872 }];
1873 let spec = HazardSpec {
1874 key: "k",
1875 ftti_ms: 100,
1876 settle_ms: 0,
1877 guards: &GUARDS,
1878 sink: NO_SINK,
1879 cell: &CELL,
1880 };
1881 let mut mon = [MonitorState::default(); 2];
1882 mon[0].violated_last_window = true;
1883 let v = run(&spec, &mon, &[], 50);
1884 assert_eq!((v[0].rule, v[0].measured), ("hazard-detected", 0));
1885 PUB_CELL.count.fetch_add(5, Ordering::Relaxed);
1886 assert!(run(&spec, &mon, &[], 60).is_empty());
1887 }
1888}
1889
1890#[cfg(test)]
1891mod tests {
1892 use super::*;
1893
1894 static CELL: PubMonitorCell = PubMonitorCell::new();
1895
1896 /// phase-496 W1 -- the Autoware Safety Island's shape: 14 rate rows and
1897 /// no age rows fit an executor sized 14/0, and a 15th rate row is refused
1898 /// with the knob to raise NAMED, not truncated.
1899 #[test]
1900 fn a_table_past_the_knob_is_refused_and_names_it() {
1901 assert_eq!(check_capacity_against(14, 0, 14, 0), Ok(()));
1902 let full = check_capacity_against(15, 0, 14, 0).expect_err("15 > 14 must refuse");
1903 assert_eq!(full.knob, "NROS_EXECUTOR_MAX_MONITORS");
1904 assert_eq!((full.rows, full.capacity), (15, 14));
1905 let said = std::format!("{full}");
1906 assert!(
1907 said.contains("raise NROS_EXECUTOR_MAX_MONITORS"),
1908 "the refusal must name the knob: {said}"
1909 );
1910 let age = check_capacity_against(0, 1, 14, 0).expect_err("an age row past 0 must refuse");
1911 assert_eq!(age.knob, "NROS_EXECUTOR_MAX_AGE_MONITORS");
1912 // The two tables are separate knobs: a full age table does not borrow
1913 // from the rate one.
1914 assert!(check_capacity_against(3, 9, 14, 8).is_err());
1915 }
1916
1917 /// The generated consts ARE the public ones, so the installers and the
1918 /// spin loop read one number.
1919 #[test]
1920 fn the_public_bounds_are_the_generated_config() {
1921 assert_eq!(MAX_MONITORS, crate::config::MAX_MONITORS);
1922 assert_eq!(MAX_AGE_MONITORS, crate::config::MAX_AGE_MONITORS);
1923 assert_eq!(check_table_capacity(MAX_MONITORS, MAX_AGE_MONITORS), Ok(()));
1924 assert!(check_table_capacity(MAX_MONITORS + 1, 0).is_err());
1925 }
1926
1927 fn spec(min_milli: u32) -> MonitorSpec {
1928 MonitorSpec {
1929 topic: "/chatter",
1930 fqn: "/demo/talker/chatter",
1931 min_rate_hz_milli: min_milli,
1932 max_latency_ms: 0,
1933 route_trigger: "",
1934 route_max_latency_ms: 0,
1935 route_deadline_ms: 0,
1936 reaction_id: 0,
1937 mechanism: 0,
1938 cell: &CELL,
1939 }
1940 }
1941
1942 #[test]
1943 fn slow_publisher_fires_once_until_recovery() {
1944 CELL.count.store(0, Ordering::Relaxed);
1945 let s = spec(100_000); // 100 Hz declared
1946 let mut st = MonitorState::default();
1947
1948 // t=0: opens window.
1949 assert!(check_rate(&s, &mut st, 0).is_none());
1950 // 5 publishes in 5 s = 1 Hz — violation.
1951 CELL.count.store(5, Ordering::Relaxed);
1952 let v = check_rate(&s, &mut st, RATE_CHECK_INTERVAL_US).expect("fires");
1953 assert_eq!(v.rule, "rate-hierarchy-runtime");
1954 assert_eq!(v.fqn, "/demo/talker/chatter");
1955 assert_eq!(v.measured, 1_000);
1956 assert_eq!(v.declared, 100_000);
1957 // Still slow next window — suppressed (no re-report spam).
1958 CELL.count.store(10, Ordering::Relaxed);
1959 assert!(check_rate(&s, &mut st, 2 * RATE_CHECK_INTERVAL_US).is_none());
1960 // Recovers (500 publishes in 5 s = 100 Hz) — clean window resets.
1961 CELL.count.store(510, Ordering::Relaxed);
1962 assert!(check_rate(&s, &mut st, 3 * RATE_CHECK_INTERVAL_US).is_none());
1963 // Degrades again — fires again.
1964 CELL.count.store(511, Ordering::Relaxed);
1965 assert!(check_rate(&s, &mut st, 4 * RATE_CHECK_INTERVAL_US).is_some());
1966 }
1967
1968 /// phase-474 I8 -- the island's first window after arming read 10 Hz
1969 /// publishers at 9984 mHz and never again. The monitor tick runs at the
1970 /// top of a spin, before that spin's dispatch publishes, so in steady
1971 /// state every window boundary sits one spin after a publish and the
1972 /// count matches the span. The window arming opens sits wherever the
1973 /// arming spin fell: off the publisher's phase, it spans 50 publishes
1974 /// over 5007 ms and reads short. A stream at exactly its declared rate,
1975 /// armed mid-period, must not be judged slow.
1976 #[test]
1977 fn a_stream_at_its_rate_armed_mid_period_is_not_judged_slow() {
1978 static C8: PubMonitorCell = PubMonitorCell::new();
1979 let s = MonitorSpec {
1980 topic: "/i8",
1981 fqn: "/n/i8",
1982 min_rate_hz_milli: 10_000, // 10 Hz declared
1983 max_latency_ms: 0,
1984 route_trigger: "",
1985 route_max_latency_ms: 0,
1986 route_deadline_ms: 0,
1987 reaction_id: 0,
1988 mechanism: 0,
1989 cell: &C8,
1990 };
1991 // Published at 0, 100 ms, ..., 50 s: 501 samples before arming.
1992 C8.count.store(501, Ordering::Relaxed);
1993 let mut st = MonitorState::default();
1994 // Arming spin, off the 10 ms grid (a wake for another entry).
1995 assert!(check_rate(&s, &mut st, 50_003_000).is_none());
1996 let mut t = 50_010_000u64;
1997 while t <= 80_000_000 {
1998 // The tick, then the spin's dispatch: the 10 Hz timer publishes.
1999 let v = check_rate(&s, &mut st, t);
2000 assert!(v.is_none(), "judged slow at {t} us: {v:?}");
2001 if t.is_multiple_of(100_000) {
2002 C8.count.fetch_add(1, Ordering::Relaxed);
2003 }
2004 t += 10_000;
2005 }
2006 // And a stream that IS slow is still judged in the first window:
2007 // half its rate from arming on.
2008 static C9: PubMonitorCell = PubMonitorCell::new();
2009 let slow = MonitorSpec { cell: &C9, ..s };
2010 let mut st = MonitorState::default();
2011 assert!(check_rate(&slow, &mut st, 50_003_000).is_none());
2012 let mut t = 50_010_000u64;
2013 let mut fired = None;
2014 while t <= 60_000_000 && fired.is_none() {
2015 fired = check_rate(&slow, &mut st, t).map(|v| (t, v.measured));
2016 if t.is_multiple_of(200_000) {
2017 C9.count.fetch_add(1, Ordering::Relaxed);
2018 }
2019 t += 10_000;
2020 }
2021 let (at, measured) = fired.expect("a 5 Hz stream against 10 Hz is judged");
2022 assert!(measured <= 5_100, "{measured}");
2023 // Judged one window after its first sample: at most one of its
2024 // periods (200 ms) plus a tick later than the window arming opened.
2025 assert!(at <= 50_003_000 + RATE_CHECK_INTERVAL_US + 210_000, "{at}");
2026 // A stream silent from arming on is judged at the first window's end,
2027 // not held open waiting for a sample that never comes.
2028 static C10: PubMonitorCell = PubMonitorCell::new();
2029 let dead = MonitorSpec { cell: &C10, ..s };
2030 let mut st = MonitorState::default();
2031 assert!(check_rate(&dead, &mut st, 50_003_000).is_none());
2032 let v = check_rate(&dead, &mut st, 50_003_000 + RATE_CHECK_INTERVAL_US)
2033 .expect("silent stream judged");
2034 assert_eq!(v.measured, 0);
2035 }
2036
2037 /// phase-474 I9 -- the island's 10 Hz publishers read 9999 of 10000 mHz
2038 /// in every window once I8 aligned the first one: the spin that closes a
2039 /// window runs some hundreds of microseconds after the interval, so the
2040 /// quotient floors under the declared rate while the count is exact. The
2041 /// count judges; a stream at its declared rate checked late is not slow,
2042 /// a stream at half its rate still is.
2043 #[test]
2044 fn a_stream_at_its_rate_checked_late_is_not_judged_slow() {
2045 static C11: PubMonitorCell = PubMonitorCell::new();
2046 let s = MonitorSpec {
2047 topic: "/i9",
2048 fqn: "/n/i9",
2049 min_rate_hz_milli: 10_000,
2050 max_latency_ms: 0,
2051 route_trigger: "",
2052 route_max_latency_ms: 0,
2053 route_deadline_ms: 0,
2054 reaction_id: 0,
2055 mechanism: 0,
2056 cell: &C11,
2057 };
2058 let mut st = MonitorState::default();
2059 // Publishes at 0, 100 ms, ...; the monitor tick runs 700 us after
2060 // each publish, so every window spans the interval plus 700 us.
2061 let mut t = 0u64;
2062 while t <= 60_000_000 {
2063 if t.is_multiple_of(100_000) {
2064 C11.count.fetch_add(1, Ordering::Relaxed);
2065 }
2066 let v = check_rate(&s, &mut st, t + 700);
2067 assert!(v.is_none(), "judged slow at {t} us: {v:?}");
2068 t += 10_000;
2069 }
2070 // Half the rate, checked the same way: judged, measured about 5 Hz.
2071 static C12: PubMonitorCell = PubMonitorCell::new();
2072 let slow = MonitorSpec { cell: &C12, ..s };
2073 let mut st = MonitorState::default();
2074 let mut t = 0u64;
2075 let mut fired = None;
2076 while t <= 20_000_000 && fired.is_none() {
2077 if t.is_multiple_of(200_000) {
2078 C12.count.fetch_add(1, Ordering::Relaxed);
2079 }
2080 fired = check_rate(&slow, &mut st, t + 700).map(|v| v.measured);
2081 t += 10_000;
2082 }
2083 let measured = fired.expect("a 5 Hz stream against 10 Hz is judged");
2084 assert!((4_900..=5_100).contains(&measured), "{measured}");
2085 }
2086
2087 #[test]
2088 fn compliant_and_uncontracted_stay_silent() {
2089 static C2: PubMonitorCell = PubMonitorCell::new();
2090 let s = MonitorSpec {
2091 topic: "/t",
2092 fqn: "/n/t",
2093 min_rate_hz_milli: 500, // 0.5 Hz
2094 max_latency_ms: 0,
2095 route_trigger: "",
2096 route_max_latency_ms: 0,
2097 route_deadline_ms: 0,
2098 reaction_id: 0,
2099 mechanism: 0,
2100 cell: &C2,
2101 };
2102 let mut st = MonitorState::default();
2103 assert!(check_rate(&s, &mut st, 0).is_none());
2104 C2.count.store(5, Ordering::Relaxed); // 1 Hz measured ≥ 0.5 Hz declared
2105 assert!(check_rate(&s, &mut st, RATE_CHECK_INTERVAL_US).is_none());
2106
2107 // min_rate 0 = uncontracted: never fires, no state.
2108 let s0 = MonitorSpec {
2109 topic: "/t",
2110 fqn: "/n/t",
2111 min_rate_hz_milli: 0,
2112 max_latency_ms: 0,
2113 route_trigger: "",
2114 route_max_latency_ms: 0,
2115 route_deadline_ms: 0,
2116 reaction_id: 0,
2117 mechanism: 0,
2118 cell: &C2,
2119 };
2120 let mut st0 = MonitorState::default();
2121 assert!(check_rate(&s0, &mut st0, 10 * RATE_CHECK_INTERVAL_US).is_none());
2122 }
2123
2124 #[test]
2125 fn stale_take_fires_age_once_until_recovery() {
2126 static SC: SubMonitorCell = SubMonitorCell::new();
2127 let s = AgeMonitorSpec {
2128 topic: "/scan",
2129 fqn: "/perc/detector/scan",
2130 max_age_ms: 100,
2131 cell: &SC,
2132 };
2133 let mut st = AgeState::default();
2134
2135 // Fresh message: stamped 5 ms ago -- silent. Every tick here SEES a
2136 // take, so the silence rule never opens a window (phase-462 W2).
2137 SC.observe(1_000_000_000, 1_000_005_000);
2138 assert!(check_age(&s, &mut st, Some(0)).is_none());
2139 // Stale: 250 ms old — fires with the measured age.
2140 SC.observe(1_000_000_000, 1_000_250_000);
2141 let v = check_age(&s, &mut st, Some(1_000)).expect("fires");
2142 assert_eq!(v.rule, "max-age-runtime");
2143 assert_eq!(v.fqn, "/perc/detector/scan");
2144 assert_eq!(v.measured, 250);
2145 assert_eq!(v.declared, 100);
2146 // Still stale next window — suppressed.
2147 SC.observe(1_000_000_000, 1_000_300_000);
2148 assert!(check_age(&s, &mut st, Some(2_000)).is_none());
2149 // Recovers — clean window resets; stale again refires.
2150 SC.observe(1_000_000_000, 1_000_010_000);
2151 assert!(check_age(&s, &mut st, Some(3_000)).is_none());
2152 SC.observe(1_000_000_000, 1_000_999_000);
2153 assert!(check_age(&s, &mut st, Some(4_000)).is_some());
2154 }
2155
2156 /// phase-462 W2 -- an input that stops coming is a violation of the same
2157 /// `max_age_ms` promise, and the age rule cannot see it.
2158 #[test]
2159 fn a_silent_subscription_fires_once_until_it_is_fed() {
2160 static SC2: SubMonitorCell = SubMonitorCell::new();
2161 let s = AgeMonitorSpec {
2162 topic: "/scan",
2163 fqn: "/perc/detector/scan",
2164 max_age_ms: 100,
2165 cell: &SC2,
2166 };
2167 let mut st = AgeState::default();
2168
2169 // t=0 opens the lease; a first check is never a verdict.
2170 SC2.observe(1_000_000_000, 1_000_005_000);
2171 assert!(check_age(&s, &mut st, Some(0)).is_none());
2172 // 50 ms of nothing, inside the declared 100 ms window.
2173 assert!(check_age(&s, &mut st, Some(50_000)).is_none());
2174 // 120 ms of nothing: the lease is up.
2175 let v = check_age(&s, &mut st, Some(120_000)).expect("silence fires");
2176 assert_eq!(v.rule, "silence-runtime");
2177 assert_eq!(v.fqn, "/perc/detector/scan");
2178 assert_eq!(v.measured, 0, "nothing was taken");
2179 assert_eq!(v.declared, 100, "the window the contract declares");
2180 // Still silent -- one fault, not one per tick.
2181 assert!(check_age(&s, &mut st, Some(500_000)).is_none());
2182 // Fed again: the lease restarts and a later silence reports afresh.
2183 SC2.observe(1_000_000_000, 1_000_005_000);
2184 assert!(check_age(&s, &mut st, Some(600_000)).is_none());
2185 assert!(check_age(&s, &mut st, Some(650_000)).is_none());
2186 assert!(check_age(&s, &mut st, Some(800_000)).is_some());
2187 }
2188
2189 /// The negative control for the same rule, in three parts: an
2190 /// uncontracted endpoint, a fed endpoint, and a build with no clock.
2191 #[test]
2192 fn silence_needs_a_contract_a_gap_and_a_clock() {
2193 static SC3: SubMonitorCell = SubMonitorCell::new();
2194 // No age contract: nothing is promised, so nothing is judged.
2195 let uncontracted = AgeMonitorSpec {
2196 topic: "/t",
2197 fqn: "/n/t",
2198 max_age_ms: 0,
2199 cell: &SC3,
2200 };
2201 let mut st = AgeState::default();
2202 assert!(check_age(&uncontracted, &mut st, Some(0)).is_none());
2203 assert!(check_age(&uncontracted, &mut st, Some(10_000_000)).is_none());
2204
2205 // Contracted and fed on every tick: silent rule, silent drain.
2206 let s = AgeMonitorSpec {
2207 max_age_ms: 100,
2208 ..uncontracted
2209 };
2210 let mut st = AgeState::default();
2211 for tick in 0..10u64 {
2212 SC3.observe(1_000_000_000, 1_000_001_000);
2213 assert!(
2214 check_age(&s, &mut st, Some(tick * 1_000_000)).is_none(),
2215 "a fed endpoint never reports silence"
2216 );
2217 }
2218
2219 // No clock: degrade, never guess. A window measured in spins would be
2220 // a tolerance nobody declared.
2221 let mut st = AgeState::default();
2222 assert!(check_age(&s, &mut st, None).is_none());
2223 assert!(check_age(&s, &mut st, None).is_none());
2224 }
2225
2226 #[test]
2227 fn peek_stamp_reads_le_time_and_rejects_unstamped() {
2228 // Encapsulation header (4B) + sec=100 nsec=5000 at offset 4.
2229 let mut raw = [0u8; 12];
2230 raw[4..8].copy_from_slice(&100i32.to_le_bytes());
2231 raw[8..12].copy_from_slice(&5_000u32.to_le_bytes());
2232 assert_eq!(peek_stamp_us(&raw, 4), Some(100_000_005));
2233 // Zero / negative sec = unstamped: no age sample.
2234 assert_eq!(peek_stamp_us(&[0u8; 12], 4), None);
2235 // Short buffer: no panic, no sample.
2236 assert_eq!(peek_stamp_us(&raw[..8], 4), None);
2237 }
2238
2239 #[test]
2240 fn slow_path_fires_latency_once_until_recovery() {
2241 static C3: PubMonitorCell = PubMonitorCell::new();
2242 let s = MonitorSpec {
2243 topic: "/cmd",
2244 fqn: "/ctrl/control/cmd",
2245 min_rate_hz_milli: 0,
2246 max_latency_ms: 10,
2247 route_trigger: "",
2248 route_max_latency_ms: 0,
2249 route_deadline_ms: 0,
2250 reaction_id: 0,
2251 mechanism: 0,
2252 cell: &C3,
2253 };
2254 let mut st = MonitorState::default();
2255 // 4 ms dispatch — within budget.
2256 C3.max_latency_us.store(4_000, Ordering::Relaxed);
2257 assert!(check_latency(&s, &mut st).is_none());
2258 assert_eq!(C3.max_latency_us.load(Ordering::Relaxed), 0, "drained");
2259 // 25 ms dispatch — fires.
2260 C3.max_latency_us.store(25_000, Ordering::Relaxed);
2261 let v = check_latency(&s, &mut st).expect("fires");
2262 assert_eq!(v.rule, "max-latency-runtime");
2263 assert_eq!(v.measured, 25);
2264 assert_eq!(v.declared, 10);
2265 // Still slow — suppressed; recovery resets.
2266 C3.max_latency_us.store(30_000, Ordering::Relaxed);
2267 assert!(check_latency(&s, &mut st).is_none());
2268 C3.max_latency_us.store(1_000, Ordering::Relaxed);
2269 assert!(check_latency(&s, &mut st).is_none());
2270 C3.max_latency_us.store(30_000, Ordering::Relaxed);
2271 assert!(check_latency(&s, &mut st).is_some());
2272 }
2273}
2274
2275/// Issue #514 — emit one violation to the log.
2276///
2277/// A log line is deliberately the floor rather than a `/diagnostics`
2278/// publication: it needs no publisher, no topic wiring, and no contract
2279/// on the reporting path itself, so it works on a bare RTOS image and
2280/// during boot. Publishing the same verdicts as `DiagnosticArray`
2281/// belongs on top of this, not instead of it.
2282///
2283/// Free function rather than an `Executor` method because every call
2284/// site sits inside a loop that already borrows the executor's spec
2285/// tables.
2286pub(crate) fn log_violation(v: &Violation) {
2287 nros_log::log_warn!(
2288 nros_log::get_logger("nros"),
2289 "contract violation: {} {} measured={} declared={}",
2290 v.rule,
2291 v.fqn,
2292 v.measured,
2293 v.declared
2294 );
2295}
2296
2297/// phase-474 I1 -- the drain-and-report hook's line: the issue-0514 line plus
2298/// the verdict's sequence number and the counters, so a reader who sees line
2299/// `#12` after `#9` knows two were missed, and `dropped` says whether the ring
2300/// lost any before this drain.
2301fn log_drained_violation(seq: u32, total: u32, dropped: u32, v: &Violation) {
2302 nros_log::log_warn!(
2303 nros_log::get_logger("nros"),
2304 "contract violation #{}: {} {} measured={} declared={} (total={} dropped={})",
2305 seq,
2306 v.rule,
2307 v.fqn,
2308 v.measured,
2309 v.declared,
2310 total,
2311 dropped
2312 );
2313}
2314
2315/// Issue #505 — periodic activations a timer dropped because its
2316/// executor was blocked past the period boundary.
2317///
2318/// This rule exists because `check_rate` cannot see an isolated stall:
2319/// it samples publish counts over a ~5 s window, so 20 missed
2320/// activations of a 100 Hz loop are a 0.4% rate deficit — under any
2321/// sane declared minimum, silence. (And under
2322/// [`TimerOverrunPolicy::CatchUp`](super::arena::TimerOverrunPolicy)
2323/// the replayed activations refill the window entirely, so the rate
2324/// rule reports a HEALTHY loop while the tier is stalling.) The
2325/// overrun counter is exact, needs no window, and does not depend on
2326/// clock resolution.
2327///
2328/// `overruns` is the timer's monotonic saturating counter;
2329/// `last_reported` is the value at the previous check, so the verdict
2330/// is on the delta. Returns a violation when more than `tolerated`
2331/// activations were dropped since the last check.
2332pub(crate) fn check_timer_overrun(
2333 overruns: u32,
2334 last_reported: &mut u32,
2335 tolerated: u32,
2336) -> Option<Violation> {
2337 let dropped = overruns.saturating_sub(*last_reported);
2338 *last_reported = overruns;
2339 if dropped <= tolerated {
2340 return None;
2341 }
2342 Some(Violation {
2343 rule: "timer-overrun-runtime",
2344 // Timer entries carry no name at this altitude; same stand-in
2345 // as `deadline-miss-runtime`.
2346 fqn: "timer",
2347 measured: dropped,
2348 declared: tolerated,
2349 })
2350}
2351
2352/// Issue #515 — report a spin wake that arrived so late the cadence it
2353/// claims cannot have been met.
2354///
2355/// Like `check_timer_overrun` and unlike the rate/age/latency rules, this
2356/// needs no baked spec table. The bound is the spin period ITSELF: the
2357/// caller passes it to `spin_once` as the pacing quantum, it is what
2358/// `system.toml` declares as `spin_period_us`, and a wake later than a full
2359/// period past its predecessor means an activation's worth of cadence was
2360/// lost. That is a contract failure for any declared period, so there is
2361/// nothing further to declare.
2362///
2363/// The tolerance is one whole period rather than zero, and deliberately so.
2364/// Sub-period lateness is ordinary scheduling noise -- on the measured FVP
2365/// lane the executor is late on a large fraction of wakes while still
2366/// holding its rate -- and a rule that fired on each one would report a
2367/// healthy system as broken thousands of times a second. What is NOT
2368/// ordinary is being a full period late, because by then the wake that
2369/// should have happened in between never did.
2370///
2371/// phase-474 I11 -- for a `spin_once(period)` loop with no declared cadence
2372/// the lateness is release minus scheduled release on the executor's release
2373/// grid (a dropped release counts its age when dropped), not entry-to-entry
2374/// minus the period; see `Executor::record_release_jitter`.
2375///
2376/// `max_jitter_us` is the executor's high-water since the last check and
2377/// `last_reported` the value at the previous one, so the verdict is on the
2378/// delta -- the same shape as the overrun counter, and for the same reason:
2379/// a maximum that has not moved is not a new fault.
2380pub(crate) fn check_release_jitter(
2381 max_jitter_us: u64,
2382 last_reported: &mut u64,
2383 period_us: u64,
2384) -> Option<Violation> {
2385 if period_us == 0 || max_jitter_us <= *last_reported {
2386 return None;
2387 }
2388 *last_reported = max_jitter_us;
2389 if max_jitter_us < period_us {
2390 return None;
2391 }
2392 Some(Violation {
2393 rule: "release-jitter-runtime",
2394 // Same stand-in as the timer and deadline rules: the spin loop is
2395 // not an endpoint and carries no fqn at this altitude.
2396 fqn: "spin",
2397 measured: max_jitter_us.min(u32::MAX as u64) as u32,
2398 declared: period_us.min(u32::MAX as u64) as u32,
2399 })
2400}
2401
2402/// Report a spin thread whose stack has come closer to its end than the
2403/// declared minimum.
2404///
2405/// Unlike every other rule here the bound CANNOT be derived from something
2406/// already declared, and that is worth stating rather than papering over.
2407/// `check_timer_overrun` and `check_release_jitter` both judge against a
2408/// period the caller already passes in; there is no equivalent for a stack.
2409/// The executor never sees `stack_bytes` -- it lives in the spawn attr and
2410/// goes no further -- and the total is not portably queryable either:
2411/// FreeRTOS exposes the high-water mark and not the size it was taken
2412/// against, so even a percentage cannot be computed. A minimum headroom is
2413/// therefore a real declaration, and `min_bytes == 0` means the caller has
2414/// not made one, which disables the rule.
2415///
2416/// Reports on the WORST case, not on each crossing: `worst_reported` holds
2417/// the lowest headroom already reported, so a stack hovering just under the
2418/// bound says so once and then only when it gets worse. The same delta
2419/// discipline as the overrun and jitter rules, inverted because for headroom
2420/// smaller is worse.
2421pub(crate) fn check_stack_headroom(
2422 unused_bytes: usize,
2423 min_bytes: usize,
2424 worst_reported: &mut usize,
2425) -> Option<Violation> {
2426 if min_bytes == 0 || unused_bytes >= min_bytes {
2427 return None;
2428 }
2429 // `usize::MAX` is the "nothing reported yet" sentinel: any real headroom
2430 // is below it, so the first breach always reports.
2431 if *worst_reported != usize::MAX && unused_bytes >= *worst_reported {
2432 return None;
2433 }
2434 *worst_reported = unused_bytes;
2435 Some(Violation {
2436 rule: "stack-headroom-runtime",
2437 // The spin thread is not an endpoint; same stand-in as the timer,
2438 // deadline and jitter rules.
2439 fqn: "stack",
2440 measured: unused_bytes.min(u32::MAX as usize) as u32,
2441 declared: min_bytes.min(u32::MAX as usize) as u32,
2442 })
2443}
2444
2445#[cfg(test)]
2446mod stack_headroom_rule_tests {
2447 use super::*;
2448
2449 /// No declared minimum means no claim to breach.
2450 #[test]
2451 fn a_zero_minimum_disables_the_rule() {
2452 let mut worst = usize::MAX;
2453 assert!(check_stack_headroom(8, 0, &mut worst).is_none());
2454 }
2455
2456 #[test]
2457 fn headroom_at_the_bound_is_not_a_breach() {
2458 let mut worst = usize::MAX;
2459 assert!(check_stack_headroom(1024, 1024, &mut worst).is_none());
2460 assert!(check_stack_headroom(2048, 1024, &mut worst).is_none());
2461 }
2462
2463 #[test]
2464 fn reports_the_first_breach_with_both_numbers() {
2465 let mut worst = usize::MAX;
2466 let v = check_stack_headroom(512, 1024, &mut worst).expect("under the bound reports");
2467 assert_eq!(v.rule, "stack-headroom-runtime");
2468 assert_eq!(v.measured, 512);
2469 assert_eq!(v.declared, 1024);
2470 }
2471
2472 /// Smaller is worse for headroom, so the delta runs the other way.
2473 #[test]
2474 fn only_a_new_low_is_a_new_fault() {
2475 let mut worst = usize::MAX;
2476 assert!(check_stack_headroom(512, 1024, &mut worst).is_some());
2477 assert!(check_stack_headroom(512, 1024, &mut worst).is_none());
2478 assert!(check_stack_headroom(600, 1024, &mut worst).is_none());
2479 let v = check_stack_headroom(100, 1024, &mut worst).expect("a new low reports");
2480 assert_eq!(v.measured, 100);
2481 }
2482}
2483
2484#[cfg(test)]
2485mod publish_stamp_tests {
2486 use super::*;
2487
2488 /// CDR: 4-byte encapsulation header, then `Time { i32 sec; u32 nanosec }`
2489 /// little-endian, so `sec` sits at byte 4 — the layout `STAMP_OFFSET`
2490 /// encodes.
2491 fn cdr_with_stamp(sec: i32, nanosec: u32) -> [u8; 12] {
2492 let mut b = [0u8; 12];
2493 b[4..8].copy_from_slice(&sec.to_le_bytes());
2494 b[8..12].copy_from_slice(&nanosec.to_le_bytes());
2495 b
2496 }
2497
2498 #[test]
2499 fn records_the_age_of_what_was_published() {
2500 let cell = PubMonitorCell::new();
2501 let raw = cdr_with_stamp(10, 0); // stamped at 10_000_000 us
2502 observe_publish_stamp(&cell, &raw, 4, 10_500_000);
2503 assert_eq!(
2504 cell.last_publish_stamp_age_us.load(Ordering::Relaxed),
2505 500_000,
2506 "published data was half a second old"
2507 );
2508 }
2509
2510 /// The signature of a node that RE-STAMPED: it publishes data whose
2511 /// stamp is now, so downstream age is single-hop, not end-to-end.
2512 #[test]
2513 fn a_restamping_node_shows_near_zero_age() {
2514 let cell = PubMonitorCell::new();
2515 let raw = cdr_with_stamp(10, 0);
2516 observe_publish_stamp(&cell, &raw, 4, 10_000_000);
2517 assert_eq!(cell.last_publish_stamp_age_us.load(Ordering::Relaxed), 0);
2518 }
2519
2520 /// A state, not a window maximum: one stale message at startup must not
2521 /// pin the value for the life of the process.
2522 #[test]
2523 fn the_latest_publish_replaces_the_previous() {
2524 let cell = PubMonitorCell::new();
2525 observe_publish_stamp(&cell, &cdr_with_stamp(10, 0), 4, 12_000_000);
2526 assert_eq!(
2527 cell.last_publish_stamp_age_us.load(Ordering::Relaxed),
2528 2_000_000
2529 );
2530 observe_publish_stamp(&cell, &cdr_with_stamp(20, 0), 4, 20_100_000);
2531 assert_eq!(
2532 cell.last_publish_stamp_age_us.load(Ordering::Relaxed),
2533 100_000,
2534 "a fresh publish replaces the old age rather than maxing with it"
2535 );
2536 }
2537
2538 /// An unset stamp is not an age of `now`. `peek_stamp_us` rejects
2539 /// `sec <= 0`, so a zeroed header records nothing at all.
2540 #[test]
2541 fn an_unstamped_message_records_nothing() {
2542 let cell = PubMonitorCell::new();
2543 observe_publish_stamp(&cell, &cdr_with_stamp(0, 0), 4, 5_000_000);
2544 assert_eq!(
2545 cell.last_publish_stamp_age_us.load(Ordering::Relaxed),
2546 0,
2547 "no stamp means no observation, not an enormous age"
2548 );
2549 }
2550}
2551
2552/// Per-SchedContext liveness accounting, one per SC slot.
2553///
2554/// `Default` is DERIVED, not hand-written: every field's default is its type's
2555/// own (`false`, `0`), so a manual impl restates four of them and is one edit
2556/// away from disagreeing with the struct — `clippy::derivable_impls` is a
2557/// `-D warnings` error in the embedded lane, which is where this was caught.
2558#[derive(Debug, Clone, Copy, Default)]
2559pub(crate) struct AliveState {
2560 pub opened: bool,
2561 pub window_start_us: u64,
2562 pub count_at_window_start: u32,
2563 /// Report-once-until-recovery, matching the rate rule.
2564 pub violated_last_window: bool,
2565}
2566
2567/// One SchedContext's alive-supervision record: the counter the dispatch
2568/// path bumps and the window state the monitor tick reads.
2569///
2570/// A region of the carved backing (`storage::carve`), one per SC the
2571/// executor was sized for — NOT an inline `[_; MAX_SC]` in the header.
2572/// `MAX_SC` is a sizing knob, so an inline table grows the `Executor` value
2573/// with it, which is the regression issue 0961 exists to prevent.
2574#[derive(Debug, Clone, Copy, Default)]
2575pub(crate) struct AliveSlot {
2576 pub dispatches: u32,
2577 pub state: AliveState,
2578}
2579
2580/// Report a SchedContext that has not dispatched AT ALL over a full window
2581/// while declaring a period that says it should have.
2582///
2583/// AUTOSAR's Watchdog Manager separates ALIVE supervision -- did it run at
2584/// all, at roughly the right rate -- from DEADLINE supervision, did it finish
2585/// in time. Every rule here is the second kind or a variant of it: they fire
2586/// when something happens and is wrong. Nothing fires when a callback stops
2587/// happening altogether.
2588///
2589/// `rate-hierarchy-runtime` does not cover it. That rule counts PUBLISHES on
2590/// a contracted publisher, so a timer callback that silently stops firing
2591/// while its topic is published from elsewhere -- another tier, a bridge, a
2592/// republisher -- leaves it satisfied. The gap is a callback, not an
2593/// endpoint.
2594///
2595/// The bound needs no declaration: `period_us` is already on the SC, and a
2596/// context claiming a period that produced no activation in a window many
2597/// times longer than it is a contract failure for any period. `period_us ==
2598/// 0` declares no cadence, so nothing is judged.
2599///
2600/// ZERO, not a fraction. A throttled-but-alive context is a different fault
2601/// with different evidence -- `SporadicState` already counts window skips and
2602/// dispatches for exactly that -- and picking a percentage here would invent
2603/// a tolerance nobody declared. Zero activations against a declared period is
2604/// unambiguous.
2605pub(crate) fn check_alive(
2606 period_us: u64,
2607 dispatches: u32,
2608 state: &mut AliveState,
2609 now_us: u64,
2610) -> Option<Violation> {
2611 if period_us == 0 {
2612 return None;
2613 }
2614 if !state.opened {
2615 state.opened = true;
2616 state.window_start_us = now_us;
2617 state.count_at_window_start = dispatches;
2618 return None;
2619 }
2620 let window_us = now_us.saturating_sub(state.window_start_us);
2621 if window_us < RATE_CHECK_INTERVAL_US {
2622 return None;
2623 }
2624 let fired = dispatches.wrapping_sub(state.count_at_window_start);
2625 state.window_start_us = now_us;
2626 state.count_at_window_start = dispatches;
2627
2628 // Only judge a window long enough to have contained an activation. A
2629 // period longer than the check interval is not late merely for not having
2630 // fired yet.
2631 if window_us < period_us {
2632 return None;
2633 }
2634 if fired > 0 {
2635 state.violated_last_window = false;
2636 return None;
2637 }
2638 if state.violated_last_window {
2639 return None; // still silent — already reported
2640 }
2641 state.violated_last_window = true;
2642 Some(Violation {
2643 rule: "alive-supervision-runtime",
2644 // SCs carry no name at this altitude; same stand-in as the deadline
2645 // rule, which reports against the SC too.
2646 fqn: "sched-context",
2647 measured: 0,
2648 declared: (window_us / period_us).min(u32::MAX as u64) as u32,
2649 })
2650}
2651
2652#[cfg(test)]
2653mod alive_supervision_tests {
2654 use super::*;
2655
2656 const W: u64 = RATE_CHECK_INTERVAL_US;
2657
2658 #[test]
2659 fn a_zero_period_declares_no_cadence() {
2660 let mut st = AliveState::default();
2661 assert!(check_alive(0, 0, &mut st, 0).is_none());
2662 assert!(check_alive(0, 0, &mut st, W * 4).is_none());
2663 }
2664
2665 /// The first observation opens the window; there is no verdict yet.
2666 #[test]
2667 fn the_first_window_only_opens() {
2668 let mut st = AliveState::default();
2669 assert!(check_alive(1_000, 0, &mut st, 0).is_none());
2670 assert!(st.opened);
2671 }
2672
2673 #[test]
2674 fn silence_across_a_full_window_reports() {
2675 let mut st = AliveState::default();
2676 check_alive(1_000, 0, &mut st, 0);
2677 let v = check_alive(1_000, 0, &mut st, W).expect("no dispatch in a full window");
2678 assert_eq!(v.rule, "alive-supervision-runtime");
2679 assert_eq!(v.measured, 0);
2680 assert_eq!(v.declared, (W / 1_000) as u32);
2681 }
2682
2683 /// Any activation at all clears it -- this rule asks whether the context
2684 /// is alive, not whether it is keeping up.
2685 #[test]
2686 fn a_single_dispatch_is_enough() {
2687 let mut st = AliveState::default();
2688 check_alive(1_000, 0, &mut st, 0);
2689 assert!(check_alive(1_000, 1, &mut st, W).is_none());
2690 }
2691
2692 /// Report once, then stay quiet until it recovers, like the rate rule.
2693 #[test]
2694 fn continued_silence_is_not_a_new_fault() {
2695 let mut st = AliveState::default();
2696 check_alive(1_000, 0, &mut st, 0);
2697 assert!(check_alive(1_000, 0, &mut st, W).is_some());
2698 assert!(check_alive(1_000, 0, &mut st, W * 2).is_none());
2699 // Recovery, then silence again, reports afresh.
2700 assert!(check_alive(1_000, 5, &mut st, W * 3).is_none());
2701 assert!(check_alive(1_000, 5, &mut st, W * 4).is_some());
2702 }
2703
2704 /// A period longer than the window has not had its chance yet.
2705 #[test]
2706 fn a_period_longer_than_the_window_is_not_judged() {
2707 let mut st = AliveState::default();
2708 check_alive(W * 10, 0, &mut st, 0);
2709 assert!(check_alive(W * 10, 0, &mut st, W).is_none());
2710 }
2711}
2712
2713#[cfg(test)]
2714mod release_jitter_rule_tests {
2715 use super::*;
2716
2717 /// Sub-period lateness is noise, not a violation -- otherwise a
2718 /// healthy-but-jittery loop reports thousands of faults a second.
2719 #[test]
2720 fn tolerates_lateness_within_one_period() {
2721 let mut last = 0;
2722 assert!(check_release_jitter(4_000, &mut last, 5_000).is_none());
2723 assert_eq!(last, 4_000, "still recorded, so the next delta is honest");
2724 }
2725
2726 /// A full period late means the wake that belonged in between never
2727 /// happened.
2728 #[test]
2729 fn reports_a_wake_a_whole_period_late() {
2730 let mut last = 0;
2731 let v = check_release_jitter(5_000, &mut last, 5_000).expect("one period late reports");
2732 assert_eq!(v.rule, "release-jitter-runtime");
2733 assert_eq!(v.measured, 5_000);
2734 assert_eq!(v.declared, 5_000);
2735 }
2736
2737 /// The verdict is on the DELTA. A high-water that has not moved is the
2738 /// same fault already reported, not a new one.
2739 #[test]
2740 fn an_unchanged_maximum_is_not_a_new_fault() {
2741 let mut last = 0;
2742 assert!(check_release_jitter(9_000, &mut last, 5_000).is_some());
2743 assert!(check_release_jitter(9_000, &mut last, 5_000).is_none());
2744 assert!(check_release_jitter(12_000, &mut last, 5_000).is_some());
2745 }
2746
2747 /// No declared period means no cadence to be late for.
2748 #[test]
2749 fn a_zero_period_declares_nothing() {
2750 let mut last = 0;
2751 assert!(check_release_jitter(1_000_000, &mut last, 0).is_none());
2752 }
2753}
2754
2755#[cfg(test)]
2756mod timer_overrun_rule_tests {
2757 use super::*;
2758
2759 #[test]
2760 fn reports_the_delta_not_the_total() {
2761 let mut last = 0;
2762 let v = check_timer_overrun(19, &mut last, 0).expect("first drop reports");
2763 assert_eq!(v.rule, "timer-overrun-runtime");
2764 assert_eq!(v.measured, 19);
2765 // Same total on the next check is not a new fault.
2766 assert!(check_timer_overrun(19, &mut last, 0).is_none());
2767 // Only the newly dropped activations are reported.
2768 assert_eq!(check_timer_overrun(25, &mut last, 0).unwrap().measured, 6);
2769 }
2770
2771 #[test]
2772 fn a_clean_timer_never_reports() {
2773 let mut last = 0;
2774 for _ in 0..10 {
2775 assert!(check_timer_overrun(0, &mut last, 0).is_none());
2776 }
2777 }
2778
2779 #[test]
2780 fn tolerance_suppresses_small_drops() {
2781 let mut last = 0;
2782 assert!(check_timer_overrun(2, &mut last, 2).is_none());
2783 assert_eq!(check_timer_overrun(6, &mut last, 2).unwrap().measured, 4);
2784 }
2785}
2786
2787// phase-436 E4 — when the stack-headroom query is due. The decision is pure so
2788// the three cases are pinned without a clock or a port.
2789#[cfg(test)]
2790mod stack_headroom_throttle_tests {
2791 use super::*;
2792
2793 #[test]
2794 fn the_first_check_is_immediate_with_or_without_a_clock() {
2795 assert!(stack_headroom_check_due(Some(5), None, 0));
2796 assert!(stack_headroom_check_due(None, None, 0));
2797 }
2798
2799 #[test]
2800 fn a_clocked_check_waits_one_full_interval() {
2801 let i = STACK_HEADROOM_CHECK_INTERVAL_US;
2802 assert!(!stack_headroom_check_due(
2803 Some(1_000 + i - 1),
2804 Some(1_000),
2805 0
2806 ));
2807 assert!(stack_headroom_check_due(Some(1_000 + i), Some(1_000), 0));
2808 }
2809
2810 #[test]
2811 fn a_clockless_build_checks_once_per_stride_of_spins() {
2812 let s = STACK_HEADROOM_CHECK_SPIN_STRIDE;
2813 assert!(!stack_headroom_check_due(None, Some(0), s - 1));
2814 assert!(stack_headroom_check_due(None, Some(0), s));
2815 }
2816}
2817
2818/// phase-474 I1 -- the wire vocabulary and the SWD record.
2819#[cfg(test)]
2820mod ring_record_tests {
2821 use super::*;
2822
2823 fn v(rule: &'static str, fqn: &'static str, measured: u32) -> Violation {
2824 Violation {
2825 rule,
2826 fqn,
2827 measured,
2828 declared: 10,
2829 }
2830 }
2831
2832 #[test]
2833 fn every_rule_the_executor_reports_has_a_stable_code() {
2834 // The order is the wire format: a capture or a RAM dump decodes by it.
2835 assert_eq!(rule_code("rate-hierarchy-runtime"), 1);
2836 assert_eq!(rule_code("max-latency-runtime"), 3);
2837 assert_eq!(rule_code("release-jitter-runtime"), 9);
2838 assert_eq!(rule_code("no-such-rule"), 0);
2839 for (i, r) in RULE_IDS.iter().enumerate() {
2840 assert_eq!(rule_name(rule_code(r)), Some(*r), "code {}", i + 1);
2841 }
2842 assert_eq!(rule_name(0), None);
2843 assert_eq!(rule_name(RULE_IDS.len() as u32 + 1), None);
2844 }
2845
2846 #[test]
2847 fn fqn_hash_is_fnv1a_32() {
2848 // Reference values of 32-bit FNV-1a, so a Python decoder can match.
2849 assert_eq!(fqn_hash(""), 0x811c_9dc5);
2850 assert_eq!(fqn_hash("a"), 0xe40c_292c);
2851 assert_eq!(fqn_hash("foobar"), 0xbf9c_f968);
2852 }
2853
2854 #[test]
2855 fn a_violation_is_four_marker_events() {
2856 let w = violation_marker_words(5, &v("max-latency-runtime", "/n/out", 25));
2857 assert_eq!(w[0], (MARKER_VIOLATION, (5 << 8) | 3));
2858 assert_eq!(w[1], (MARKER_VIOLATION_FQN, fqn_hash("/n/out")));
2859 assert_eq!(w[2], (MARKER_VIOLATION_MEASURED, 25));
2860 assert_eq!(w[3], (MARKER_VIOLATION_DECLARED, 10));
2861 // The sequence number keeps its low 24 bits; the rule its low 8.
2862 let w = violation_marker_words(0x0100_0007, &v("silence-runtime", "/x", 0));
2863 assert_eq!(w[0].1, (7 << 8) | 7);
2864 }
2865
2866 #[test]
2867 fn the_record_keeps_the_latest_and_counts_what_it_overwrote() {
2868 let r: ViolationRecord<3> = ViolationRecord::new();
2869 assert_eq!(r.magic.load(Ordering::Relaxed), RECORD_MAGIC);
2870 assert_eq!(r.capacity.load(Ordering::Relaxed), 3);
2871 for m in 1..=5 {
2872 assert_eq!(r.store(&v("timer-overrun-runtime", "timer", m)), m);
2873 }
2874 assert_eq!(r.total.load(Ordering::Relaxed), 5);
2875 assert_eq!(r.dropped.load(Ordering::Relaxed), 2);
2876 assert_eq!(r.head.load(Ordering::Relaxed), 5 % 3);
2877 let mut seen = std::vec::Vec::new();
2878 r.newest_first(|seq, rule, fqn, measured, declared| {
2879 assert_eq!(rule, rule_code("timer-overrun-runtime"));
2880 assert_eq!(fqn, fqn_hash("timer"));
2881 assert_eq!(declared, 10);
2882 seen.push((seq, measured));
2883 });
2884 assert_eq!(seen, [(5, 5), (4, 4), (3, 3)], "newest first, oldest gone");
2885 }
2886
2887 /// The layout a debugger decodes: `u32` words only, header then slots, the
2888 /// same size on every target. `scripts/read-violation-record.py` reads it
2889 /// by these numbers.
2890 #[test]
2891 fn the_record_is_packed_u32_words() {
2892 use core::mem::{offset_of, size_of};
2893 assert_eq!(size_of::<ViolationSlot>(), RECORD_SLOT_WORDS as usize * 4);
2894 assert_eq!(
2895 offset_of!(ViolationRecord<4>, slots),
2896 RECORD_HEADER_WORDS as usize * 4
2897 );
2898 assert_eq!(
2899 size_of::<ViolationRecord<4>>(),
2900 (RECORD_HEADER_WORDS + 4 * RECORD_SLOT_WORDS) as usize * 4
2901 );
2902 assert_eq!(offset_of!(ViolationRecord<4>, total), 16);
2903 assert_eq!(offset_of!(ViolationRecord<4>, head), 20);
2904 assert_eq!(offset_of!(ViolationRecord<4>, dropped), 24);
2905 assert_eq!(offset_of!(ViolationRecord<4>, suppressed_before_arm), 28);
2906 assert_eq!(offset_of!(ViolationRecord<4>, armed), 32);
2907 assert_eq!(offset_of!(ViolationSlot, fqn_addr), 20);
2908 }
2909}
2910
2911/// phase10-B3 -- the route rule's pure half: binding, the earliest unanswered
2912/// take, judging only on a publish of the output, report once until clean.
2913#[cfg(test)]
2914mod route_tests {
2915 use super::*;
2916
2917 static CELL: PubMonitorCell = PubMonitorCell::new();
2918 static OTHER: PubMonitorCell = PubMonitorCell::new();
2919 static TABLE: [MonitorSpec; 2] = [
2920 MonitorSpec {
2921 topic: "/out",
2922 fqn: "/n/out",
2923 min_rate_hz_milli: 0,
2924 max_latency_ms: 0,
2925 route_trigger: "/in",
2926 route_max_latency_ms: 149,
2927 route_deadline_ms: 0,
2928 reaction_id: 0,
2929 mechanism: 0,
2930 cell: &CELL,
2931 },
2932 MonitorSpec {
2933 topic: "/other",
2934 fqn: "/n/other",
2935 min_rate_hz_milli: 0,
2936 max_latency_ms: 0,
2937 route_trigger: "",
2938 route_max_latency_ms: 0,
2939 route_deadline_ms: 0,
2940 reaction_id: 0,
2941 mechanism: 0,
2942 cell: &OTHER,
2943 },
2944 ];
2945
2946 fn snapshot() -> [u32; MAX_MONITORS] {
2947 let mut c = [0u32; MAX_MONITORS];
2948 for (k, s) in TABLE.iter().enumerate() {
2949 c[k] = s.cell.count.load(Ordering::Relaxed);
2950 }
2951 c
2952 }
2953
2954 fn judge(end_us: u64, before: &[u32; MAX_MONITORS]) -> Vec<(u32, u32)> {
2955 let mut out = Vec::new();
2956 route_judge(&TABLE, before, end_us, |_, v, record, react| {
2957 assert!(record && !react, "an unsupervised route only records");
2958 assert_eq!(v.rule, "route-latency-runtime");
2959 assert_eq!(v.fqn, "/n/out");
2960 out.push((v.measured, v.declared));
2961 });
2962 out
2963 }
2964
2965 extern crate alloc;
2966 use alloc::vec::Vec;
2967
2968 #[test]
2969 fn route_rule() {
2970 assert!(route_active(&TABLE));
2971 route_bind(&TABLE, "/elsewhere", 1);
2972 assert_eq!(
2973 CELL.route.load(Ordering::Relaxed),
2974 0,
2975 "wrong topic binds nothing"
2976 );
2977 route_bind(&TABLE, "/in", 3);
2978 route_bind(&TABLE, "/in", 5);
2979 assert_eq!(
2980 CELL.route.load(Ordering::Relaxed),
2981 4,
2982 "first subscription keeps it"
2983 );
2984 assert_eq!(OTHER.route.load(Ordering::Relaxed), 0);
2985
2986 // A take on another slot starts nothing; two takes keep the earliest.
2987 route_take(&TABLE, 5, 1_000);
2988 assert_eq!(CELL.route.load(Ordering::Relaxed) & ROUTE_PENDING, 0);
2989 route_take(&TABLE, 3, 10_000);
2990 route_take(&TABLE, 3, 90_000);
2991 // A dispatch that publishes nothing judges nothing.
2992 let b = snapshot();
2993 assert!(judge(400_000, &b).is_empty());
2994 assert_ne!(CELL.route.load(Ordering::Relaxed) & ROUTE_PENDING, 0);
2995 // The output published: 10 ms -> 160 ms is 150 ms, over 149.
2996 let b = snapshot();
2997 CELL.count.fetch_add(1, Ordering::Relaxed);
2998 assert_eq!(judge(160_000, &b), [(150, 149)]);
2999 assert_eq!(
3000 CELL.route.load(Ordering::Relaxed) & ROUTE_PENDING,
3001 0,
3002 "answered"
3003 );
3004 // Over again: reported once until a route is within the budget.
3005 route_take(&TABLE, 3, 200_000);
3006 let b = snapshot();
3007 CELL.count.fetch_add(1, Ordering::Relaxed);
3008 assert!(
3009 judge(400_000, &b).is_empty(),
3010 "still over: not reported twice"
3011 );
3012 route_take(&TABLE, 3, 500_000);
3013 let b = snapshot();
3014 CELL.count.fetch_add(1, Ordering::Relaxed);
3015 assert!(judge(600_000, &b).is_empty(), "100 ms: within");
3016 assert_eq!(CELL.route.load(Ordering::Relaxed) & ROUTE_VIOLATED, 0);
3017 route_take(&TABLE, 3, 700_000);
3018 let b = snapshot();
3019 CELL.count.fetch_add(1, Ordering::Relaxed);
3020 assert_eq!(
3021 judge(1_000_000, &b),
3022 [(300, 149)],
3023 "over after clean: reported"
3024 );
3025 // A publish with no waiting take is not a route.
3026 let b = snapshot();
3027 CELL.count.fetch_add(1, Ordering::Relaxed);
3028 assert!(judge(9_000_000, &b).is_empty());
3029 // Arming drops a waiting take and the report-once state, keeps the
3030 // binding; a release unbinds.
3031 route_take(&TABLE, 3, 9_100_000);
3032 route_reset(&TABLE);
3033 assert_eq!(CELL.route.load(Ordering::Relaxed), 4);
3034 route_unbind(&TABLE, 3);
3035 assert_eq!(CELL.route.load(Ordering::Relaxed), 0);
3036 // Wrapping: the stamp is the clock's low 32 bits.
3037 route_bind(&TABLE, "/in", 3);
3038 route_take(&TABLE, 3, u32::MAX as u64 - 9_999);
3039 let b = snapshot();
3040 CELL.count.fetch_add(1, Ordering::Relaxed);
3041 assert!(
3042 judge(u32::MAX as u64 + 90_001, &b).is_empty(),
3043 "100 ms across the wrap"
3044 );
3045 }
3046
3047 #[test]
3048 fn route_rule_code_is_appended() {
3049 assert_eq!(rule_code("route-latency-runtime"), 10);
3050 assert_eq!(rule_code("release-jitter-runtime"), 9);
3051 assert_eq!(rule_code("route-deadline-runtime"), 11);
3052 }
3053
3054 static DL_CELL: PubMonitorCell = PubMonitorCell::new();
3055 static DL_PLAIN: PubMonitorCell = PubMonitorCell::new();
3056 static DL_TABLE: [MonitorSpec; 2] = [
3057 // A route with the budget only: judged on completion, never overdue.
3058 MonitorSpec {
3059 topic: "/plain",
3060 fqn: "/n/plain",
3061 min_rate_hz_milli: 0,
3062 max_latency_ms: 0,
3063 route_trigger: "/in",
3064 route_max_latency_ms: 149,
3065 route_deadline_ms: 0,
3066 reaction_id: 0,
3067 mechanism: 0,
3068 cell: &DL_PLAIN,
3069 },
3070 // phase-503 -- the same route, supervised.
3071 MonitorSpec {
3072 topic: "/out",
3073 fqn: "/n/out",
3074 min_rate_hz_milli: 0,
3075 max_latency_ms: 0,
3076 route_trigger: "/in",
3077 route_max_latency_ms: 149,
3078 route_deadline_ms: 149,
3079 reaction_id: 3,
3080 mechanism: MECHANISM_TIMER,
3081 cell: &DL_CELL,
3082 },
3083 ];
3084
3085 fn overdue(now_us: u64) -> Vec<(usize, u32, u32, bool)> {
3086 let mut out = Vec::new();
3087 route_overdue(&DL_TABLE, now_us, |row, v, first| {
3088 assert_eq!(v.rule, "route-deadline-runtime");
3089 assert_eq!(v.fqn, "/n/out");
3090 out.push((row, v.measured, v.declared, first));
3091 });
3092 out
3093 }
3094
3095 /// phase-503 -- an output that never comes is judged at its deadline, not
3096 /// never: the wake source names the due time, the check at due reports it
3097 /// once with the row the hook is called for, and the next take starts a
3098 /// fresh route.
3099 #[test]
3100 fn route_deadline_rule() {
3101 assert!(route_deadline_active(&DL_TABLE));
3102 assert!(
3103 !route_deadline_active(&TABLE),
3104 "phase10-B3 rows stay unsupervised"
3105 );
3106 assert_eq!(route_next_due_us(&DL_TABLE, 0), None, "nothing pending");
3107 route_bind(&DL_TABLE, "/in", 2);
3108 route_take(&DL_TABLE, 2, 1_000_000);
3109 // The wake source: 149 ms after the take, then counting down.
3110 assert_eq!(route_next_due_us(&DL_TABLE, 1_000_000), Some(149_000));
3111 assert_eq!(route_next_due_us(&DL_TABLE, 1_100_000), Some(49_000));
3112 // One microsecond early: not overdue, still pending.
3113 assert!(overdue(1_148_999).is_empty());
3114 assert_ne!(DL_CELL.route.load(Ordering::Relaxed) & ROUTE_PENDING, 0);
3115 assert_eq!(route_next_due_us(&DL_TABLE, 1_148_999), Some(1));
3116 // At the deadline plus one 1 ms tick: reported for row 1, first.
3117 assert_eq!(overdue(1_150_000), [(1, 150, 149, true)]);
3118 let w = DL_CELL.route.load(Ordering::Relaxed);
3119 assert_eq!(w & ROUTE_PENDING, 0, "the take is dropped");
3120 assert_ne!(w & ROUTE_VIOLATED, 0);
3121 // The unsupervised row keeps its take for the completion verdict.
3122 assert_ne!(DL_PLAIN.route.load(Ordering::Relaxed) & ROUTE_PENDING, 0);
3123 assert_eq!(route_next_due_us(&DL_TABLE, 1_150_000), None);
3124 assert!(overdue(5_000_000).is_empty(), "judged once per take");
3125 // Still never answered: the next take misses too. The reaction is
3126 // requested again, the violation is not recorded twice.
3127 route_take(&DL_TABLE, 2, 6_000_000);
3128 assert_eq!(overdue(6_200_000), [(1, 200, 149, false)]);
3129 // A take answered in time clears the mark (route_judge's recovery).
3130 route_take(&DL_TABLE, 2, 7_000_000);
3131 let mut before = [0u32; MAX_MONITORS];
3132 for (k, s) in DL_TABLE.iter().enumerate() {
3133 before[k] = s.cell.count.load(Ordering::Relaxed);
3134 }
3135 DL_CELL.count.fetch_add(1, Ordering::Relaxed);
3136 let mut judged = Vec::new();
3137 route_judge(&DL_TABLE, &before, 7_010_000, |_, v, _, _| {
3138 judged.push(v.measured)
3139 });
3140 assert!(judged.is_empty(), "10 ms: within");
3141 assert_eq!(DL_CELL.route.load(Ordering::Relaxed) & ROUTE_VIOLATED, 0);
3142 // Over again after recovery: first again.
3143 route_take(&DL_TABLE, 2, 8_000_000);
3144 assert_eq!(overdue(8_149_000), [(1, 149, 149, true)]);
3145 // Across the clock's 32-bit wrap.
3146 route_take(&DL_TABLE, 2, u32::MAX as u64 - 9_999);
3147 assert_eq!(
3148 route_next_due_us(&DL_TABLE, u32::MAX as u64 + 1),
3149 Some(139_000)
3150 );
3151 assert!(overdue(u32::MAX as u64 + 139_000).is_empty());
3152 assert_eq!(overdue(u32::MAX as u64 + 139_001).len(), 1);
3153 }
3154
3155 static LC_CELL: PubMonitorCell = PubMonitorCell::new();
3156 static LC_TABLE: [MonitorSpec; 1] = [MonitorSpec {
3157 topic: "/out",
3158 fqn: "/n/late",
3159 min_rate_hz_milli: 0,
3160 max_latency_ms: 0,
3161 route_trigger: "/in",
3162 route_max_latency_ms: 149,
3163 route_deadline_ms: 149,
3164 reaction_id: 5,
3165 mechanism: MECHANISM_EXECUTOR,
3166 cell: &LC_CELL,
3167 }];
3168
3169 /// `(row, rule, measured, declared, record, react)` of one `route_judge`
3170 /// call after the output was published once.
3171 #[allow(clippy::type_complexity)]
3172 fn complete(end_us: u64) -> Vec<(usize, &'static str, u32, u32, bool, bool)> {
3173 let mut before = [0u32; MAX_MONITORS];
3174 before[0] = LC_CELL.count.load(Ordering::Relaxed);
3175 LC_CELL.count.fetch_add(1, Ordering::Relaxed);
3176 let mut out = Vec::new();
3177 route_judge(&LC_TABLE, &before, end_us, |row, v, record, react| {
3178 assert_eq!(v.fqn, "/n/late");
3179 out.push((row, v.rule, v.measured, v.declared, record, react));
3180 });
3181 out
3182 }
3183
3184 fn lc_overdue(now_us: u64) -> usize {
3185 let mut n = 0;
3186 route_overdue(&LC_TABLE, now_us, |_, _, _| n += 1);
3187 n
3188 }
3189
3190 /// phase-503 -- a supervised route whose one long callback publishes the
3191 /// output after the deadline, before the executor's overdue check could
3192 /// run: the completion reports the deadline miss and requests the
3193 /// reaction, once per take; a deadline already caught by the check is not
3194 /// reported again at completion; an on-time completion reports nothing.
3195 #[test]
3196 fn a_late_completion_is_a_deadline_miss_once() {
3197 route_bind(&LC_TABLE, "/in", 0);
3198 // Late completion, no overdue check in between (the callback held the
3199 // spin thread): the latency verdict, plus one deadline verdict that
3200 // asks for the reaction.
3201 route_take(&LC_TABLE, 0, 1_000_000);
3202 assert_eq!(
3203 complete(1_327_000),
3204 [
3205 (0, "route-latency-runtime", 327, 149, true, false),
3206 (0, "route-deadline-runtime", 327, 149, true, true),
3207 ]
3208 );
3209 assert_eq!(LC_CELL.route.load(Ordering::Relaxed) & ROUTE_PENDING, 0);
3210 assert_ne!(LC_CELL.route.load(Ordering::Relaxed) & ROUTE_VIOLATED, 0);
3211 // The take was answered: the overdue check has nothing left.
3212 assert_eq!(lc_overdue(1_400_000), 0, "no second verdict");
3213 // Late again while still marked: the reaction again, no record.
3214 route_take(&LC_TABLE, 0, 2_000_000);
3215 assert_eq!(
3216 complete(2_200_000),
3217 [(0, "route-deadline-runtime", 200, 149, false, true)]
3218 );
3219 // On time: nothing, and the mark clears.
3220 route_take(&LC_TABLE, 0, 3_000_000);
3221 assert!(complete(3_100_000).is_empty());
3222 assert_eq!(LC_CELL.route.load(Ordering::Relaxed) & ROUTE_VIOLATED, 0);
3223 // The deadline already fired (the check ran in time, or the timer
3224 // mechanism's): the late output that follows judges nothing.
3225 route_take(&LC_TABLE, 0, 4_000_000);
3226 assert_eq!(lc_overdue(4_150_000), 1);
3227 assert!(
3228 complete(4_327_000).is_empty(),
3229 "no second call at completion"
3230 );
3231 // Exactly at the deadline counts as overdue, as in route_overdue;
3232 // within the budget and before it does not.
3233 route_take(&LC_TABLE, 0, 5_000_000);
3234 let at = complete(5_149_000);
3235 assert_eq!(at.len(), 1);
3236 assert_eq!(at[0].1, "route-deadline-runtime");
3237 route_take(&LC_TABLE, 0, 6_000_000);
3238 assert!(complete(6_148_999).is_empty());
3239 }
3240}