rhdl_dsig_scalable_core/

relay.rs

//! Relay stations and delay lines: registers that let slice traffic cross dies.
//!
//! # The problem: handshakes that span three SLRs
//!
//! U280 is three dies (SLRs) stacked on an interposer. The HBM ports, the
//! AXI-Lite control port and therefore the engine's shared blocks (control
//! registers, [message reader](crate::msg_reader) and
//! [HBM writers](crate::hbm_writer)) belong at the bottom of SLR0, the
//! [certificate unit](crate::cert_unit) sits in SLR1, and build v9
//! (2026-10-17) pinned each [slice](crate::slice) wholly into one SLR
//! (`fpga/u280/engine/floorplan.tcl`) so that no slice straddles a die.
//!
//! That worked: every slice landed inside its SLR. But every signal between
//! a slice and a shared block was built for neighbours, with logic on both
//! ends of the wire. The slices in SLR2 were two die crossings away, and v9
//! routed at WNS -4.715 ns (TNS -81,716). Of its 5,000 worst paths:
//!
//! | paths | link |
//! |---|---|
//! | 2,809 | writer → HBM: the writer's `wvalid` is combinational from the slice's beat, and `beat_ready` from HBM's `wready` goes straight back into the signer |
//! | 875 | control registers → the slices' BLAKE3 lanes (run configuration) |
//! | ~180 | certificate unit ↔ slices (certificates in, roots out) |
//! | ~30 | slice status counters → the register-read mux |
//!
//! The worst one ran from a writer counter in SLR1 through the writer's logic,
//! into slice 3's signer in SLR2, and back down to the HBM interconnect in
//! SLR0: 9.6 ns for a 4.878 ns clock, 93 % of it wire.
//!
//! # The decision: registers on both ends of every long wire
//!
//! A long wire is cheap if a flip-flop drives it and at most one LUT sits in
//! front of the flip-flop it reaches. So every slice-boundary signal goes
//! through registers in the engine ([`crate::engine`]), by kind:
//!
//! * **Full-rate handshakes** (frame beats out) go through a [`RelayPipe`]:
//!   [`PIPE_STAGES`] Carloni relay stations ([`Relay`]). A handshake cannot
//!   simply be delayed, because the receiver's "taken" must reach the sender
//!   in the same clock.
//! * **Slower handshakes** (messages and certificates in) go through a
//!   [`ToggleLink`]: plain registers plus a one-bit toggle protocol. It moves
//!   one item per 6 clocks but costs almost no LUTs (relays cost 21,031 LUTs
//!   on these two paths in build v12). A signer takes at most one message per
//!   message-hash round trip and one certificate per 128 signatures.
//! * **Everything else** (run configuration and `run`, subtree-root pulses,
//!   the `roots_popped` credit, status counters and the fault bit) goes
//!   through a [`Delay`]: [`DELAY_STAGES`] plain registers. These signals
//!   tolerate latency: configuration is static during a run, a root is a
//!   one-clock pulse that arrives two clocks later, a late credit only makes
//!   the slice more cautious, and the host reads status long after it
//!   changes.
//!
//! # Relay stations
//!
//! A *relay station* (L. P. Carloni, "From Latency-Insensitive Design to
//! Communication-Based System-Level Design", Proc. IEEE 2015, figure 4) is a
//! skid buffer: two data registers and a two-state controller inserted into a
//! valid/ready link. Its outputs in **both** directions come from flip-flops:
//!
//! * `void_out` and the data come from the `main` register;
//! * `stop_out` ("not ready") is just "the controller is in Stall".
//!
//! It still moves one item per clock: when the far side stops, the item
//! already in flight is caught in `aux` instead of being lost. Each station
//! adds one clock of latency.
//!
//! ```text
//!   SLR2: signer ─▶ [relay 0] ─┐
//!   SLR1:                      └▶ [relay 1] ─┐
//!   SLR0:                                    └▶ [relay 2] ─▶ writer ─▶ HBM
//!            ◀── stop (registered) ── each station ── stop ◀── !beat_ready
//! ```
//!
//! RHDL ships the same station as `rhdl_fpga::lid::carloni::Carloni`. This
//! one differs in one way: its two data registers are resetless
//! ([`DFF::resetless`]), so the wide payloads (a certificate is 1,345 bits)
//! add no loads to the engine's reset tree (build v8's worst paths started at
//! the reset). Only `void` and the state reset.
//!
//! # Protocol mapping
//!
//! | handshake | relay |
//! |---|---|
//! | `valid` | `!void` |
//! | the receiver took the item this clock (`beat_ready`, `message_taken`, `certificate_taken`) | `!stop` |
//!
//! The receivers' answers are all "taken this clock", so they are low when
//! nothing is offered. A station with an empty output ignores `stop`, so
//! that is harmless. The sender sees "taken" as `valid && !stop_out` of the
//! first station.

use rhdl::prelude::*;
use rhdl_fpga::core::dff::DFF;

/// Relay stations in each [`RelayPipe`].
pub const PIPE_STAGES: usize = 3;

/// Index of a pipe's last station (RHDL kernel loops need a named bound,
/// not an expression).
const PIPE_LAST: usize = PIPE_STAGES - 1;

/// Registers in each [`Delay`].
pub const DELAY_STAGES: usize = 2;

/// Index of a delay line's last register.
const DELAY_LAST: usize = DELAY_STAGES - 1;

/// Inputs of one relay station (Carloni's names).
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct RelayIn<T: Digital> {
    /// The item from upstream (ignored while `void_in`).
    pub data_in: T,
    /// Upstream offers nothing this clock.
    pub void_in: bool,
    /// Downstream cannot take an item this clock.
    pub stop_in: bool,
}

/// Outputs of one relay station; every field comes from a register.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct RelayOut<T: Digital> {
    /// The item held for downstream.
    pub data_out: T,
    /// No item is held for downstream.
    pub void_out: bool,
    /// Upstream must hold its item (the station is full).
    pub stop_out: bool,
}

/// One Carloni relay station carrying a `T`.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct Relay<T: Digital> {
    /// The item presented downstream (resetless).
    main: DFF<T>,
    /// The item caught while downstream was stopped (resetless).
    aux: DFF<T>,
    /// `main` holds no item; resets to empty.
    void: DFF<bool>,
    /// Stall: `aux` holds an item and upstream is stopped; resets to Run.
    stall: DFF<bool>,
}

impl<T: Digital> Default for Relay<T> {
    /// Empty and running after reset; data registers don't-care.
    fn default() -> Self {
        Self {
            main: DFF::resetless(),
            aux: DFF::resetless(),
            void: DFF::new(true),
            stall: DFF::new(false),
        }
    }
}

impl<T: Digital> SynchronousIO for Relay<T> {
    type I = RelayIn<T>;
    type O = RelayOut<T>;
    type Kernel = relay_kernel<T>;
}

/// Carloni's controller (figure 4), with the state as one bit.
///
/// * **Run** (`stall` low): `main` loads the incoming item whenever
///   downstream is not stopped, or `main` is empty. If downstream stops while
///   both `main` and the input hold items, the input item goes into `aux`
///   and the station moves to Stall.
/// * **Stall**: `stop_out` is high. When downstream releases, `main` takes
///   the item from `aux` and the station returns to Run (still stopping
///   upstream for that one clock, so nothing new arrives while `aux` drains).
#[kernel]
pub fn relay_kernel<T: Digital>(cr: ClockReset, i: RelayIn<T>, q: RelayQ<T>) -> (RelayOut<T>, RelayD<T>) {
    let mut d = RelayD::<T>::dont_care();
    // Control signals, as named in the paper's figure.
    let mut sel = false;
    let mut main_en = false;
    let mut aux_en = false;
    let mut stop_out = false;
    let void_out = q.void;
    let will_stall = i.stop_in && !i.void_in && !void_out;
    d.stall = q.stall;
    if !q.stall {
        if !i.stop_in || (!i.void_in && void_out) {
            main_en = true;
        } else if will_stall {
            d.stall = true;
            aux_en = true;
        }
    } else {
        stop_out = true;
        if !i.stop_in {
            sel = true;
            main_en = true;
            d.stall = false;
        }
    }
    d.aux = if aux_en { i.data_in } else { q.aux };
    let data = if sel { q.aux } else { i.data_in };
    d.main = if main_en { data } else { q.main };
    let void = if sel { false } else { i.void_in };
    d.void = if main_en { void } else { q.void };
    let mut o = RelayOut::<T> {
        data_out: q.main,
        void_out: q.void,
        stop_out,
    };
    if cr.reset.any() {
        o.void_out = true;
        o.stop_out = true;
    }
    (o, d)
}

/// Inputs of a [`RelayPipe`].
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct PipeIn<T: Digital> {
    /// The sender's item.
    pub data: T,
    /// The sender offers `data` this clock.
    pub valid: bool,
    /// The receiver took the pipe's output item this clock.
    pub taken: bool,
}

/// Outputs of a [`RelayPipe`]; both come from registers.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct PipeOut<T: Digital> {
    /// The item for the receiver.
    pub data: T,
    /// An item is offered to the receiver.
    pub valid: bool,
    /// The pipe can take the sender's item this clock (the sender's item was
    /// taken if it also offered one).
    pub ready: bool,
}

/// [`PIPE_STAGES`] relay stations in a row: sender side first.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct RelayPipe<T: Digital> {
    /// The stations; `stages[0]` takes the sender's item.
    stages: [Relay<T>; PIPE_STAGES],
}

impl<T: Digital> Default for RelayPipe<T> {
    /// Every station empty.
    fn default() -> Self {
        Self {
            stages: core::array::from_fn(|_| Relay::default()),
        }
    }
}

impl<T: Digital> SynchronousIO for RelayPipe<T> {
    type I = PipeIn<T>;
    type O = PipeOut<T>;
    type Kernel = relay_pipe_kernel<T>;
}

/// Wire the stations: data and void flow toward the receiver, stop flows
/// back.
#[kernel]
pub fn relay_pipe_kernel<T: Digital>(_cr: ClockReset, i: PipeIn<T>, q: RelayPipeQ<T>) -> (PipeOut<T>, RelayPipeD<T>) {
    let mut d = RelayPipeD::<T>::dont_care();
    d.stages[0].data_in = i.data;
    d.stages[0].void_in = !i.valid;
    for k in 1..PIPE_STAGES {
        d.stages[k].data_in = q.stages[k - 1].data_out;
        d.stages[k].void_in = q.stages[k - 1].void_out;
    }
    for k in 0..PIPE_LAST {
        d.stages[k].stop_in = q.stages[k + 1].stop_out;
    }
    d.stages[PIPE_LAST].stop_in = !i.taken;
    let last = q.stages[PIPE_LAST];
    (
        PipeOut::<T> {
            data: last.data_out,
            valid: !last.void_out,
            ready: !q.stages[0].stop_out,
        },
        d,
    )
}

/// Forward data registers in a [`ToggleLink`] after its `held` register.
pub const TOGGLE_STAGES: usize = 2;

/// Index of a toggle link's last forward register.
const TOGGLE_LAST: usize = TOGGLE_STAGES - 1;

/// A valid/ready link for rare, wide items, built from plain registers.
///
/// # Why not a relay pipe
///
/// A [`Relay`] station chooses between its `main` and `aux` registers, which
/// costs a LUT per data bit per station. For the certificate path that came
/// to 4,760 LUTs per slice in build v12 (19,037 of its 410,003), to carry one
/// certificate per 128 signatures. This link carries the same handshake with
/// no per-bit logic: the data only ever moves forward, through plain
/// registers, and one bit in each direction carries the protocol.
///
/// # The protocol: a toggle (two-phase) handshake
///
/// ```text
///   sender side                                         receiver side
///   item ─▶ [held] ─▶ [fwd 0] ─▶ [fwd 1] ─────────────▶ data, valid = seq != last
///           [seq ] ─▶ [seq 0] ─▶ [seq 1] ──────────────┘
///   ready = seq == ack ◀── [ack 1] ◀── [ack 0] ◀── [last] ◀── taken
/// ```
///
/// * The sender side loads an item into `held` and flips `seq`, but only when
///   `seq == ack`: the receiver has taken everything sent so far.
/// * `held` and `seq` travel forward together through [`TOGGLE_STAGES`]
///   registers. The receiver sees a new item while the delayed `seq` differs
///   from `last`, the `seq` of the item it took most recently.
/// * Taking the item copies the delayed `seq` into `last`, which ends `valid`
///   at the next clock and travels back as `ack`.
///
/// `held` keeps the item until the receiver has taken it, so the forward
/// registers need no enable and no reset: they copy every clock. Every
/// output comes from a register or compares two registers. One item makes a
/// round trip in `2 * TOGGLE_STAGES + 2` clocks (6 with two stages), so this
/// link suits items that arrive far less often than that.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct ToggleLink<T: Digital> {
    /// The item being delivered (resetless; loaded only when `ready`).
    held: DFF<T>,
    /// Flips once per item loaded; resets low.
    seq: DFF<bool>,
    /// `held`, delayed (resetless).
    fwd: [DFF<T>; TOGGLE_STAGES],
    /// `seq`, delayed alongside `fwd`.
    fwd_seq: [DFF<bool>; TOGGLE_STAGES],
    /// The `seq` of the item the receiver took most recently; resets low.
    last: DFF<bool>,
    /// `last`, delayed back to the sender side.
    ack: [DFF<bool>; TOGGLE_STAGES],
}

impl<T: Digital> Default for ToggleLink<T> {
    /// Empty and ready after reset; data registers don't-care.
    fn default() -> Self {
        Self {
            held: DFF::resetless(),
            seq: DFF::new(false),
            fwd: core::array::from_fn(|_| DFF::resetless()),
            fwd_seq: core::array::from_fn(|_| DFF::new(false)),
            last: DFF::new(false),
            ack: core::array::from_fn(|_| DFF::new(false)),
        }
    }
}

impl<T: Digital> SynchronousIO for ToggleLink<T> {
    type I = PipeIn<T>;
    type O = PipeOut<T>;
    type Kernel = toggle_link_kernel<T>;
}

/// Load, shift and compare (see [`ToggleLink`]).
#[kernel]
pub fn toggle_link_kernel<T: Digital>(cr: ClockReset, i: PipeIn<T>, q: ToggleLinkQ<T>) -> (PipeOut<T>, ToggleLinkD<T>) {
    let mut d = ToggleLinkD::<T>::dont_care();
    let ready = q.seq == q.ack[TOGGLE_LAST];
    let load = i.valid && ready;
    d.held = if load { i.data } else { q.held };
    d.seq = if load { !q.seq } else { q.seq };
    d.fwd[0] = q.held;
    d.fwd_seq[0] = q.seq;
    for k in 1..TOGGLE_STAGES {
        d.fwd[k] = q.fwd[k - 1];
        d.fwd_seq[k] = q.fwd_seq[k - 1];
    }
    let valid = q.fwd_seq[TOGGLE_LAST] != q.last;
    d.last = if i.taken && valid { q.fwd_seq[TOGGLE_LAST] } else { q.last };
    d.ack[0] = q.last;
    for k in 1..TOGGLE_STAGES {
        d.ack[k] = q.ack[k - 1];
    }
    let mut o = PipeOut::<T> {
        data: q.fwd[TOGGLE_LAST],
        valid,
        ready,
    };
    if cr.reset.any() {
        o.valid = false;
        o.ready = false;
    }
    (o, d)
}

/// [`DELAY_STAGES`] plain registers: the output is the input from
/// `DELAY_STAGES` clocks ago. Resets to `T::dont_care()` (all zeros), so a
/// delayed `valid`, `run` or fault bit is low after reset.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct Delay<T: Digital> {
    /// The registers; `stages[0]` takes the input.
    stages: [DFF<T>; DELAY_STAGES],
}

impl<T: Digital> Default for Delay<T> {
    /// Every register reset to zero.
    fn default() -> Self {
        Self {
            stages: core::array::from_fn(|_| DFF::new(T::dont_care())),
        }
    }
}

impl<T: Digital> SynchronousIO for Delay<T> {
    type I = T;
    type O = T;
    type Kernel = delay_kernel<T>;
}

/// Shift the input one register along per clock.
#[kernel]
pub fn delay_kernel<T: Digital>(_cr: ClockReset, i: T, q: DelayQ<T>) -> (T, DelayD<T>) {
    let mut d = DelayD::<T>::dont_care();
    d.stages[0] = i;
    for k in 1..DELAY_STAGES {
        d.stages[k] = q.stages[k - 1];
    }
    (q.stages[DELAY_LAST], d)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Test payload: a number and some filler.
    type Item = (b16, [b8; 8]);

    /// The item carrying number `n`.
    fn item(n: u32) -> Item {
        (b16(u128::from(n)), core::array::from_fn(|k| b8(u128::from((n + k as u32) & 0xff))))
    }

    /// Drive `count` items through a link (a [`RelayPipe`] or a
    /// [`ToggleLink`]) with the given sender
    /// and receiver patterns (`true` = offer / take this clock, by cycle
    /// number). The receiver takes an item only when one is offered and its
    /// pattern allows, like the engine's receivers. Returns the received
    /// item numbers and the cycles it took.
    fn run<P>(pipe: P, count: u32, offer: impl Fn(u32) -> bool, accept: impl Fn(u32) -> bool) -> (Vec<u32>, u32)
    where
        P: Synchronous + SynchronousIO<I = PipeIn<Item>, O = PipeOut<Item>>,
    {
        let mut state = pipe.init();
        let input = |n: u32, valid: bool, taken: bool| PipeIn::<Item> {
            data: item(n),
            valid,
            taken,
        };
        // Two clocks of reset.
        for _ in 0..2 {
            pipe.sim(clock_reset(clock(false), reset(true)), input(0, false, false), &mut state);
            pipe.sim(clock_reset(clock(true), reset(true)), input(0, false, false), &mut state);
        }
        let mut sent = 0;
        let mut received = Vec::new();
        let mut cycle = 0;
        while received.len() < count as usize {
            assert!(cycle < 100 * count + 100, "pipe stalled");
            // The sender offers its next item when its pattern says so.
            let offering = sent < count && offer(cycle);
            // Both outputs are registered, so they do not depend on this
            // clock's inputs; read them in the low half of the clock.
            let out = pipe.sim(clock_reset(clock(false), reset(false)), input(sent, offering, false), &mut state);
            // The receiver takes the output item when offered and allowed.
            let take = out.valid && accept(cycle);
            if take {
                received.push(out.data.0.raw() as u32);
                assert_eq!(out.data, item(out.data.0.raw() as u32), "payload intact");
            }
            let was_sent = offering && out.ready;
            // Present the final inputs, then the rising edge.
            let now = input(sent, offering, take);
            pipe.sim(clock_reset(clock(false), reset(false)), now, &mut state);
            pipe.sim(clock_reset(clock(true), reset(false)), now, &mut state);
            if was_sent {
                sent += 1;
            }
            cycle += 1;
        }
        (received, cycle)
    }

    /// Always offering and always taking, the pipe moves one item per clock
    /// after its [`PIPE_STAGES`] clocks of latency.
    #[test]
    fn full_rate_with_three_clocks_of_latency() {
        let (got, cycles) = run(RelayPipe::<Item>::default(), 200, |_| true, |_| true);
        assert_eq!(got, (0..200).collect::<Vec<_>>());
        assert!(cycles <= 200 + PIPE_STAGES as u32 + 1, "{cycles} cycles");
    }

    /// Random gaps on both sides: every item arrives once, in order, intact.
    #[test]
    fn random_backpressure_keeps_order() {
        // xorshift patterns with fixed seeds; `keep` of 8 clocks are on.
        let pattern = |seed: u64, keep: u64| {
            move |c: u32| {
                let mut x = seed ^ (u64::from(c) + 1).wrapping_mul(0x9e37_79b9_7f4a_7c15);
                x ^= x << 13;
                x ^= x >> 7;
                x ^= x << 17;
                x % 8 < keep
            }
        };
        for (up, down) in [(7, 3), (3, 7), (5, 5), (8, 1), (1, 8)] {
            let (got, _) = run(RelayPipe::<Item>::default(), 300, pattern(0x1234 + up, up), pattern(0x9876 + down, down));
            assert_eq!(got, (0..300).collect::<Vec<_>>(), "offer {up}/8, take {down}/8");
            let (got, _) = run(ToggleLink::<Item>::default(), 100, pattern(0x1234 + up, up), pattern(0x9876 + down, down));
            assert_eq!(got, (0..100).collect::<Vec<_>>(), "toggle: offer {up}/8, take {down}/8");
        }
    }

    /// Always offering and always taking, a [`ToggleLink`] delivers one item
    /// per round trip of `2 * TOGGLE_STAGES + 2` clocks, never two at once.
    #[test]
    fn toggle_link_one_item_per_round_trip() {
        let round_trip = 2 * TOGGLE_STAGES as u32 + 2;
        let (got, cycles) = run(ToggleLink::<Item>::default(), 50, |_| true, |_| true);
        assert_eq!(got, (0..50).collect::<Vec<_>>());
        assert!(cycles <= 50 * round_trip + round_trip, "{cycles} cycles");
        assert!(cycles >= 49 * round_trip, "{cycles} cycles: faster than a round trip?");
    }

    /// A [`ToggleLink`]'s outputs depend only on its registers.
    #[test]
    fn toggle_link_outputs_are_registered() {
        let link = ToggleLink::<Item>::default();
        let mut state = link.init();
        let idle = PipeIn::<Item> {
            data: item(0),
            valid: false,
            taken: false,
        };
        link.sim(clock_reset(clock(false), reset(true)), idle, &mut state);
        link.sim(clock_reset(clock(true), reset(true)), idle, &mut state);
        let a = link.sim(clock_reset(clock(false), reset(false)), idle, &mut state);
        let busy = PipeIn::<Item> {
            data: item(1),
            valid: true,
            taken: true,
        };
        let b = link.sim(clock_reset(clock(false), reset(false)), busy, &mut state);
        assert_eq!(a, b);
    }

    /// The pipe's outputs in both directions come from registers: changing
    /// only this clock's inputs (no clock edge) changes nothing.
    #[test]
    fn outputs_are_registered() {
        let pipe = RelayPipe::<Item>::default();
        let mut state = pipe.init();
        let idle = PipeIn::<Item> {
            data: item(0),
            valid: false,
            taken: false,
        };
        pipe.sim(clock_reset(clock(false), reset(true)), idle, &mut state);
        pipe.sim(clock_reset(clock(true), reset(true)), idle, &mut state);
        let a = pipe.sim(clock_reset(clock(false), reset(false)), idle, &mut state);
        let busy = PipeIn::<Item> {
            data: item(1),
            valid: true,
            taken: true,
        };
        let b = pipe.sim(clock_reset(clock(false), reset(false)), busy, &mut state);
        assert_eq!(a, b);
    }

    /// A [`Delay`] outputs zero during and after reset until real inputs
    /// have shifted through, then the input from [`DELAY_STAGES`] clocks ago.
    #[test]
    fn delay_shifts_by_its_stages() {
        let delay = Delay::<b16>::default();
        let mut state = delay.init();
        for _ in 0..2 {
            delay.sim(clock_reset(clock(false), reset(true)), b16(0xffff), &mut state);
            delay.sim(clock_reset(clock(true), reset(true)), b16(0xffff), &mut state);
        }
        for t in 0..10_u128 {
            let out = delay.sim(clock_reset(clock(false), reset(false)), b16(t + 1), &mut state);
            let expected = if t < DELAY_STAGES as u128 { 0 } else { t + 1 - DELAY_STAGES as u128 };
            assert_eq!(out, b16(expected), "clock {t}");
            delay.sim(clock_reset(clock(true), reset(false)), b16(t + 1), &mut state);
        }
    }
}