rhdl_dsig_scalable_core/verify/

unit.rs

//! One verify unit: its HBM record reader, its verify core, its verdict writer.
//!
//! ```text
//!   HBM records{u} ──16-beat bursts──▶ RecordReader ──beats──▶ VerifyCore ──verdicts──▶ VerdictWriter ──▶ HBM verdicts{u}
//!                                       (64-beat FIFO)            │  ▲                 (64 per beat)
//!                                                       cert request  answer
//! ```
//!
//! * The [`RecordReader`] streams the unit's `count × 26` beats from its
//!   record buffer in 1 KiB bursts. It asks for a burst only when its FIFO
//!   has room for all of it, so it can always take read data (`rready` high),
//!   and it hands out up to one beat per clock.
//! * The [`VerdictWriter`] packs verdict bytes 64 to a beat and writes each
//!   beat once it is full (or holds the last verdict), one beat per burst.
//!   A run is done only when every beat's write response came back.

use crate::axi4::{RESP_OKAY, ReadRequest, ReadResponse, WriteRequest, WriteResponse};
use crate::verify::RECORD_BEATS;
use crate::verify::certs::{CertRequest, CertResult};
use crate::verify::datapath::{Beat, CertWarm, CoreInput, Verdict, VerifyCore};
use rhdl::prelude::*;
use rhdl_fpga::core::{
    dff::DFF,
    ram::synchronous::{In as RamIn, SyncBRAM, Write as RamWrite},
};

/// Beats per read burst (1 KiB; a 4 KiB-aligned base keeps bursts in a page).
pub const BURST_BEATS: usize = 16;
/// Beat FIFO capacity.
pub const FIFO_BEATS: usize = 64;

const _: () = assert!(
    RECORD_BEATS == 26 || RECORD_BEATS == 27,
    "the beat count below is count·16 + count·8 + count·2 (+ count for 27-beat records)"
);
/// Profile 2's records have a 27th beat (see [`crate::verify::RECORD_BYTES`]).
const EXTRA_BEAT: bool = RECORD_BEATS == 27;

// --------------------------------------------------------------- reader

/// Reader inputs.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct ReaderInput {
    /// Byte address of record 0 (4 KiB aligned).
    pub base: b64,
    /// Beats to read: records × [`RECORD_BEATS`].
    pub beats: b32,
    /// Issue reads (high while the kernel runs).
    pub run: bool,
    /// The memory's read channel.
    pub memory: ReadResponse,
    /// The consumer takes the presented beat this cycle.
    pub take: bool,
}

/// Reader outputs.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct ReaderOutput {
    /// Read requests to memory.
    pub memory: ReadRequest,
    /// The next beat, in order.
    pub beat: Beat,
    /// A read response was not OKAY (sticky).
    pub error: bool,
}

/// Reader counters and flags.
#[derive(Clone, Copy, Debug, PartialEq, Digital, Default)]
#[doc(hidden)]
pub struct ReaderControl {
    /// `beats` and `run`, registered: they come from the cluster through a
    /// delay line and feed wide compares.
    beats: b32,
    run: bool,
    /// The read request on the memory port, held in registers so that it
    /// leaves the unit without logic (it crosses to another die).
    ar_valid: bool,
    ar_addr: b64,
    ar_len: b8,
    /// Beats requested (including the one on the port).
    requested: b32,
    /// Beats received into the FIFO.
    received: b32,
    /// Beats read out of the FIFO.
    drained: b32,
    /// A FIFO read is in flight (its data arrives next cycle).
    reading: bool,
    /// Beats in the output buffer (0..=2).
    count: b2,
    /// Oldest output-buffer entry.
    head: bool,
    /// Sticky response error.
    error: bool,
}

/// The record reader.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct RecordReader {
    /// 64 × 64-byte beat FIFO.
    fifo: SyncBRAM<[b8; 64], 6>,
    /// Counters.
    control: DFF<ReaderControl>,
    /// Two-entry output buffer.
    out: DFF<[[b8; 64]; 2]>,
}

impl Default for RecordReader {
    /// An idle reader.
    fn default() -> Self {
        Self {
            fifo: SyncBRAM::default(),
            control: DFF::new(ReaderControl::default()),
            out: DFF::resetless(),
        }
    }
}

impl SynchronousIO for RecordReader {
    type I = ReaderInput;
    type O = ReaderOutput;
    type Kernel = record_reader_kernel;
}

/// The reader's logic.
#[kernel]
pub fn record_reader_kernel(clock_reset: ClockReset, input: ReaderInput, q: RecordReaderQ) -> (ReaderOutput, RecordReaderD) {
    // No reset gating here: every register this logic feeds resets itself,
    // and keeping the reset out of the logic cones keeps it off the routed
    // kernel's failing paths (verifier build v1).
    let resetting = clock_reset.reset.any();
    let _ = resetting;
    let mut c = q.control;
    let mut out = q.out;
    c.beats = input.beats;
    c.run = input.run;
    // AR: the request register empties when the port takes it; it loads
    // the next burst when it is empty and a whole burst fits in the FIFO,
    // so a 16-beat burst can go out every other clock.
    if q.control.ar_valid && input.memory.arready {
        c.ar_valid = false;
    }
    let in_fifo = q.control.requested - q.control.drained;
    let room = in_fifo <= b32((FIFO_BEATS - BURST_BEATS) as u128);
    let remaining = q.control.beats - q.control.requested;
    let burst: b32 = if remaining < b32(BURST_BEATS as u128) { remaining } else { b32(BURST_BEATS as u128) };
    let load = !q.control.ar_valid && q.control.run && room && q.control.requested < q.control.beats;
    if load {
        c.ar_valid = true;
        c.ar_addr = input.base + (q.control.requested.resize::<64>() << 6);
        c.ar_len = (burst - b32(1)).resize();
        c.requested = q.control.requested + burst;
    }
    // R → FIFO.
    let write = input.memory.rvalid;
    if write {
        c.received = q.control.received + b32(1);
        if input.memory.rresp != RESP_OKAY {
            c.error = true;
        }
    }
    // Output buffer: the beat read last cycle arrives.
    let mut count = q.control.count;
    if q.control.reading {
        let slot = q.control.head ^ (count == b2(1));
        if slot {
            out[1] = q.fifo;
        } else {
            out[0] = q.fifo;
        }
        count = count + b2(1);
    }
    // The consumer takes the head.
    let have = q.control.count != b2(0);
    if input.take && have {
        c.head = !q.control.head;
        count = count - b2(1);
    }
    c.count = count;
    // Read the next FIFO beat while the buffer will have room for it (the
    // beat taken this cycle frees its entry, so one beat per clock flows).
    let pending = q.control.count.resize::<3>() + if q.control.reading { b3(1) } else { b3(0) }
        - if input.take && have { b3(1) } else { b3(0) };
    let read = q.control.received != q.control.drained && pending < b3(2);
    c.reading = read;
    if read {
        c.drained = q.control.drained + b32(1);
    }
    let head = if q.control.head { q.out[1] } else { q.out[0] };
    let d = RecordReaderD {
        fifo: RamIn::<[b8; 64], 6> {
            read_addr: q.control.drained.resize(),
            write: RamWrite::<[b8; 64], 6> {
                addr: q.control.received.resize(),
                value: input.memory.rdata,
                enable: write,
            },
        },
        control: c,
        out,
    };
    (
        ReaderOutput {
            memory: ReadRequest {
                arvalid: q.control.ar_valid,
                araddr: q.control.ar_addr,
                arlen: q.control.ar_len,
                rready: true,
            },
            beat: Beat {
                valid: have,
                data: head,
            },
            error: q.control.error,
        },
        d,
    )
}

// --------------------------------------------------------------- writer

/// Writer inputs.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct WriterInput {
    /// Byte address of verdict 0 (64-byte aligned).
    pub base: b64,
    /// Verdicts expected.
    pub count: b32,
    /// The next verdict.
    pub verdict: Verdict,
    /// The memory's write channel.
    pub memory: WriteResponse,
}

/// Writer outputs.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct WriterOutput {
    /// Write requests to memory.
    pub memory: WriteRequest,
    /// The writer takes a verdict this cycle.
    pub ready: bool,
    /// Every verdict is written and acknowledged.
    pub done: bool,
    /// A write response was not OKAY (sticky).
    pub error: bool,
}

/// Writer state.
#[derive(Clone, Copy, Debug, PartialEq, Digital, Default)]
#[doc(hidden)]
pub struct WriterControl {
    /// Bytes in the accumulator (0..64).
    fill: b7,
    /// Verdicts taken.
    taken: b32,
    /// A full beat waits to be written.
    sending: bool,
    /// Its address was accepted.
    aw_done: bool,
    /// Its data was accepted.
    w_done: bool,
    /// Beats sent (the next beat's number).
    sent: b32,
    /// Beats acknowledged.
    acked: b32,
    /// Sticky response error.
    error: bool,
}

/// Writer data.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct WriterData {
    /// Verdicts being collected.
    acc: [b8; 64],
    /// The beat being written.
    beat: [b8; 64],
}

/// The verdict writer.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct VerdictWriter {
    /// Counters and handshake flags.
    control: DFF<WriterControl>,
    /// Accumulator and send buffer (reset to zero so padding bytes are zero).
    data: DFF<WriterData>,
}

impl Default for VerdictWriter {
    /// An idle writer with zeroed buffers.
    fn default() -> Self {
        Self {
            control: DFF::new(WriterControl::default()),
            data: DFF::new(WriterData {
                acc: [b8(0); 64],
                beat: [b8(0); 64],
            }),
        }
    }
}

impl SynchronousIO for VerdictWriter {
    type I = WriterInput;
    type O = WriterOutput;
    type Kernel = verdict_writer_kernel;
}

/// The writer's logic.
#[allow(clippy::needless_range_loop)]
#[kernel]
pub fn verdict_writer_kernel(clock_reset: ClockReset, input: WriterInput, q: VerdictWriterQ) -> (WriterOutput, VerdictWriterD) {
    // No reset gating here: every register this logic feeds resets itself,
    // and keeping the reset out of the logic cones keeps it off the routed
    // kernel's failing paths (verifier build v1).
    let resetting = clock_reset.reset.any();
    let _ = resetting;
    let mut c = q.control;
    let mut data = q.data;
    // Send: AW and W independently, then the beat is out.
    let awvalid = q.control.sending && !q.control.aw_done;
    let wvalid = q.control.sending && !q.control.w_done;
    let aw_now = q.control.aw_done || (awvalid && input.memory.awready);
    let w_now = q.control.w_done || (wvalid && input.memory.wready);
    if q.control.sending {
        if aw_now && w_now {
            c.sending = false;
            c.aw_done = false;
            c.w_done = false;
            c.sent = q.control.sent + b32(1);
        } else {
            c.aw_done = aw_now;
            c.w_done = w_now;
        }
    }
    // B: count acknowledgements.
    if input.memory.bvalid {
        c.acked = q.control.acked + b32(1);
        if input.memory.bresp != RESP_OKAY {
            c.error = true;
        }
    }
    // Take a verdict while no beat waits to be sent.
    let ready = !q.control.sending;
    if ready && input.verdict.valid {
        let mut acc = q.data.acc;
        for i in 0..64 {
            if q.control.fill == b7(i as u128) {
                acc[i] = input.verdict.byte;
            }
        }
        let taken = q.control.taken + b32(1);
        c.taken = taken;
        if q.control.fill == b7(63) || taken == input.count {
            data.beat = acc;
            data.acc = [b8(0); 64];
            c.fill = b7(0);
            c.sending = true;
        } else {
            data.acc = acc;
            c.fill = q.control.fill + b7(1);
        }
    }
    let done = q.control.taken == input.count && !q.control.sending && q.control.acked == q.control.sent;
    let d = VerdictWriterD { control: c, data };
    (
        WriterOutput {
            memory: WriteRequest {
                awvalid,
                awaddr: input.base + (q.control.sent.resize::<64>() << 6),
                awlen: b8(0),
                wvalid,
                wdata: q.data.beat,
                wlast: true,
                bready: true,
            },
            ready,
            done,
            error: q.control.error,
        },
        d,
    )
}

// ----------------------------------------------------------------- unit

/// Unit inputs.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct UnitInput {
    /// The kernel runs.
    pub run: bool,
    /// Address of the unit's records.
    pub records_base: b64,
    /// Address of the unit's verdicts.
    pub verdicts_base: b64,
    /// Records for this unit.
    pub count: b32,
    /// Record read channel.
    pub records: ReadResponse,
    /// Verdict write channel.
    pub verdicts: WriteResponse,
    /// The arbiter took this unit's certificate request.
    pub cert_taken: bool,
    /// An Ed25519 answer for this unit.
    pub cert_result: CertResult,
}

/// Unit outputs.
#[derive(Clone, Copy, Debug, PartialEq, Digital)]
pub struct UnitOutput {
    /// Record read channel.
    pub records: ReadRequest,
    /// Verdict write channel.
    pub verdicts: WriteRequest,
    /// A certificate check for the Ed25519 core.
    pub cert_request: CertRequest,
    /// Records retired.
    pub retired: b32,
    /// Valid records retired.
    pub valid: b32,
    /// Certificate checks issued.
    pub cert_checks: b32,
    /// Certificate checks that failed.
    pub cert_fails: b32,
    /// All verdicts written and acknowledged.
    pub done: bool,
    /// A read or write response failed: bit 0 read, bit 1 write.
    pub error: b2,
}

/// One verify unit.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct VerifyUnit<const TABLE_MASK: usize> {
    /// Record stream from HBM.
    reader: RecordReader,
    /// The verify core.
    core: VerifyCore<TABLE_MASK>,
    /// Verdicts to HBM.
    writer: VerdictWriter,
}

impl<const TABLE_MASK: usize> Default for VerifyUnit<TABLE_MASK> {
    /// An idle unit.
    fn default() -> Self {
        Self {
            reader: RecordReader::default(),
            core: VerifyCore::default(),
            writer: VerdictWriter::default(),
        }
    }
}

impl<const TABLE_MASK: usize> SynchronousIO for VerifyUnit<TABLE_MASK> {
    type I = UnitInput;
    type O = UnitOutput;
    type Kernel = verify_unit_kernel<TABLE_MASK>;
}

/// The unit's wiring.
#[kernel]
pub fn verify_unit_kernel<const TABLE_MASK: usize>(
    _clock_reset: ClockReset,
    input: UnitInput,
    q: VerifyUnitQ<TABLE_MASK>,
) -> (UnitOutput, VerifyUnitD<TABLE_MASK>) {
    // count × 26 = count·16 + count·8 + count·2 (count × 27: plus count).
    let beats = (input.count << 4) + (input.count << 3) + (input.count << 1) + (if EXTRA_BEAT { input.count } else { b32(0) });
    let take = q.reader.beat.valid && q.core.beat_ready;
    let d = VerifyUnitD::<TABLE_MASK> {
        reader: ReaderInput {
            base: input.records_base,
            beats,
            run: input.run,
            memory: input.records,
            take,
        },
        core: CoreInput {
            beat: q.reader.beat,
            slot: b4(0),
            check_epoch: false,
            expected_epoch: b64(0),
            warm: CertWarm {
                valid: false,
                slot: b4(0),
                commitment: [b8(0); 64],
                signature: [b8(0); 64],
            },
            verdict_ready: q.writer.ready,
            cert_taken: input.cert_taken,
            cert_result: input.cert_result,
        },
        writer: WriterInput {
            base: input.verdicts_base,
            count: input.count,
            verdict: q.core.verdict,
            memory: input.verdicts,
        },
    };
    (
        UnitOutput {
            records: q.reader.memory,
            verdicts: q.writer.memory,
            cert_request: q.core.cert_request,
            retired: q.core.retired,
            valid: q.core.valid,
            cert_checks: q.core.cert_checks,
            cert_fails: q.core.cert_fails,
            done: q.writer.done,
            error: (if q.reader.error { b2(1) } else { b2(0) }) | (if q.writer.error { b2(2) } else { b2(0) }),
        },
        d,
    )
}