rhdl_ed25519_scalable_core/

machine.rs

//! The barrel-threaded field machine: the engine that does the arithmetic.
//!
//! A [`FieldMachine`] owns one 255-bit multiplier ([`FieldMulPar`](crate::fe17::FieldMulPar), 225 DSPs, or the 150-DSP [`FieldMulKara`](crate::fe17_kara::FieldMulKara) with `ED25519_MULTIPLIER=LEAN150`,
//! a new product every clock, 23 clocks of latency), one adder
//! ([`FieldAddSub`]), one table scanner, a program ROM, and `K = 2^KB`
//! hardware **contexts**: independent jobs (signatures) resident at once.
//!
//! # Why barrel threading
//!
//! A pipelined multiplier only helps if something always has a multiply
//! ready. Instead of finding parallelism inside one job, the machine runs `K`
//! jobs that take turns: in clock `t`, context `t mod K` issues one bundle.
//! A context's next turn is `K >= 32` clocks later, after every result it
//! could need is back (the longest write-back is 25 clocks). So:
//!
//! - programs need no latency scheduling and the hardware no hazard logic;
//! - the multiplier is busy every clock the program has a multiply;
//! - each context's registers live in block RAM, addressed
//!   `register * K + context`, so contexts cost BRAM rather than LUTs.
//!
#![doc = concat!("<div style=\"overflow-x:auto\">", include_str!("../docs/images/barrel-machine.svg"), "</div>")]
//!
//! # Pipeline
//!
//! ```text
//! cycle t-1       ROM prefetch for the next head context
//! cycle t     S1  head context issues: bank, table-row and digit addresses
//! cycle t+1   S2  operands selected; scan accumulator updated
//! cycle t+2   S3  M and A units issue; OUT captured; lookup negated
//! cycle t+3   S4  bank L (one of two slots) committed
//! t+2+23          multiplier write-back       t+2+5  adder write-back
//! ```
//!
//! # Registers
//!
//! Each bank has exactly one writer, so there are no write-port conflicts:
//! **M** (multiplier results), **A** (adder results), **I** (job inputs from
//! the loader), **C** (the program's constants, one copy for all contexts),
//! and **L** (the point selected by the last table lookup, two slots per
//! context so one lookup can commit while the previous one is still being
//! read). Banks are replicated once per read port (four operands per
//! bundle). An operand in bank I with index 96 or more (`CONST_BASE`) reads C.
//!
//! Register addresses interleave contexts: register `r` of context `c` is
//! word `r * K + c` (`r << KB | c`). Multiplier and adder tags carry
//! `c << 7 | r` so a result returning 23 or 5 cycles later finds its own
//! word without any bookkeeping.
//!
//! # Where the memories live, and why
//!
//! The first layout put every bank in block RAM, and on a full chip block
//! RAM ran out before LUTs or DSPs: a fixed-base machine used 72 BRAM36
//! (banks M and A 16 each, bank I 16, base-point tables 22, program 2.5),
//! and the compress machine 289, of which 228 were its input bank. Two
//! observations fixed most of it:
//!
//! - **Banks M, A and I are written at run time and never preloaded**, which
//!   is exactly what UltraRAM can hold; the chip barely used its 960
//!   UltraRAMs. Constants used to sit in bank I as preloaded contents (one
//!   copy per context), which is what kept bank I in block RAM. So
//!   constants moved to their own 32-entry ROM C (a few LUTs, the same for
//!   every context), and the compress stage writes its identity defaults at
//!   reset instead of preloading them.
//! - **Machines without inputs** (fixed-base, verify) never write bank I, so
//!   it shrinks to the minimum and lives in LUT RAM.
//!
//! [`MachineMemory::SPECIALIZED`] (the default) is that layout;
//! [`MachineMemory::AS_BUILT`] reproduces the one mapped in Vivado on
//! 2026-10-14. The new layout is projected at about 24 BRAM36 + 32 URAM per
//! fixed-base machine; it is **not yet mapped**, and UltraRAM timing at
//! 205 MHz is the thing to check.
//!
//! # Instructions
//!
//! An [`InstrW`] bundle has three slots, all usable at once: **M** (multiply;
//! 7-bit destination, two operands of bank + 7-bit index), **A** (add,
//! subtract, or `OUT`), and **T** (one table-scan step: digit, entry pair,
//! 9-bit table row, bank-L slot). Bundles are produced by
//! [`microcode::Builder`](crate::microcode::Builder); [`encode_bundle`] packs
//! them into ROM words.
//!
//! # Constant-address table lookup
//!
//! A lookup of a signed digit runs one T step per pair of table entries. Each
//! step reads both entries at a public row; the context's accumulator keeps
//! the one whose number equals the digit's magnitude, and the last step
//! negates it if the digit is negative and commits it to bank L. No secret
//! value forms an address, an enable or a branch. (A logical constant-time
//! property, not a power or EM guarantee.) The tables hold `e · 2^offset(i) · B`
//! for each of the 54 digits, 372 rows in two BRAM images.
//!
//! # Backpressure
//!
//! An `OUT` bundle issues only when the single output register and the
//! pipeline hold no other output; otherwise the context retries on its next
//! turn. This is the machine's only stall.
//!
//! # Parameters and cost
//!
//! `FieldMachine<KB, K, RA, IA, PB, TB, KT>`: contexts `K = 2^KB`, bank
//! address widths `RA` (M/A) and `IA` (I), program-counter width `PB`,
//! base-point table rows `2^TB`, key-table rows `2^KT`.
//! [`FbMachine`](crate::sign_core::FbMachine) is `<5, 32, 9, 5, 10, 9, 1>`,
//! [`CpMachine`](crate::sign_core::CpMachine) is `<6, 64, 12, 13, 9, 1, 1>`.
//! Vivado (as-built layout, routed at 4.878 ns) put a fixed-base machine at
//! 10,110 LUT, 13,795 FF, 225 DSP and 72 BRAM36, with +0.407 ns of slack.

use rhdl::prelude::*;
use rhdl_fpga::core::{
    constant::Constant,
    dff::DFF,
    ram::synchronous::{In as RamIn, RamStyle, SyncBRAM, Write as RamWrite},
};

use super::{
    fe17::{
        Fe17, FieldAddIn, FieldAddSub, FieldMulIn, limbs_from_digits, p_minus,
        digits_from_limbs,
    },
    host::{DIGITS, Fp, Niels, TABLE_ROWS, basepoint_table, digit_entries, table_row},
    microcode::{AOp, Bank, Bundle, Operand, Program},
};

// Generated by `build.rs` from `ED25519_MULTIPLIER`: `FieldMul`, the
// multiplier every field machine of this build uses.
include!(concat!(env!("OUT_DIR"), "/multiplier.rs"));
// Generated by `build.rs` from `ED25519_URAM_MACHINES`: `URAM_MACHINES`.
include!(concat!(env!("OUT_DIR"), "/placement.rs"));

/// Bank encodings in [`OperandW::bank`]: bank M (multiplier results).
pub const BANK_M: u128 = 0;
/// Bank A (adder results).
pub const BANK_A: u128 = 1;
/// Bank I (inputs, indices below [`CONST_BASE`]) and the constant ROM C
/// (indices [`CONST_BASE`] and up). They share a code so the operand field
/// stays two bits wide.
pub const BANK_I: u128 = 2;
/// Bank L (selected lookup point).
pub const BANK_L: u128 = 3;

/// Encoded operand: which bank to read and which register in it.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct OperandW {
    /// Bank code ([`BANK_M`], [`BANK_A`], [`BANK_I`] or [`BANK_L`]).
    pub bank: b2,
    /// Register index within the context (bank I: 64 and up selects
    /// constant `index - 64`; bank L: `4 * slot + coordinate`).
    pub index: b7,
}

/// Encoded VLIW bundle stored in program ROM, one word per program counter.
///
/// The three slots are independent; a disabled slot's other fields are
/// ignored. Operands are read in S1/S2 and the units issue in S3.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct InstrW {
    /// Multiply slot enabled.
    pub m_en: bool,
    /// Multiply destination register (bank M).
    pub m_dst: b7,
    /// First factor (read port 0; never bank L).
    pub m_a: OperandW,
    /// Second factor (read port 1; the only operand programs point at bank L).
    pub m_b: OperandW,
    /// Add slot enabled.
    pub a_en: bool,
    /// 0 add, 1 subtract, 2 output (`OUT` sends `a_a` to the output port
    /// instead of using the adder).
    pub a_op: b2,
    /// Add destination register (bank A); unused by `OUT`.
    pub a_dst: b7,
    /// First operand (read port 2), also the value an `OUT` emits.
    pub a_a: OperandW,
    /// Second operand (read port 3).
    pub a_b: OperandW,
    /// Program output index reported with an `OUT` event.
    pub out_index: b6,
    /// Scan slot enabled.
    pub t_en: bool,
    /// Digit slot 0..128 (base-point scalar 0..54, key scalar 64..118).
    pub t_digit: b7,
    /// Scan the key table instead of the base-point table.
    pub t_key: bool,
    /// Entry pair 0..8: this step compares the digit magnitude with
    /// entries `2 * pair + 1` and `2 * pair + 2`.
    pub t_pair: b3,
    /// Table row of the pair (`table_row(digit) + pair`), a public constant.
    pub t_row: b9,
    /// Bank-L slot committed by `t_finish`.
    pub t_lslot: bool,
    /// Reset the accumulator to the identity first (first step of a lookup).
    pub t_clear: bool,
    /// Commit the accumulator, negated if the digit is negative, to bank L
    /// (last step of a lookup).
    pub t_finish: bool,
    /// Last bundle of its program; the context retires after issuing it.
    pub end: bool,
}

/// Affine Niels point in radix-`2^17` digits: the form table entries are
/// stored in, ready for a mixed addition.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct Niels17 {
    /// `y + x`.
    pub ypx: Fe17,
    /// `y - x`.
    pub ymx: Fe17,
    /// `2dxy`.
    pub xy2d: Fe17,
}

/// Bits of a key-slot number: 3 (8 public keys) by default, 4 (16) with the
/// feature `key-slots-16` (uBFT: a replica verifies up to 11 peers). Each slot
/// owns 512 rows of every verify machine's key table, so 16 slots double the
/// key-table memory.
#[cfg(not(feature = "key-slots-16"))]
pub const KEY_SLOT_BITS: usize = 3;
/// Bits of a key-slot number (feature `key-slots-16`: 16 slots).
#[cfg(feature = "key-slots-16")]
pub const KEY_SLOT_BITS: usize = 4;
/// Public keys a known-key verifier holds at once.
pub const KEY_SLOTS: usize = 1 << KEY_SLOT_BITS;
/// Rows of a verify machine's key table: 512 per slot.
pub const KEY_TABLE_BITS: usize = 9 + KEY_SLOT_BITS;
/// A key-slot number.
pub type KeySlot = Bits<KEY_SLOT_BITS>;

/// Key slot 0, for requests that look up no key. A kernel generic over a
/// width (`CB`) cannot write `bits(0)` here: rustc would infer that width.
#[kernel]
pub fn no_key() -> KeySlot {
    bits(0)
}

/// Start request: context `ctx` (idle, inputs loaded) begins at bundle
/// `entry`.
///
/// The request is latched as pending and takes effect on the context's next
/// turn; the caller may assert it on any cycle.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct StartReq<const KB: usize>
where
    rhdl::bits::W<KB>: BitWidth,
{
    /// Request valid this cycle.
    pub valid: bool,
    /// Context to start (`KB` bits).
    pub ctx: Bits<KB>,
    /// Opaque job tag reported with outputs.
    pub tag: b16,
    /// Key-table slot for key lookups (row `key * 512 + row`).
    pub key: KeySlot,
    /// First bundle (programs share one ROM; see `microcode::concat`).
    pub entry: b12,
}

/// Digits write for the table scanner (sign-magnitude: bit 5 = sign).
///
/// Writes all 128 digit slots of one context at once, so a context's scalar
/// recoding is one memory word read by every T step.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct DigitsWrite<const KB: usize>
where
    rhdl::bits::W<KB>: BitWidth,
{
    /// Write valid this cycle.
    pub valid: bool,
    /// Context whose digits are replaced.
    pub ctx: Bits<KB>,
    /// Sign-magnitude digits, magnitude (0..=16) in bits 4..0: slots 0..54
    /// for the base-point scalar, 64..118 for the key scalar, zero elsewhere.
    pub digits: [b6; 128],
}

/// One key-table coordinate write (the key-loading path): one coordinate
/// of one entry per cycle, so a full entry pair takes six writes.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct TableWrite {
    /// Write valid this cycle.
    pub valid: bool,
    /// Row: `key * 512 + table row` (truncated to the `KT`-bit table).
    pub row: Bits<KEY_TABLE_BITS>,
    /// Odd image (entries `2*pair + 2`) instead of the even one
    /// (entries `2*pair + 1`).
    pub odd: bool,
    /// Coordinate: 0 `y+x`, 1 `y-x`, 2 `2dxy` (3 writes nothing).
    pub coord: b2,
    /// Coordinate value.
    pub value: Fe17,
}

/// Bank-I write: one input register of one context, from the loader.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct InputWrite<const KB: usize>
where
    rhdl::bits::W<KB>: BitWidth,
{
    /// Write valid this cycle.
    pub valid: bool,
    /// Context.
    pub ctx: Bits<KB>,
    /// Input register index (below [`CONST_BASE`]).
    pub index: b7,
    /// Value.
    pub value: Fe17,
}

/// Machine inputs, all sampled every cycle and independent of each other.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct MachineIn<const KB: usize>
where
    rhdl::bits::W<KB>: BitWidth,
{
    /// Start a context.
    pub start: StartReq<KB>,
    /// Load scan digits.
    pub digits: DigitsWrite<KB>,
    /// Load an input register.
    pub input: InputWrite<KB>,
    /// Load a key-table entry.
    pub table: TableWrite,
    /// The consumer takes [`MachineOut::out`] this cycle. The output
    /// register is freed for new `OUT` bundles from the next cycle on.
    pub out_accept: bool,
}

/// Output-port event from an `OUT` bundle slot.
///
/// The machine has one output register, which is why `OUT` bundles are the
/// only ones that can stall.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct OutEvent<const KB: usize>
where
    rhdl::bits::W<KB>: BitWidth,
{
    /// Event valid; stays set until [`MachineIn::out_accept`].
    pub valid: bool,
    /// Context that emitted it.
    pub ctx: Bits<KB>,
    /// Job tag given at start.
    pub tag: b16,
    /// Program output index (`InstrW::out_index`).
    pub index: b6,
    /// Emitted value.
    pub value: Fe17,
}

/// Machine outputs.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct MachineOut<const KB: usize>
where
    rhdl::bits::W<KB>: BitWidth,
{
    /// Output event, held until accepted.
    pub out: OutEvent<KB>,
    /// Context retired its last bundle this cycle (one-cycle pulse when that
    /// bundle reaches S3). All of its earlier `OUT`s have been captured; an
    /// `OUT` in the last bundle itself appears on `out` one cycle later.
    pub done: bool,
    /// Retired context.
    pub done_ctx: Bits<KB>,
    /// Retired job tag.
    pub done_tag: b16,
}

/// Per-context thread state, held in the rotating ring.
///
/// Entries carry no context number: the ring rotates by one each clock, so
/// `ring[k]` always belongs to context `slot + k (mod K)`.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct Ring<const PB: usize>
where
    rhdl::bits::W<PB>: BitWidth,
{
    /// The context is running a program.
    active: bool,
    /// Program counter of the context's next bundle.
    pc: Bits<PB>,
    /// Job tag of the running program.
    tag: b16,
}

/// Control word that travels with a bundle down stages S2 and S3.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct Stage<const KB: usize, const PB: usize>
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<PB>: BitWidth,
{
    /// A bundle issued (false for stalled or idle turns).
    valid: bool,
    /// The bundle is its program's last (`InstrW::end`).
    last: bool,
    /// Issuing context.
    ctx: Bits<KB>,
    /// Job tag of the issuing context.
    tag: b16,
}

/// All flip-flop state of a [`FieldMachine`] outside its memories and units.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct MachineRegs<const KB: usize, const PB: usize, const K: usize>
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<PB>: BitWidth,
{
    /// Context whose turn it is this cycle (owner of `ring[0]`); wraps mod K.
    slot: Bits<KB>,
    /// Thread state of every context, rotated one place per clock.
    ring: [Ring<PB>; K],
    /// A start request is waiting for the context's next turn.
    pending: [bool; K],
    /// Job tag of the pending start.
    pending_tag: [b16; K],
    /// Key-table slot of each context, latched at start.
    key: [KeySlot; K],
    /// Entry bundle of the pending start.
    pending_entry: [b12; K],
    /// The single output register.
    out: OutEvent<KB>,
    /// S2 control (bundle issued last cycle).
    s2: Stage<KB, PB>,
    /// S2 bundle; slot enables are cleared when the bundle did not issue.
    s2_instr: InstrW,
    /// S2 scan digit, picked in S1 from the digit word read a cycle ahead.
    /// (Until 2026-10-17 the 128:1 pick sat in S2 in front of the
    /// magnitude compare and the 765-bit select: the 8-machine signer's
    /// worst routed path, -0.581 ns, 87 % route.)
    s2_digit: b6,
    /// This cycle's head got its start or its digit word one cycle before
    /// its turn, after its bundle and digit word were read ahead: a start
    /// waits a turn.
    dig_fresh: bool,
    /// The head's bundle, read from the ROM two slots ahead and registered
    /// here, so S1 decodes from flip-flops. (Until 2026-10-17 S1 used the ROM
    /// output directly; with the digit pick in S1 that was the 6-machine
    /// verifier's worst routed path, -0.370 ns, ROM -> t_digit -> 128:1 pick.)
    s1_instr: InstrW,
    /// S3 control.
    s3: Stage<KB, PB>,
    /// S3 bundle.
    s3_instr: InstrW,
    /// S3 operands, in port order: `m_a`, `m_b`, `a_a`, `a_b`.
    s3_ops: [Fe17; 4],
    /// S3 copy of the scan accumulator after this bundle's T step.
    s3_acc: Niels17,
    /// S3 sign of the digit the T step scanned.
    s3_negative: bool,
    /// S4: commit `s4_l` to bank L this cycle.
    s4_commit: bool,
    /// S4: bank-L slot to commit to.
    s4_lslot: bool,
    /// S4: context to commit for.
    s4_ctx: Bits<KB>,
    /// S4: the selected point, already negated if the digit was negative.
    s4_l: Niels17,
    /// The write-only inputs (start, digits, input and table writes),
    /// registered on arrival and acted on one cycle later, in order. The
    /// controller's dispatch (a priority pick over machines and contexts)
    /// and the machine's wide latching (`pending[ctx]` and friends) then
    /// sit in different cycles. (Until 2026-10-17 they shared one: the
    /// 8-machine signer's worst routed path, -1.338 ns, 90 % route, from
    /// `fb_free` to a machine's clock enables.) `out_accept` is a handshake
    /// and stays combinational.
    in_start: StartReq<KB>,
    /// Registered digit write.
    in_digits: DigitsWrite<KB>,
    /// Registered input write.
    in_input: InputWrite<KB>,
    /// Registered table write.
    in_table: TableWrite,
}

/// Constants a program can address (operand index `CONST_BASE + c`): 32,
/// the size of ROM C, which fills the 7-bit index field above the inputs.
pub const CONST_SLOTS: usize = 32;
/// Bank-I operand index of constant 0; lower indices are inputs. 96 leaves
/// room for the compress program's 96 inputs (32 points x 3 coordinates).
pub const CONST_BASE: usize = 96;

/// The constant ROM C: [`CONST_SLOTS`] constants as a literal and a
/// registered read, with the ports of a [`SyncBRAM`] (the write port is
/// ignored).
///
/// This was a `SyncBRAM` with its write disabled, which Vivado 2023.2
/// mapped to RAM32M16 LUT RAM. In the aux machine (constants 1, d, √−1,
/// …) synthesis trimmed the bit columns that are zero in every entry,
/// kept the INIT strings in the original layout, and wired the output
/// flip-flops packed: the card's netlist read d with 16 wrong bits and
/// rejected valid keys (DSig, build v3, 2026-10-19). A literal and a mux
/// leave no RAM to infer.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct ConstRom {
    /// The constants, slot `c` at index `c`.
    table: Constant<[Fe17; CONST_SLOTS]>,
    /// The read register (one clock of latency, as a `SyncBRAM`).
    out: DFF<Fe17>,
}

impl ConstRom {
    /// A ROM holding `consts` (slot, value); missing slots read zero.
    fn new(consts: &[(Bits<5>, Fe17)]) -> Self {
        let mut table = [Fe17::zero(); CONST_SLOTS];
        for &(slot, value) in consts {
            table[slot.raw() as usize] = value;
        }
        Self { table: Constant::new(table), out: DFF::new(Fe17::zero()) }
    }
}

/// Port types of [`ConstRom`]: those of a 32-word `SyncBRAM`.
impl SynchronousIO for ConstRom {
    type I = RamIn<Fe17, 5>;
    type O = Fe17;
    type Kernel = const_rom_kernel;
}

/// Read slot `read_addr` into the register.
#[kernel]
pub fn const_rom_kernel(_cr: ClockReset, i: RamIn<Fe17, 5>, q: ConstRomQ) -> (Fe17, ConstRomD) {
    let mut d = ConstRomD::dont_care();
    d.table = ();
    d.out = q.table[i.read_addr];
    (q.out, d)
}

/// Register banks M, A and I and the constant ROM C, one copy per read
/// port. Bank I holds per-context inputs; C holds the program's constants
/// once for all contexts (32 entries, LUT ROM).
///
/// Each memory has one read and one write port; four copies of each bank
/// with identical writes give the four read ports a bundle needs. Reads have
/// one clock of latency.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct RegBanks<const RA: usize, const IA: usize>
where
    rhdl::bits::W<RA>: BitWidth,
    rhdl::bits::W<IA>: BitWidth,
{
    /// Bank M copies, one per read port (`2^RA` words).
    m: [SyncBRAM<Fe17, RA>; 4],
    /// Bank A copies, one per read port (`2^RA` words).
    a: [SyncBRAM<Fe17, RA>; 4],
    /// Bank I copies, one per read port (`2^IA` words).
    i: [SyncBRAM<Fe17, IA>; 4],
    /// Constant ROM C copies, one per read port (32 words).
    c: [ConstRom; 4],
}

/// Port bundle of [`RegBanks`].
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct RegBanksIn<const RA: usize, const IA: usize>
where
    rhdl::bits::W<RA>: BitWidth,
    rhdl::bits::W<IA>: BitWidth,
{
    /// Bank M ports, indexed by read port.
    pub m: [RamIn<Fe17, RA>; 4],
    /// Bank A ports.
    pub a: [RamIn<Fe17, RA>; 4],
    /// Bank I ports.
    pub i: [RamIn<Fe17, IA>; 4],
    /// Constant ROM ports.
    pub c: [RamIn<Fe17, 5>; 4],
}

/// Read data of [`RegBanks`], one cycle after the read address.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct RegBanksOut {
    /// Bank M copies.
    pub m: [Fe17; 4],
    /// Bank A copies.
    pub a: [Fe17; 4],
    /// Bank I copies.
    pub i: [Fe17; 4],
    /// Constant ROM copies.
    pub c: [Fe17; 4],
}

/// Port types of [`RegBanks`].
impl<const RA: usize, const IA: usize> SynchronousIO for RegBanks<RA, IA>
where
    rhdl::bits::W<RA>: BitWidth,
    rhdl::bits::W<IA>: BitWidth,
{
    type I = RegBanksIn<RA, IA>;
    type O = RegBanksOut;
    type Kernel = reg_banks_kernel<RA, IA>;
}

/// Wiring for [`RegBanks`]: ports pass straight through to the memories.
#[kernel]
pub fn reg_banks_kernel<const RA: usize, const IA: usize>(
    _cr: ClockReset,
    i: RegBanksIn<RA, IA>,
    q: RegBanksQ<RA, IA>,
) -> (RegBanksOut, RegBanksD<RA, IA>)
where
    rhdl::bits::W<RA>: BitWidth,
    rhdl::bits::W<IA>: BitWidth,
{
    let mut d = RegBanksD::<RA, IA>::dont_care();
    d.m = i.m;
    d.a = i.a;
    d.i = i.i;
    d.c = i.c;
    (
        RegBanksOut {
            m: q.m,
            a: q.a,
            i: q.i,
            c: q.c,
        },
        d,
    )
}

/// Table-scan memories: bank L, the scan accumulator, digits, the base-point
/// table (`2^TB` rows) and the key table (`2^KT` rows). A machine that never
/// scans uses `TB = KT = 1`, and one without key lookups `KT = 1`.
///
/// The per-context memories (`bank_l0`, `bank_l1`, `acc`, `digits`) have
/// one word per context, addressed by context number.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct ScanMems<const KB: usize, const TB: usize, const KT: usize>
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<TB>: BitWidth,
    rhdl::bits::W<KT>: BitWidth,
{
    /// Bank L, slot 0: one committed lookup result per context.
    bank_l0: SyncBRAM<Niels17, KB>,
    /// Bank L, slot 1.
    bank_l1: SyncBRAM<Niels17, KB>,
    /// Scan accumulator: the entry selected so far by the current lookup.
    acc: SyncBRAM<Niels17, KB>,
    /// 128 sign-magnitude digits per context.
    digits: SyncBRAM<[b6; 128], KB>,
    /// Base-point table, even image (entries `2 * pair + 1`); preloaded ROM.
    table0: SyncBRAM<Niels17, TB>,
    /// Base-point table, odd image (entries `2 * pair + 2`); preloaded ROM.
    table1: SyncBRAM<Niels17, TB>,
    /// Per-key tables written at run time.
    key: KeyTables<KT>,
}

/// Port bundle of [`ScanMems`].
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct ScanMemsIn<const KB: usize, const TB: usize, const KT: usize>
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<TB>: BitWidth,
    rhdl::bits::W<KT>: BitWidth,
{
    /// Bank L, slot 0.
    pub bank_l0: RamIn<Niels17, KB>,
    /// Bank L, slot 1.
    pub bank_l1: RamIn<Niels17, KB>,
    /// Accumulator.
    pub acc: RamIn<Niels17, KB>,
    /// Digits.
    pub digits: RamIn<[b6; 128], KB>,
    /// Even base-point table entries.
    pub table0: RamIn<Niels17, TB>,
    /// Odd base-point table entries.
    pub table1: RamIn<Niels17, TB>,
    /// Key tables.
    pub key: KeyTablesIn<KT>,
}

/// Read data of [`ScanMems`].
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct ScanMemsOut {
    /// Bank L, slot 0.
    pub bank_l0: Niels17,
    /// Bank L, slot 1.
    pub bank_l1: Niels17,
    /// Accumulator.
    pub acc: Niels17,
    /// Digits.
    pub digits: [b6; 128],
    /// Even base-point table entry.
    pub table0: Niels17,
    /// Odd base-point table entry.
    pub table1: Niels17,
    /// Key-table entries (even, odd).
    pub key: [Niels17; 2],
}

/// Port types of [`ScanMems`].
impl<const KB: usize, const TB: usize, const KT: usize> SynchronousIO for ScanMems<KB, TB, KT>
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<TB>: BitWidth,
    rhdl::bits::W<KT>: BitWidth,
{
    type I = ScanMemsIn<KB, TB, KT>;
    type O = ScanMemsOut;
    type Kernel = scan_mems_kernel<KB, TB, KT>;
}

/// Wiring for [`ScanMems`]: ports pass straight through to the memories.
#[kernel]
pub fn scan_mems_kernel<const KB: usize, const TB: usize, const KT: usize>(
    _cr: ClockReset,
    i: ScanMemsIn<KB, TB, KT>,
    q: ScanMemsQ<KB, TB, KT>,
) -> (ScanMemsOut, ScanMemsD<KB, TB, KT>)
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<TB>: BitWidth,
    rhdl::bits::W<KT>: BitWidth,
{
    let mut d = ScanMemsD::<KB, TB, KT>::dont_care();
    d.bank_l0 = i.bank_l0;
    d.bank_l1 = i.bank_l1;
    d.acc = i.acc;
    d.digits = i.digits;
    d.table0 = i.table0;
    d.table1 = i.table1;
    d.key = i.key;
    (
        ScanMemsOut {
            bank_l0: q.bank_l0,
            bank_l1: q.bank_l1,
            acc: q.acc,
            digits: q.digits,
            table0: q.table0,
            table1: q.table1,
            key: q.key,
        },
        d,
    )
}

/// Key tables: two images (even/odd entries) of three coordinate RAMs each,
/// written one coordinate at a time.
///
/// Splitting by coordinate keeps the write port one [`Fe17`] wide, so the
/// loader needs no read-modify-write of a full Niels entry.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct KeyTables<const KT: usize>
where
    rhdl::bits::W<KT>: BitWidth,
{
    /// `ram[0..3]`: even image `y+x`, `y-x`, `2dxy`; `ram[3..6]`: odd image.
    ram: [SyncBRAM<Fe17, KT>; 6],
}

/// Key tables with the default memory style (left to synthesis).
impl<const KT: usize> Default for KeyTables<KT>
where
    rhdl::bits::W<KT>: BitWidth,
{
    fn default() -> Self {
        KeyTables { ram: std::array::from_fn(|_| SyncBRAM::default()) }
    }
}

impl<const KT: usize> KeyTables<KT>
where
    rhdl::bits::W<KT>: BitWidth,
{
    /// Key tables in UltraRAM (written at run time, never preloaded).
    pub fn ultra() -> Self {
        KeyTables { ram: std::array::from_fn(|_| SyncBRAM::default().with_style(RamStyle::Ultra)) }
    }
}

/// Port bundle of [`KeyTables`]: one shared read row, one coordinate write.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct KeyTablesIn<const KT: usize>
where
    rhdl::bits::W<KT>: BitWidth,
{
    /// Row read by both images.
    pub read_addr: Bits<KT>,
    /// One coordinate write.
    pub write: TableWrite,
}

/// Port types of [`KeyTables`]: the read returns the (even, odd) entries.
impl<const KT: usize> SynchronousIO for KeyTables<KT>
where
    rhdl::bits::W<KT>: BitWidth,
{
    type I = KeyTablesIn<KT>;
    type O = [Niels17; 2];
    type Kernel = key_tables_kernel<KT>;
}

/// Wiring for [`KeyTables`].
#[kernel]
pub fn key_tables_kernel<const KT: usize>(
    _cr: ClockReset,
    i: KeyTablesIn<KT>,
    q: KeyTablesQ<KT>,
) -> ([Niels17; 2], KeyTablesD<KT>)
where
    rhdl::bits::W<KT>: BitWidth,
{
    let mut d = KeyTablesD::<KT>::dont_care();
    let w = i.write;
    // RAM number = image base (0 even, 3 odd) + coordinate; exactly one of
    // the six RAMs is write-enabled, all share the address and data.
    let image = if w.odd { b3(3) } else { b3(0) };
    for k in 0..6 {
        d.ram[k] = RamIn::<Fe17, KT> {
            read_addr: i.read_addr,
            write: RamWrite::<Fe17, KT> {
                addr: w.row.resize::<KT>(),
                value: w.value,
                enable: w.valid && (b3(k as u128) == image + w.coord.resize::<3>()),
            },
        };
    }
    (
        [
            Niels17 { ypx: q.ram[0], ymx: q.ram[1], xy2d: q.ram[2] },
            Niels17 { ypx: q.ram[3], ymx: q.ram[4], xy2d: q.ram[5] },
        ],
        d,
    )
}

/// Barrel field machine. `K = 2^KB` contexts; `RA`/`IA` are bank-M/A and
/// bank-I address widths (`index * K + ctx`); `PB` is the program-counter width.
/// `TB` and `KT` are the base-point and key table address widths. See the
/// module docs for the pipeline and bank layout.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
#[rhdl(dq_no_prefix)]
pub struct FieldMachine<const KB: usize, const K: usize, const RA: usize, const IA: usize, const PB: usize, const TB: usize, const KT: usize>
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<RA>: BitWidth,
    rhdl::bits::W<IA>: BitWidth,
    rhdl::bits::W<PB>: BitWidth,
    rhdl::bits::W<TB>: BitWidth,
    rhdl::bits::W<KT>: BitWidth,
{
    /// Thread ring, pipeline registers and the output register.
    regs: DFF<MachineRegs<KB, PB, K>>,
    /// Program ROM (`2^PB` bundles, preloaded); read one cycle ahead of issue.
    rom: SyncBRAM<InstrW, PB>,
    /// Register banks M, A, I and constant ROM C.
    banks: RegBanks<RA, IA>,
    /// Bank L, scan accumulator, digits and lookup tables.
    scan: ScanMems<KB, TB, KT>,
    /// The field multiplier, fed from S3: the 225-DSP
    /// [`FieldMulPar`](crate::fe17::FieldMulPar) or, with `ED25519_MULTIPLIER=LEAN150`, the 150-DSP
    /// [`FieldMulKara`](crate::fe17_kara::FieldMulKara) (same ports and
    /// latency).
    mul: FieldMul,
    /// The adder/subtractor, fed from S3.
    add: FieldAddSub,
}

/// Encode one operand reference. Bank C shares bank I's code with the index
/// offset by [`CONST_BASE`]. Panics if the encoded index exceeds 7 bits.
/// `program` is not used.
fn encode_operand(o: Operand, _program: &Program) -> OperandW {
    let (bank, index) = match o.bank {
        Bank::M => (BANK_M, o.index as usize),
        Bank::A => (BANK_A, o.index as usize),
        Bank::I => (BANK_I, o.index as usize),
        Bank::C => (BANK_I, CONST_BASE + o.index as usize),
        Bank::L => (BANK_L, o.index as usize),
    };
    assert!(index < 128, "register index exceeds the 7-bit operand field");
    OperandW {
        bank: b2(bank),
        index: b7(index as u128),
    }
}

/// Encode a compiled bundle as a ROM word.
///
/// Empty slots are encoded disabled with harmless operands. Panics if the
/// first multiply factor is in bank L (the instruction set reserves bank L
/// for the second factor) or an operand index does not fit seven bits.
pub fn encode_bundle(b: &Bundle, program: &Program) -> InstrW {
    // Unused operands read input register 0: a valid address in any bank I.
    let none = OperandW {
        bank: b2(BANK_I),
        index: b7(0),
    };
    let mut w = InstrW {
        m_en: false,
        m_dst: b7(0),
        m_a: none,
        m_b: none,
        a_en: false,
        a_op: b2(0),
        a_dst: b7(0),
        a_a: none,
        a_b: none,
        out_index: b6(0),
        t_en: false,
        t_digit: b7(0),
        t_key: false,
        t_pair: b3(0),
        t_row: b9(0),
        t_lslot: false,
        t_clear: false,
        t_finish: false,
        end: false,
    };
    if let Some(m) = b.m {
        w.m_en = true;
        w.m_dst = b7(m.dst as u128);
        w.m_a = encode_operand(m.a, program);
        w.m_b = encode_operand(m.b, program);
        assert!(m.a.bank != Bank::L);
    }
    if let Some(a) = b.a {
        w.a_en = true;
        w.a_dst = b7(a.dst as u128);
        w.a_a = encode_operand(a.a, program);
        w.a_b = encode_operand(a.b, program);
        w.a_op = b2(match a.op {
            AOp::Add => 0,
            AOp::Sub => 1,
            AOp::Out(index) => {
                w.out_index = b6(index as u128);
                2
            }
        });
    }
    if let Some(t) = b.t {
        w.t_en = true;
        w.t_digit = b7(t.digit as u128);
        w.t_key = t.key;
        w.t_pair = b3(t.pair as u128);
        w.t_row = b9(t.row as u128);
        w.t_lslot = t.l_slot == 1;
        w.t_clear = t.clear;
        w.t_finish = t.finish;
    }
    w.end = b.end;
    w
}

/// A fixed-base table as two BRAM images (even and odd entries of each
/// pair), row `table_row(i) + pair`.
///
/// Returns `(address, entry)` preload lists for `table0` and `table1`. The
/// scanner reads both images at one row, so one T step tests two entries.
pub fn table_images(table: &[Vec<Niels>]) -> (Vec<(usize, Niels17)>, Vec<(usize, Niels17)>) {
    let mut even = Vec::new();
    let mut odd = Vec::new();
    for i in 0..DIGITS {
        for e in 0..digit_entries(i) {
            let n = table[i][e];
            let entry = Niels17 {
                ypx: Fe17::from_fp(n.ypx),
                ymx: Fe17::from_fp(n.ymx),
                xy2d: Fe17::from_fp(n.xy2d),
            };
            // Row table_row(i) + pair. `e` is zero-based, so entry number
            // (magnitude) 2*pair+1 has even e = 2*pair and goes to `even`.
            let address = table_row(i) + e / 2;
            if e % 2 == 0 {
                even.push((address, entry));
            } else {
                odd.push((address, entry));
            }
        }
    }
    // Every digit has an even entry count, so each row has both images.
    assert_eq!(even.len(), TABLE_ROWS);
    (even, odd)
}

/// A per-context scanner memory: LUT RAM when `lean` (the program never
/// scans), otherwise left to synthesis.
fn scan_mem<T: Digital, const N: usize>(lean: bool) -> SyncBRAM<T, N>
where
    rhdl::bits::W<N>: BitWidth,
{
    if lean { SyncBRAM::default().with_style(RamStyle::Distributed) } else { SyncBRAM::default() }
}

/// One copy of bank M or A: UltraRAM when `ultra`, LUT RAM when `lut`,
/// otherwise left to synthesis (block RAM).
fn bank<const RA: usize>(ultra: bool, lut: bool) -> SyncBRAM<Fe17, RA>
where
    rhdl::bits::W<RA>: BitWidth,
{
    if ultra {
        SyncBRAM::default().with_style(RamStyle::Ultra)
    } else if lut {
        SyncBRAM::default().with_style(RamStyle::Distributed)
    } else {
        SyncBRAM::default()
    }
}

/// Where a field machine keeps its memories.
///
/// The as-built machines put everything in block RAM, which is what runs
/// out first on a full chip (a fixed-base machine used 72 BRAM36: banks M
/// and A 16 each, bank I 16, base-point tables 22, program 2.5). The
/// specialized layout moves the run-time-written banks to UltraRAM, which
/// the design barely uses, and turns a constants-only bank I into LUT ROM.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct MachineMemory {
    /// Banks M and A and the key tables in UltraRAM (`ram_style = "ultra"`).
    /// Bank I also goes to UltraRAM when the program has inputs and no
    /// presets (UltraRAM cannot be preloaded).
    pub ultra_banks: bool,
    /// Bank I as LUT ROM (`ram_style = "distributed"`) when the program
    /// takes no per-context inputs, so the bank only holds constants. Also
    /// puts the constant ROM C, and the scanner's per-context memories of
    /// programs with no T steps, in LUT RAM.
    pub rom_consts: bool,
    /// Banks M and A in LUT RAM (`ram_style = "distributed"`) when
    /// `ultra_banks` is off: the layout that turns spare LUTs into
    /// machines once UltraRAM and block RAM are spent. Key tables stay in
    /// UltraRAM.
    pub lut_banks: bool,
}

impl MachineMemory {
    /// Every memory left to synthesis (block RAM), as mapped in Vivado on
    /// 2026-10-14.
    pub const AS_BUILT: MachineMemory = MachineMemory { ultra_banks: false, rom_consts: false, lut_banks: false };
    /// UltraRAM banks and LUT-ROM constants (the default for [`FieldMachine::new`]).
    pub const SPECIALIZED: MachineMemory = MachineMemory { ultra_banks: true, rom_consts: true, lut_banks: false };
    /// The block-RAM fallback for a machine that UltraRAM cannot hold:
    /// banks M and A (and key tables) in block RAM, constants still in LUT
    /// ROM. What the planner places when a build runs out of UltraRAM.
    pub const BRAM_BANKS: MachineMemory = MachineMemory { ultra_banks: false, rom_consts: true, lut_banks: false };
    /// The LUT-RAM fallback: banks M and A in LUT RAM, key tables in
    /// UltraRAM, constants in LUT ROM.
    pub const LUT_BANKS: MachineMemory = MachineMemory { ultra_banks: false, rom_consts: true, lut_banks: true };
}

/// The layout of machine `k` of a core's replicated machines (fixed-base
/// machines in a signer, verify machines in a verifier) in this build:
/// [`MachineMemory::SPECIALIZED`] for the first [`URAM_MACHINES`],
/// [`MachineMemory::LUT_BANKS`] for the next [`LUT_MACHINES`], and
/// [`MachineMemory::BRAM_BANKS`] for the rest.
pub fn placement(k: usize) -> MachineMemory {
    if k < URAM_MACHINES {
        MachineMemory::SPECIALIZED
    } else if k - URAM_MACHINES < LUT_MACHINES {
        MachineMemory::LUT_BANKS
    } else {
        MachineMemory::BRAM_BANKS
    }
}

/// Constructors: preload the program ROM, constants and tables, and pick
/// memory styles for the program.
impl<const KB: usize, const K: usize, const RA: usize, const IA: usize, const PB: usize, const TB: usize, const KT: usize>
    FieldMachine<KB, K, RA, IA, PB, TB, KT>
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<RA>: BitWidth,
    rhdl::bits::W<IA>: BitWidth,
    rhdl::bits::W<PB>: BitWidth,
    rhdl::bits::W<TB>: BitWidth,
    rhdl::bits::W<KT>: BitWidth,
{
    /// Build a machine running `program` (specialized memory layout).
    pub fn new(program: &Program) -> Self {
        Self::with_presets(program, &[])
    }

    /// Build a machine running `program` with the given memory layout.
    pub fn with_memory(program: &Program, memory: MachineMemory) -> Self {
        Self::build(program, &[], memory)
    }

    /// Build a machine whose bank-I registers `index` start as `value` in
    /// every context (used for identity defaults in partial batches).
    pub fn with_presets(program: &Program, presets: &[(usize, Fp)]) -> Self {
        Self::build(program, presets, MachineMemory::SPECIALIZED)
    }

    /// Shared constructor. Checks that the parameters fit the program, then
    /// builds every memory's preload image and style.
    fn build(program: &Program, presets: &[(usize, Fp)], memory: MachineMemory) -> Self {
        // K and KB must agree; K >= 32 is what makes hazard logic unnecessary.
        assert_eq!(K, 1 << KB);
        assert!(K >= 32, "barrel depth must cover the 25-cycle write-back");
        // Result tags are `ctx << 7 | dst` in 16 bits.
        assert!(KB <= 9, "result tags hold at most 9 context bits");
        assert!(program.bundles.len() <= 1 << PB);
        assert!(program.regs_m.max(program.regs_a) << KB <= 1 << RA);
        assert!(program.regs_i <= CONST_BASE && program.consts.len() <= CONST_SLOTS);
        assert!(program.regs_i << KB <= 1 << IA, "bank I too small");
        // Program ROM image: bundle `pc` at address `pc`.
        let rom: Vec<(Bits<PB>, InstrW)> = program
            .bundles
            .iter()
            .enumerate()
            .map(|(pc, b)| (bits(pc as u128), encode_bundle(b, program)))
            .collect();
        let consts: Vec<(Bits<5>, Fe17)> =
            program.consts.iter().enumerate().map(|(c, value)| (bits::<5>(c as u128), Fe17::from_fp(*value))).collect();
        // Bank-I preload: the same value at word `index << KB | ctx` for every
        // context.
        let mut preset_words = Vec::new();
        for &(index, value) in presets {
            assert!(index < program.regs_i);
            for ctx in 0..K {
                preset_words.push((bits::<IA>(((index << KB) | ctx) as u128), Fe17::from_fp(value)));
            }
        }
        // Bank I: never written without inputs (LUT RAM, nearly free); run-time
        // inputs without presets can live in UltraRAM.
        let no_inputs = memory.rom_consts && program.regs_i == 0 && presets.is_empty();
        let ultra_inputs = memory.ultra_banks && program.regs_i > 0 && presets.is_empty();
        // Machines that never scan the base-point table keep a 2-row stub.
        let scans = program.bundles.iter().any(|b| b.t.is_some_and(|t| !t.key));
        // Programs with no table steps at all never touch the scanner's
        // memories: keep them in LUT RAM instead of block RAM.
        let lean_scan = memory.rom_consts && !program.bundles.iter().any(|b| b.t.is_some());
        let (even, odd) = if scans {
            assert!(TABLE_ROWS <= 1 << TB, "base-point table needs {TABLE_ROWS} rows");
            let (e, o) = table_images(basepoint_table());
            (
                e.into_iter().map(|(a, v)| (bits::<TB>(a as u128), v)).collect::<Vec<_>>(),
                o.into_iter().map(|(a, v)| (bits::<TB>(a as u128), v)).collect::<Vec<_>>(),
            )
        } else {
            (Vec::new(), Vec::new())
        };
        // Reset state: all contexts idle, pipeline empty.
        let ring = Ring {
            active: false,
            pc: bits(0),
            tag: b16(0),
        };
        let stage = Stage {
            valid: false,
            last: false,
            ctx: bits(0),
            tag: b16(0),
        };
        // An all-slots-disabled bundle for the empty S2/S3 stages.
        let idle = encode_bundle(&Bundle::default(), program);
        let zero = Fe17::zero();
        let niels = Niels17 {
            ypx: zero,
            ymx: zero,
            xy2d: zero,
        };
        Self {
            regs: DFF::new(MachineRegs {
                slot: bits(0),
                ring: [ring; K],
                pending: [false; K],
                pending_tag: [b16(0); K],
                key: [bits(0); K],
                pending_entry: [b12(0); K],
                out: OutEvent {
                    valid: false,
                    ctx: bits(0),
                    tag: b16(0),
                    index: b6(0),
                    value: zero,
                },
                s2: stage,
                s2_instr: idle,
                s2_digit: b6(0),
                dig_fresh: false,
                s1_instr: idle,
                s3: stage,
                s3_instr: idle,
                s3_ops: [zero; 4],
                s3_acc: niels,
                s3_negative: false,
                s4_commit: false,
                s4_lslot: false,
                s4_ctx: bits(0),
                s4_l: niels,
                in_start: StartReq { valid: false, ctx: bits(0), tag: b16(0), key: bits(0), entry: b12(0) },
                in_digits: DigitsWrite { valid: false, ctx: bits(0), digits: [b6(0); 128] },
                in_input: InputWrite { valid: false, ctx: bits(0), index: b7(0), value: zero },
                in_table: TableWrite { valid: false, row: bits(0), odd: false, coord: b2(0), value: zero },
            }),
            rom: SyncBRAM::new(rom),
            banks: RegBanks {
                m: std::array::from_fn(|_| bank(memory.ultra_banks, memory.lut_banks)),
                a: std::array::from_fn(|_| bank(memory.ultra_banks, memory.lut_banks)),
                i: std::array::from_fn(|_| {
                    let bank = SyncBRAM::new(preset_words.clone());
                    if no_inputs {
                        bank.with_style(RamStyle::Distributed)
                    } else if ultra_inputs {
                        bank.with_style(RamStyle::Ultra)
                    } else {
                        bank
                    }
                }),
                c: std::array::from_fn(|_| ConstRom::new(&consts)),
            },
            scan: ScanMems {
                bank_l0: scan_mem(lean_scan),
                bank_l1: scan_mem(lean_scan),
                acc: scan_mem(lean_scan),
                digits: scan_mem(lean_scan),
                table0: SyncBRAM::new(even),
                table1: SyncBRAM::new(odd),
                // KT = 1 is a stub table; UltraRAM is only worth it for real ones.
                key: if (memory.ultra_banks || memory.lut_banks) && KT > 1 { KeyTables::ultra() } else { KeyTables::default() },
            },
            mul: FieldMul::default(),
            add: FieldAddSub::default(),
        }
    }
}

/// Port types of [`FieldMachine`].
impl<const KB: usize, const K: usize, const RA: usize, const IA: usize, const PB: usize, const TB: usize, const KT: usize>
    SynchronousIO for FieldMachine<KB, K, RA, IA, PB, TB, KT>
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<RA>: BitWidth,
    rhdl::bits::W<IA>: BitWidth,
    rhdl::bits::W<PB>: BitWidth,
    rhdl::bits::W<TB>: BitWidth,
    rhdl::bits::W<KT>: BitWidth,
{
    type I = MachineIn<KB>;
    type O = MachineOut<KB>;
    type Kernel = field_machine_kernel<KB, K, RA, IA, PB, TB, KT>;
}

/// The group identity (the point `(0, 1)`) in Niels form: `y + x = 1`,
/// `y - x = 1`, `2dxy = 0`. A lookup of digit 0 returns it, and adding it is
/// a no-op, so zero digits need no special case in the program.
#[kernel]
pub fn niels_identity() -> Niels17 {
    let mut one = [b17(0); 15];
    one[0] = b17(1);
    Niels17 {
        ypx: Fe17 { d: one },
        ymx: Fe17 { d: one },
        xy2d: Fe17 { d: [b17(0); 15] },
    }
}

/// `-x mod p` for canonical `x` (zero stays zero).
///
/// Combinational. [`p_minus`] alone would map 0 to `p`, which is not
/// canonical, hence the zero test.
#[kernel]
pub fn field_neg(x: Fe17) -> Fe17 {
    // OR-reduce the digits.
    let mut zero = true;
    for k in 0..15 {
        if x.d[k] != b17(0) {
            zero = false;
        }
    }
    if zero {
        x
    } else {
        Fe17 {
            d: digits_from_limbs(p_minus(limbs_from_digits(x.d))),
        }
    }
}

/// A memory port bundle that reads `addr` and never writes.
///
/// The write side still needs defined values, so it reuses `addr` and a
/// caller-supplied `filler` (usually the memory's current output) rather
/// than leaving them as don't-cares.
#[kernel]
fn read_disabled<T: Digital, const N: usize>(addr: Bits<N>, filler: T) -> RamIn<T, N>
where
    rhdl::bits::W<N>: BitWidth,
{
    RamIn::<T, N> {
        read_addr: addr,
        write: RamWrite::<T, N> {
            addr: addr,
            value: filler,
            enable: false,
        },
    }
}

/// State transition for [`FieldMachine`].
///
/// One call handles every pipeline stage at once, each working on a
/// different bundle: S1 for the context whose turn it is, S2 and S3 for the
/// bundles issued one and two cycles earlier, S4 for the bank-L commit, and
/// the write-back of whatever the units return this cycle. Values read from
/// `q` are memory outputs addressed in the previous cycle; values written to
/// `d` take effect at the next edge.
#[kernel]
pub fn field_machine_kernel<
    const KB: usize,
    const K: usize,
    const RA: usize,
    const IA: usize,
    const PB: usize,
    const TB: usize,
    const KT: usize,
>(
    _cr: ClockReset,
    i: MachineIn<KB>,
    q: Q<KB, K, RA, IA, PB, TB, KT>,
) -> (MachineOut<KB>, D<KB, K, RA, IA, PB, TB, KT>)
where
    rhdl::bits::W<KB>: BitWidth,
    rhdl::bits::W<RA>: BitWidth,
    rhdl::bits::W<IA>: BitWidth,
    rhdl::bits::W<PB>: BitWidth,
    rhdl::bits::W<TB>: BitWidth,
    rhdl::bits::W<KT>: BitWidth,
{
    let r = q.regs;
    // Next register state; starts as a copy so untouched fields hold.
    let mut n = r;
    let mut d = D::<KB, K, RA, IA, PB, TB, KT>::dont_care();
    // Register the write-only inputs and act on last cycle's (see
    // `MachineRegs::in_start`); the output handshake stays direct.
    n.in_start = i.start;
    n.in_digits = i.digits;
    n.in_input = i.input;
    n.in_table = i.table;
    let i = MachineIn::<KB> { start: r.in_start, digits: r.in_digits, input: r.in_input, table: r.in_table, out_accept: i.out_accept };
    // ---- External start requests: latch them; the context starts on its
    // next turn.
    if i.start.valid {
        n.pending[i.start.ctx] = true;
        n.pending_tag[i.start.ctx] = i.start.tag;
        n.key[i.start.ctx] = i.start.key;
        n.pending_entry[i.start.ctx] = i.start.entry;
    }

    // ---- Output register handshake.
    if i.out_accept {
        n.out.valid = false;
    }
    // An OUT already in S2 or S3 will claim the register when it reaches S3,
    // so a new OUT may issue only if neither stage holds one. Stalled
    // bundles have a_en cleared and do not count.
    let out_in_flight = (r.s2_instr.a_en && r.s2_instr.a_op == b2(2))
        || (r.s3_instr.a_en && r.s3_instr.a_op == b2(2));
    // Uses the registered valid, so an accept frees the register one cycle
    // later.
    let out_free = !r.out.valid && !out_in_flight;

    // ---- S1: the head context issues its prefetched bundle.
    let slot = r.slot;
    let head = r.ring[0];
    // This context's bundle: read two cycles ago, registered last cycle.
    let instr = r.s1_instr;
    n.s1_instr = q.rom;
    // An idle context with a pending start begins now at its entry bundle,
    // unless the start came too late for the read-ahead (dig_fresh).
    let starting = !head.active && r.pending[slot] && !r.dig_fresh;
    let wants = head.active || starting;
    let is_out = instr.a_en && instr.a_op == b2(2);
    // The only stall: an OUT with the output register busy waits a full turn.
    let issue = wants && (!is_out || out_free);
    let pc = if head.active { head.pc } else { r.pending_entry[slot].resize::<PB>() };
    let tag = if starting { r.pending_tag[slot] } else { head.tag };
    if starting && issue {
        n.pending[slot] = false;
    }
    let last = instr.end;
    // Rotate: the head moves to the tail with its advanced program counter.
    for k in 1..K {
        n.ring[k - 1] = r.ring[k];
    }
    n.ring[K - 1] = if issue {
        Ring::<PB> {
            // Retiring after the last bundle returns the context to idle.
            active: !last,
            pc: if last { bits::<PB>(0) } else { pc + bits::<PB>(1) },
            tag: tag,
        }
    } else {
        // Not issued: keep the state unchanged and retry next turn.
        head
    };
    // Wraps at K = 2^KB.
    n.slot = slot + bits::<KB>(1);
    // Prefetch the bundle of the context two turns ahead: the ROM read and
    // s1_instr take a cycle each. Its pc is final, since it last issued
    // K - 2 >= 1 cycles ago.
    let following = r.ring[2];
    let next_slot = slot + bits::<KB>(1);
    let ahead_slot = slot + bits::<KB>(2);
    let ahead_pc = if following.active {
        following.pc
    } else if i.start.valid && i.start.ctx == ahead_slot {
        // A start arriving this cycle is not yet in pending_entry; bypass it.
        i.start.entry.resize::<PB>()
    } else {
        r.pending_entry[ahead_slot].resize::<PB>()
    };
    d.rom = read_disabled::<InstrW, PB>(ahead_pc, instr);

    // S1 register-file addresses: word `index << KB | ctx` in M, A and I,
    // and the low six index bits in C. All four are read; S2 picks the bank.
    let ctx1 = slot;
    let operands = [instr.m_a, instr.m_b, instr.a_a, instr.a_b];
    for p in 0..4 {
        let index = operands[p].index;
        let ra = (index.resize::<RA>() << (KB as u128)) | ctx1.resize::<RA>();
        let ia = (index.resize::<IA>() << (KB as u128)) | ctx1.resize::<IA>();
        d.banks.m[p].read_addr = ra;
        d.banks.a[p].read_addr = ra;
        d.banks.i[p].read_addr = ia;
        // Constant `c` sits at operand index CONST_BASE + c (the subtraction
        // wraps for input indices, whose ROM read is not used).
        d.banks.c[p].read_addr = (index - b7(CONST_BASE as u128)).resize::<5>();
    }
    // Per-context scanner words, and both table images at the bundle's
    // public row.
    d.scan.bank_l0.read_addr = ctx1;
    d.scan.bank_l1.read_addr = ctx1;
    d.scan.acc.read_addr = ctx1;
    // The digit word is read a cycle ahead, for the next head, so this
    // bundle's digit is picked here and S2 sees a 6-bit register. Digits
    // are written only together with a start, to an idle context, so the
    // word changes under a running context never; a write in the same
    // cycle as the read-ahead holds that start back one turn.
    d.scan.digits.read_addr = next_slot;
    n.s2_digit = q.scan.digits[instr.t_digit];
    // A start for the next head arriving now missed its bundle's read-ahead.
    n.dig_fresh = (i.digits.valid && i.digits.ctx == next_slot) || (i.start.valid && i.start.ctx == next_slot);
    d.scan.table0 = read_disabled::<Niels17, TB>(instr.t_row.resize::<TB>(), q.scan.table0);
    d.scan.table1 = read_disabled::<Niels17, TB>(instr.t_row.resize::<TB>(), q.scan.table1);
    // Key rows: key * 512 + row, truncated to KT bits.
    let key_row = ((r.key[slot].resize::<16>() << 9) | instr.t_row.resize::<16>()).resize::<KT>();
    d.scan.key.read_addr = key_row;
    n.s2 = Stage::<KB, PB> {
        valid: issue,
        last: last,
        ctx: ctx1,
        tag: tag,
    };
    n.s2_instr = instr;
    // A bundle that did not issue becomes a bubble: no unit, write or OUT.
    if !issue {
        n.s2_instr.m_en = false;
        n.s2_instr.a_en = false;
        n.s2_instr.t_en = false;
    }

    // ---- S2: operand selection and the constant-address scan.
    // Bank read data addressed in S1 is on q.banks now.
    let ins2 = r.s2_instr;
    let ops2 = [ins2.m_a, ins2.m_b, ins2.a_a, ins2.a_b];
    for p in 0..4 {
        let bank = ops2[p].bank;
        n.s3_ops[p] = if bank == b2(BANK_M) {
            q.banks.m[p]
        } else if bank == b2(BANK_A) {
            q.banks.a[p]
        } else if bank == b2(BANK_I) {
            // Indices from CONST_BASE up select the constant ROM.
            if ops2[p].index >= b7(CONST_BASE as u128) { q.banks.c[p] } else { q.banks.i[p] }
        } else {
            // Bank L. Programs only use it as the second multiply factor
            // (encode_bundle and the Builder enforce this), but the select
            // is generated for every port. Index bit 2 picks the slot and
            // bits 1..0 the coordinate (3 also reads 2dxy).
            let index = ops2[p].index;
            let l = if (index & b7(4)) != b7(0) {
                q.scan.bank_l1
            } else {
                q.scan.bank_l0
            };
            let coordinate = index & b7(3);
            if coordinate == b7(0) {
                l.ypx
            } else if coordinate == b7(1) {
                l.ymx
            } else {
                l.xy2d
            }
        };
    }
    // Constant-address scan step. The digit was selected (in S1) from the
    // context's digit word by the public slot number; its value only steers
    // data multiplexers, never an address or enable.
    let digit = r.s2_digit;
    let magnitude = digit & b6(31);
    let negative = (digit & b6(32)) != b6(0);
    // Entry numbers this step covers: first (even image) and first + 1
    // (odd image).
    let first = (ins2.t_pair.resize::<6>() << 1) + b6(1);
    // The first step of a lookup starts from the identity, so magnitude 0
    // selects the identity.
    let mut acc = if ins2.t_clear { niels_identity() } else { q.scan.acc };
    let (entry0, entry1) = if ins2.t_key {
        (q.scan.key[0], q.scan.key[1])
    } else {
        (q.scan.table0, q.scan.table1)
    };
    if magnitude == first {
        acc = entry0;
    }
    if magnitude == first + b6(1) {
        acc = entry1;
    }
    // The accumulator is written back every T step, one cycle later (in
    // S3, below) from the registered `s3_acc`, so the RAM's write port is
    // fed straight from a register: the digit mux, the compare and the
    // 765-bit select end at flip-flops, not at the LUT-RAM. The context's
    // next step reads it K >= 2 cycles after this step, so the write still
    // lands first, and the read in S1 is another context's.
    n.s3_acc = acc;
    n.s3_negative = negative;
    n.s3 = r.s2;
    n.s3_instr = ins2;

    // ---- S3: issue the arithmetic units, emit outputs, commit bank L.
    let ins3 = r.s3_instr;
    let ctx3 = r.s3.ctx;
    // The scan accumulator write of the step that was in S2 last cycle.
    d.scan.acc.write = RamWrite::<Niels17, KB> {
        addr: ctx3,
        value: r.s3_acc,
        enable: ins3.t_en,
    };
    let ops3 = r.s3_ops;
    // Tags: ctx << 7 | destination register. The write-back below unpacks
    // them into the bank address, so results need no other tracking.
    let mul_tag = (ctx3.resize::<16>() << 7) | ins3.m_dst.resize::<16>();
    d.mul = FieldMulIn {
        valid: ins3.m_en,
        a: ops3[0],
        b: ops3[1],
        tag: mul_tag,
    };
    let add_tag = (ctx3.resize::<16>() << 7) | ins3.a_dst.resize::<16>();
    // OUT uses the A slot's operand but not the adder.
    d.add = FieldAddIn {
        valid: ins3.a_en && ins3.a_op != b2(2),
        sub: ins3.a_op == b2(1),
        a: ops3[2],
        b: ops3[3],
        tag: add_tag,
    };
    // Conditional negation gets its own stage; bank L commits at S4.
    // Negating a Niels point swaps y+x and y-x and negates 2dxy.
    n.s4_commit = ins3.t_en && ins3.t_finish;
    n.s4_lslot = ins3.t_lslot;
    n.s4_ctx = ctx3;
    n.s4_l = if r.s3_negative {
        Niels17 {
            ypx: r.s3_acc.ymx,
            ymx: r.s3_acc.ypx,
            xy2d: field_neg(r.s3_acc.xy2d),
        }
    } else {
        r.s3_acc
    };
    // S4: commit the point registered last cycle into the chosen slot. Two
    // slots let a program start the next lookup while it still reads the
    // previous result.
    d.scan.bank_l0.write = RamWrite::<Niels17, KB> {
        addr: r.s4_ctx,
        value: r.s4_l,
        enable: r.s4_commit && !r.s4_lslot,
    };
    d.scan.bank_l1.write = RamWrite::<Niels17, KB> {
        addr: r.s4_ctx,
        value: r.s4_l,
        enable: r.s4_commit && r.s4_lslot,
    };
    // OUT: capture a_a into the output register. S1's out_free check
    // guarantees the register is empty.
    if ins3.a_en && ins3.a_op == b2(2) {
        n.out = OutEvent::<KB> {
            valid: true,
            ctx: ctx3,
            tag: r.s3.tag,
            index: ins3.out_index,
            value: ops3[2],
        };
    }

    // ---- Write-back: one writer per bank, replicated to every read copy.
    // Unit results arriving this cycle: unpack tag (ctx << 7 | dst) into
    // word dst << KB | ctx.
    let m = q.mul;
    let m_addr = ((m.tag & b16(127)).resize::<RA>() << (KB as u128)) | (m.tag >> 7).resize::<RA>();
    let a = q.add;
    let a_addr = ((a.tag & b16(127)).resize::<RA>() << (KB as u128)) | (a.tag >> 7).resize::<RA>();
    // Loader writes to bank I use the same interleaved layout.
    let i_addr = (i.input.index.resize::<IA>() << (KB as u128)) | i.input.ctx.resize::<IA>();
    for p in 0..4 {
        d.banks.m[p].write = RamWrite::<Fe17, RA> {
            addr: m_addr,
            value: m.r,
            enable: m.valid,
        };
        d.banks.a[p].write = RamWrite::<Fe17, RA> {
            addr: a_addr,
            value: a.r,
            enable: a.valid,
        };
        d.banks.i[p].write = RamWrite::<Fe17, IA> {
            addr: i_addr,
            value: i.input.value,
            enable: i.input.valid,
        };
        // The constant ROM is never written.
        d.banks.c[p].write = RamWrite::<Fe17, 5> {
            addr: b5(0),
            value: i.input.value,
            enable: false,
        };
    }
    // Loader writes to the digit memory and key tables; the base-point
    // tables are ROMs.
    d.scan.digits.write = RamWrite::<[b6; 128], KB> {
        addr: i.digits.ctx,
        value: i.digits.digits,
        enable: i.digits.valid,
    };
    d.scan.table0.write.enable = false;
    d.scan.table1.write.enable = false;
    d.scan.key.write = i.table;
    d.regs = n;
    // done pulses when the last bundle of a program is in S3.
    let output = MachineOut::<KB> {
        out: r.out,
        done: r.s3.valid && r.s3.last,
        done_ctx: ctx3,
        done_tag: r.s3.tag,
    };
    (output, d)
}

/// Host-side helper: pack signed digits into the hardware sign-magnitude form
/// (base-point scalar in slots 0..54; slots 64.. stay zero). For machines
/// without key lookups.
pub fn pack_digits(digits: &[i8; 64]) -> [b6; 128] {
    pack_two(digits, &[0; 64])
}

/// Host-side helper: base-point digits in slots 0..54 and key digits in
/// slots 64..118.
///
/// Panics if either scalar has nonzero digits at or above [`DIGITS`]. Each
/// slot is `|d|` in bits 4..0 with bit 5 set for negative `d`.
pub fn pack_two(base: &[i8; 64], key: &[i8; 64]) -> [b6; 128] {
    assert!(base[DIGITS..].iter().all(|&d| d == 0) && key[DIGITS..].iter().all(|&d| d == 0));
    std::array::from_fn(|k| {
        let d = if k < 64 { base[k] } else { key[k - 64] };
        let magnitude = d.unsigned_abs() as u128;
        let sign = if d < 0 { 32 } else { 0 };
        b6(magnitude | sign)
    })
}