rhdl_ed25519_scalable_core/

verify_core.rs

//! Known-key strict verification core.
//!
//! [`KnownKeyVerifyCore`] verifies Ed25519 signatures of 64-byte messages
//! for up to [`KEY_SLOTS`] public keys loaded in advance (8, or 16 with the
//! feature `key-slots-16`), with the strict rules and
//! error codes of the `StrictVerifyCore` in `rhdl_ed25519_fast_core`: `S` must
//! be canonical, `A` and `R` must be canonical curve points and not of low
//! order, and `[S]B = R + [k]A` must hold (no cofactor).
//!
//! # Dataflow
//!
//! ```text
//! request ──> admission ──> SHA ring: H(R || A || M) ──> scalar ring: k = H mod l
//!               │ (key slot, S < l, R canonical, R low order)        │
//!               │                                                    v
//!               │         verify machine: [S]B + [k](-A) <── recode S and k
//!               │                       │ X, Y, Z
//!               v                       v
//!            result <── compare with R <── CompressStage (32 points per inversion)
//!               ^                  │ mismatch
//!               └── aux machine: decode R (BAD_R or EQUATION_FAILURE)
//! ```
//!
//! Each verify machine scans two tables per verification: the base-point
//! table (ROM) for `S` and the key's table for `k`, so the equation costs two
//! fixed-base walks (775 bundles) instead of a variable-base multiplication.
//! The key table holds multiples of `-A`, so the machine computes
//! `[S]B + [k](-A)`, which equals `R` exactly when the equation holds.
//!
//! # Contexts and tags
//!
//! A request is accepted into a one-entry admission register, where its key
//! is matched and its immediate result decided. On the next cycle (or as
//! soon as a context, a credit and a SHA resource are free) it is admitted:
//! an admitted request takes one of 2,048 *contexts* from a free list. The
//! context number indexes the per-request memories ([`VerifyMems`]: `R`,
//! `S` and the request facts), so only the 11-bit context and the 3- or 4-bit key
//! slot travel with the job. They are packed into the 16-bit tag as
//! `slot << 11 | ctx` through the SHA and scalar rings; the verify machine
//! and the compress stage carry the context alone. The context returns to
//! the free list when its result is reported. Requests that fail at
//! admission (unknown key, invalid key, non-canonical `R`) never take one.
//!
//! # Key loading
//!
//! `load_key_valid` with an encoded `A` and a slot starts a key load on the
//! aux machine: decode `A` ([`decode_candidate`]), compute the 54 table bases
//! of `-A` ([`table_bases`]), then 54 digit jobs ([`digit_table`]) whose
//! outputs are written into every verify machine's key table. The writes
//! pass from machine to machine, one per cycle, so a slot becomes usable
//! `NV + 2` cycles after its last digit job retires (about 105k cycles in
//! all). A key that does not decode is kept as *invalid*, so its requests
//! get [`VERIFY_BAD_PUBLIC_KEY`]. `ready` is low while any key loads, and
//! while a key load is offered: a load offered in the same cycle as a
//! request goes first. A load starts only with no request held or in
//! flight.
//!
//! The aux machine's 32 contexts are split by job: context 0 decodes (a key
//! or an `R`), context 1 computes table bases, and contexts 2..32 run digit
//! jobs, up to 30 at once. Its tags carry the job kind in bits 8.. (one of
//! the `AUX_*` constants) and, for digit jobs, the digit number in bits 0..8.
//!
//! In the key tables, digit `i` occupies the rows from [`digit_row`]`(i)`:
//! eight rows (16 entries, two per row) for the 39 radix-32 digits and four
//! rows (8 entries) for the 15 radix-16 digits, 372 rows per key. Key slot
//! `s` owns rows `s * 512 ..`.
//!
//! # Errors
//!
//! In priority order: [`VERIFY_BAD_PUBLIC_KEY`], [`VERIFY_BAD_R`],
//! [`VERIFY_SMALL_PUBLIC_KEY`], [`VERIFY_SMALL_R`], [`VERIFY_NONCANONICAL_S`],
//! [`VERIFY_EQUATION_FAILURE`]; [`VERIFY_UNKNOWN_KEY`] for a key that is not
//! loaded. When the computed point does not encode to `R`, the aux machine
//! decodes `R` to tell an invalid `R` from a failed equation (a slow path
//! only invalid signatures take).
//!
//! The errors are found in three places:
//!
//! - **At admission**: unknown key, invalid key, and `R` whose `y` is not
//!   below `p` ([`VERIFY_BAD_R`]). These are reported at once.
//! - **At the compare**: if the computed point encodes to `R`, then `R` is a
//!   valid point encoding, so [`VERIFY_BAD_R`] cannot apply and the result
//!   is the first of the small-key, small-`R` and non-canonical-`S` facts
//!   recorded at admission, or [`VERIFY_OK`].
//! - **After the `R` decode** (mismatch only): [`VERIFY_BAD_R`] if `R` does
//!   not decode, else the same admission facts, else
//!   [`VERIFY_EQUATION_FAILURE`].
//!
//! The admission facts are not reported at admission because
//! [`VERIFY_BAD_R`] outranks them and needs the decode, which the compare
//! avoids in the common case. So every request past admission runs the
//! equation, even one whose `S` is already known to be non-canonical; its
//! computed point is then irrelevant.
//!
//! # Deadlock freedom
//!
//! Admission takes a compress credit, as in [`crate::sign_core`], so every
//! verify-machine output finds a compress slot. The rest of the flow has no
//! cycle: compare waits only for the result register, which drains every
//! cycle, and the aux machine's jobs (key loading, `R` decodes) never wait on
//! the verification path.

use rhdl::prelude::*;
use rhdl_fpga::core::{
    dff::DFF,
    ram::synchronous::{In as RamIn, SyncBRAM, Write as RamWrite},
};

use crate::{
    compress_stage::{COMPRESS_SLOTS, CompressIn, CompressStage, PointIn},
    fe17::{Fe17, add_limb, digits_from_limbs, limbs_from_digits, nb_const},
    machine::{
        DigitsWrite, FieldMachine, InputWrite, KEY_SLOTS, KEY_TABLE_BITS, KeySlot, MachineIn,
        StartReq, TableWrite,
    },
    microcode::concat,
    programs::{decode_candidate, digit_table, table_bases, verify_known_key},
    scalar_ring::{ScalarJob, ScalarRing, ScalarRingIn},
    sha_ring::{ShaDone, ShaJob, ShaRing, ShaRingIn, ShaRingOut},
    sign_core::{
        challenge_block, challenge_second_pad, digest_le, idle_table_write, ram_read, recode_mixed,
        words_to_digits15, words_to_digits31,
    },
    wire::{ScalarWords, VerifyRequest, Words512},
};

/// Verification succeeded.
pub const VERIFY_OK: u128 = 0;
/// The public key is not a canonical curve point.
pub const VERIFY_BAD_PUBLIC_KEY: u128 = 1;
/// `R` is not a canonical curve point.
pub const VERIFY_BAD_R: u128 = 2;
/// The public key has low order.
pub const VERIFY_SMALL_PUBLIC_KEY: u128 = 3;
/// `R` has low order.
pub const VERIFY_SMALL_R: u128 = 4;
/// `S` is not below `l`.
pub const VERIFY_NONCANONICAL_S: u128 = 5;
/// `[S]B = R + [k]A` does not hold.
pub const VERIFY_EQUATION_FAILURE: u128 = 6;
/// The public key is not loaded in any slot.
pub const VERIFY_UNKNOWN_KEY: u128 = 7;

/// Verify machine: 32 contexts, base-point table, key tables for 8 keys.
///
/// Parameters: `KB = 5` (32 contexts), bank-M/A address width 9 (16
/// registers per context), bank-I width 5 (the program has no inputs),
/// program-counter width 10 (775 bundles), base-point table `2^9` rows (372
/// used) and key tables `2^KEY_TABLE_BITS` rows ([`KEY_SLOTS`] slots of 512).
pub type VerifyMachine = FieldMachine<5, 32, 9, 5, 10, 9, KEY_TABLE_BITS>;
/// Aux machine: key loading and `R` decoding (four programs in one ROM).
///
/// Parameters: 32 contexts, bank-M/A width 11 (64 registers per context),
/// bank-I width 9 (16 input registers per context), program-counter width
/// 12, and no table scanner (`TB = KT = 1`).
pub type AuxMachine = FieldMachine<5, 32, 11, 9, 12, 1, 1>;

/// Key slot state: nothing loaded; the slot matches no request.
const KEY_EMPTY: u128 = 0;
/// Key slot state: tables are being built. Admission is closed meanwhile.
const KEY_LOADING: u128 = 1;
/// Key slot state: tables complete, requests are verified.
const KEY_READY: u128 = 2;
/// Key slot state: the key did not decode; requests get
/// [`VERIFY_BAD_PUBLIC_KEY`].
const KEY_INVALID: u128 = 3;

/// Aux job kind (tag bits 8..): point decode on context 0.
const AUX_DECODE: u128 = 0;
/// Aux job kind: table bases on context 1.
const AUX_BASES: u128 = 1;
/// Aux job kind: one digit's table on a context in 2..32; tag bits 0..8
/// hold the digit.
const AUX_DIGIT: u128 = 2;

/// Key-load state: no load in progress (admission open).
const KL_IDLE: u128 = 0;
/// Key-load state: decoding `A` on aux context 0.
const KL_DECODE: u128 = 1;
/// Key-load state: feeding and running the table-bases job of `-A`.
const KL_BASES: u128 = 2;
/// Key-load state: feeding and running the 54 digit jobs.
const KL_DIGITS: u128 = 3;

/// Decode-collector owner: free.
const DEC_NONE: u128 = 0;
/// Decode-collector owner: the key load (decoding `A`).
const DEC_KEY: u128 = 1;
/// Decode-collector owner: the slow path (decoding a request's `R`).
const DEC_R: u128 = 2;

/// Inputs of [`KnownKeyVerifyCore`].
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct VerifyIn {
    /// Verify `request` when [`VerifyOut::ready`] (taken in the same cycle).
    pub valid: bool,
    /// Request: 64-byte message, encoded `A`, `R || S`, and a tag.
    pub request: VerifyRequest,
    /// Load `load_key` into `load_slot` when [`VerifyOut::load_key_ready`]
    /// (taken in the same cycle). A slot's previous key is replaced.
    pub load_key_valid: bool,
    /// Encoded public key `A` (32 bytes as four little-endian words).
    /// Requests match a slot by these exact bytes.
    pub load_key: ScalarWords,
    /// Key slot, 0..[`KEY_SLOTS`].
    pub load_slot: KeySlot,
    /// The shared SHA-512 port (`SH = 1`; ignored otherwise), as a ring's
    /// outputs: see [`crate::sha_client::SharedShaPort`].
    pub sha_ext: ShaRingOut,
}

/// Outputs of [`KnownKeyVerifyCore`].
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct VerifyOut {
    /// A request offered this cycle is accepted. Never depends on the
    /// offered request or on `valid`: it depends on the core's registers and
    /// on `load_key_valid` (a key load offered in the same cycle goes first).
    pub ready: bool,
    /// One-cycle result pulse. There is no back-pressure: the consumer must
    /// take every result.
    pub valid: bool,
    /// Verification succeeded.
    pub ok: bool,
    /// Error code (0 on success).
    pub error: b8,
    /// Tag of the request.
    pub tag: b16,
    /// Requests in flight: contexts taken and not yet returned (0..=2048).
    /// Requests rejected at admission are not counted.
    pub in_flight: b12,
    /// A key load can start: no load is in progress and the aux machine's
    /// decode path is idle.
    pub load_key_ready: bool,
    /// Slot states: 0 empty, 1 loading, 2 ready, 3 invalid key.
    pub key_state: [b2; KEY_SLOTS],
    /// Sticky invariant violation: the compress credit count left its pool
    /// (the only invariant checked so far).
    pub internal_error: bool,
    /// Jobs for the shared SHA-512 port (`SH = 1`; idle otherwise).
    pub sha_ext: ShaRingIn,
}

/// Request facts recorded at admission, per context, for the final error
/// decision.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct Meta {
    /// The request's tag, reported with the result.
    tag: b16,
    /// Key slot the request matched.
    slot: KeySlot,
    /// The key's encoding is one of the eight low-order encodings.
    a_small: bool,
    /// `R`'s encoding is one of the eight low-order encodings.
    r_small: bool,
    /// `S < l`.
    canonical_s: bool,
}

/// A challenge digest waiting for a scalar ring (`k = H mod l`).
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct ScStage {
    /// The register holds a digest.
    valid: bool,
    /// `slot << 11 | ctx`, as given to the SHA job.
    tag: b16,
    /// The 512-bit digest as 31 little-endian 17-bit digits.
    x: [b17; 31],
}

/// A request whose `k` is known, waiting for a free verify-machine context.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct VmStage {
    /// The register holds a request.
    valid: bool,
    /// `s` has been read from memory and recoded (one cycle after `valid`).
    has_mem: bool,
    /// Context of the request.
    ctx: b11,
    /// Recoded digits of `k` (sign-magnitude, 54 used).
    k: [b6; 64],
    /// Recoded digits of `S` (sign-magnitude, 54 used).
    s: [b6; 64],
    /// Key slot: selects the key table the machine scans.
    slot: KeySlot,
}

/// An encoded result point waiting to be compared with its `R`.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct CmpStage {
    /// The register holds a point.
    valid: bool,
    /// Context of the request.
    ctx: b11,
    /// Encoding of `[S]B + [k](-A)` (four little-endian words).
    enc: [b64; 4],
}

/// The result register: drives the result ports for one cycle.
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct FinStage {
    /// A result is reported this cycle.
    valid: bool,
    /// Return `ctx` to the free list (false for admission rejects, which
    /// took no context).
    free_ctx: bool,
    /// Context to return.
    ctx: b11,
    /// The request's tag.
    tag: b16,
    /// One of the `VERIFY_*` codes.
    error: b8,
}

/// Registers of [`KnownKeyVerifyCore`] (`NV` verify machines).
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
#[doc(hidden)]
pub struct VerifyRegs<const NV: usize> {
    // Key slots.
    /// Encoded key of each slot, compared byte for byte with requests' `A`.
    key_enc: [[b64; 4]; KEY_SLOTS],
    /// `KEY_*` state of each slot.
    key_state: [b2; KEY_SLOTS],
    /// The slot's key encoding is of low order (reported per request as
    /// [`VERIFY_SMALL_PUBLIC_KEY`]).
    key_small: [bool; KEY_SLOTS],
    // Key-load state machine.
    /// `KL_*` state; one key loads at a time.
    kl_state: b2,
    /// Slot being loaded.
    kl_slot: KeySlot,
    /// The bases job has been started (stops the `KL_BASES` feed).
    kl_issued: bool,
    /// Bases outputs received so far, which is also the next write address
    /// in the bases memory (54 bases x 4 coordinates = 216).
    kl_count: b8,
    /// Next digit job to feed (0..54; 54 means all fed).
    kl_feed_digit: b6,
    /// Step of the input feed: 0..2 in `KL_BASES`, 0..5 in `KL_DIGITS`.
    kl_feed_step: b3,
    /// Aux context chosen for the digit job being fed.
    kl_feed_ctx: b5,
    /// Digit jobs retired (54 completes the load).
    kl_digits_done: b6,
    /// Aux contexts 2..32 running a digit job, one bit per context.
    aux_busy: b32,
    /// After the last digit job retires, cycles left until its table writes
    /// have passed down `tbl` into every machine (0: no drain running). The
    /// slot turns READY when it reaches 0.
    kl_drain: b7,
    /// Key-table writes on their way down the machines: machine `k` takes
    /// `tbl[k]`, and `tbl[k]` moves to `tbl[k + 1]` the next cycle. Every
    /// machine holds a copy of every table, so all of them see the same
    /// writes. Driving them in the same cycle let Vivado merge the machines'
    /// identical input registers into one, which fed table memories all over
    /// the die (−2.12 ns in the routed L200 verifier, 2026-10-19). Shifted
    /// one machine per cycle, no two registers are equal, and each hop goes
    /// to a neighbour.
    tbl: [TableWrite; NV],
    /// First bundle of each aux program: decode, bases, 16-entry digit
    /// table, 8-entry digit table (see [`aux_programs`]).
    aux_entry: [b12; 4],
    // Decode collector (key or R, one at a time).
    /// `DEC_*`: whose decode runs on aux context 0.
    dec_owner: b2,
    /// Write `dec_y` and start the decode program next cycle.
    dec_feed: bool,
    /// `y` of the point being decoded; during a key load it stays `y` of
    /// `A` (equal to `y` of `-A`) for the bases job.
    dec_y: Fe17,
    /// Sign bit of the encoding being decoded (the parity of `x`).
    dec_sign: bool,
    /// Context of the request whose `R` is being decoded.
    dec_ctx: b11,
    /// Decode outputs `x`, `x * sqrt(-1)`, `v x^2`, `u`. After a key decode,
    /// entry 0 is replaced by `x` of `-A` for the bases job.
    dec_vals: [Fe17; 4],
    /// Decode outputs received (0..4), then the finishing steps: 4, 5 and 6
    /// form limb 0, 1 and 2 of `p - x`, `p - x sqrt(-1)` and `p - u`
    /// (`dec_neg`); at 7 the two root tests are registered (`dec_plain`,
    /// `dec_flipped`) and at 8 the root is chosen.
    dec_count: b4,
    /// `v x^2 = u` (the candidate `x` is a root), registered at step 7.
    dec_plain: bool,
    /// `v x^2 = -u` (`x sqrt(-1)` is a root), registered at step 7.
    /// Comparing and selecting in one cycle (255-bit compare, then the root
    /// and parity selects) was a −2.16 ns path of the routed L200 verifier
    /// (2026-10-19).
    dec_flipped: bool,
    /// `p - x`, `p - x sqrt(-1)` and `p - u` as limbs, formed one limb per
    /// cycle so no cycle holds more than one 85-bit carry chain. (One-cycle
    /// negations in series were the worst path of the routed verifiers:
    /// -2.165 ns at 6 machines, 2026-10-17.)
    dec_neg: [[b85; 3]; 3],
    /// Carries into the next limb of each `dec_neg` entry.
    dec_neg_c: [bool; 3],
    /// A decode program is running on aux context 0 (from its start until
    /// it retires). A new decode waits for it: its input write would
    /// otherwise overwrite the running program's input.
    dec_running: bool,
    // Admission register.
    /// `adm_req` holds an accepted request that has not yet taken a context.
    adm_valid: bool,
    /// The accepted request.
    adm_req: VerifyRequest,
    /// Its immediate result (`VERIFY_OK` if it goes on to be verified),
    /// decided when it was accepted.
    adm_error: b8,
    /// Its admission facts, decided when it was accepted.
    adm_meta: Meta,
    // Free contexts.
    /// `free_head` holds a free context ready for admission.
    free_head_valid: bool,
    /// Next free context, prefetched so admission needs no RAM latency.
    free_head: b11,
    /// A free-list read was issued last cycle.
    free_fetching: bool,
    /// Free-list read pointer (wraps at 2048).
    free_rd: b11,
    /// Free-list write pointer (wraps at 2048).
    free_wr: b11,
    /// Entries in the free-list RAM, not counting `free_head` or one being
    /// fetched.
    free_count: b12,
    /// Compress-stage credits: point slots this core may still fill.
    credits: b12,
    // Stages.
    /// Challenge digest waiting for a scalar ring.
    sc: ScStage,
    /// Request waiting for a verify-machine context.
    vm: VmStage,
    /// Free contexts of each verify machine, one bit per context.
    vm_free: [b32; NV],
    /// `vm_free[k] != 0`, registered: the dispatch picks a machine from one
    /// bit per machine instead of OR-ing 32 free bits of every machine in
    /// the same cycle (with 24 machines that pick, 14 levels from `vm_free`
    /// back to `vm_free`, was the routed L200 verifier's worst path,
    /// −2.225 ns, 2026-10-19).
    vm_has_free: [bool; NV],
    /// Verify-machine outputs as seen last cycle and not granted then:
    /// offer and whether it is an `X`. A machine holds `out` until accepted,
    /// so a registered offer is still valid (same coordinate) this cycle.
    out_req: [bool; NV],
    out_req_x: [bool; NV],
    /// The coordinate picked last cycle, offered to the compress stage now,
    /// and the machine it came from (one-hot). That machine keeps offering
    /// it until the stage takes it.
    pt: PointIn,
    pt_src: [bool; NV],
    /// Encoded point waiting for its compare.
    cmp: CmpStage,
    /// A mismatched request waits for the aux decode collector.
    slow_valid: bool,
    /// Its context.
    slow_ctx: b11,
    /// Its encoded `R`.
    slow_r: [b64; 4],
    /// Its admission facts.
    slow_meta: Meta,
    /// Admission facts of the request whose `R` is being decoded.
    dec_meta: Meta,
    /// Result register.
    fin: FinStage,
    /// Sticky internal error: set when the credit count leaves its pool.
    error: bool,
}

/// Per-context memories, plus the free list and the key-load bases.
///
/// All are block RAMs with a one-cycle read latency, indexed by context
/// (2,048 entries) except `bases`.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
pub struct VerifyMems {
    /// Encoded `R` of each request, read at the compare.
    r_enc: SyncBRAM<[b64; 4], 11>,
    /// `S` of each request, read when the verify machine is fed.
    s_words: SyncBRAM<[b64; 4], 11>,
    /// Admission facts of each request, read at the compare.
    meta: SyncBRAM<Meta, 11>,
    /// Ring buffer of free context numbers, preloaded with 0..2048.
    free_list: SyncBRAM<b11, 11>,
    /// The 54 table bases of `-A` during a key load, `X, Y, Z, T` of base
    /// `i` at `4i..4i + 4`.
    bases: SyncBRAM<Fe17, 8>,
}

/// Port bundle of [`VerifyMems`].
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct VerifyMemsIn {
    /// Encoded `R`, by context.
    pub r_enc: RamIn<[b64; 4], 11>,
    /// `S`, by context.
    pub s_words: RamIn<[b64; 4], 11>,
    /// Request facts, by context.
    pub meta: RamIn<Meta, 11>,
    /// Free contexts (ring buffer).
    pub free_list: RamIn<b11, 11>,
    /// Table bases during key loading.
    pub bases: RamIn<Fe17, 8>,
}

/// Read data of [`VerifyMems`].
#[derive(Copy, Clone, Debug, PartialEq, Digital)]
pub struct VerifyMemsOut {
    /// Encoded `R` at last cycle's read address.
    pub r_enc: [b64; 4],
    /// `S` at last cycle's read address.
    pub s_words: [b64; 4],
    /// Request facts at last cycle's read address.
    pub meta: Meta,
    /// Free context at last cycle's read address.
    pub free_list: b11,
    /// Table base coordinate at last cycle's read address.
    pub bases: Fe17,
}

/// Ports and kernel of [`VerifyMems`].
impl SynchronousIO for VerifyMems {
    /// Input port bundle.
    type I = VerifyMemsIn;
    /// Output port bundle.
    type O = VerifyMemsOut;
    /// Kernel computing the outputs and the next state.
    type Kernel = verify_mems_kernel;
}

/// Wiring for [`VerifyMems`]: each port goes straight to its RAM.
#[kernel]
pub fn verify_mems_kernel(_cr: ClockReset, i: VerifyMemsIn, q: VerifyMemsQ) -> (VerifyMemsOut, VerifyMemsD) {
    let mut d = VerifyMemsD::dont_care();
    d.r_enc = i.r_enc;
    d.s_words = i.s_words;
    d.meta = i.meta;
    d.free_list = i.free_list;
    d.bases = i.bases;
    (
        VerifyMemsOut { r_enc: q.r_enc, s_words: q.s_words, meta: q.meta, free_list: q.free_list, bases: q.bases },
        d,
    )
}

/// Known-key strict verifier with `NV` verify machines, `NSHA` SHA-512 rings
/// and `NSC` scalar rings; with `SH = 1` it hashes through the shared
/// SHA-512 port ([`VerifyOut::sha_ext`]) instead and its rings stay idle
/// (use `NSHA = 1`; synthesis removes an idle ring).
///
/// Limits (checked in `Default`): `1 <= NV <= 64`, `NSHA >= 1`, `NSC >= 1`,
/// `SH <= 1`.
#[derive(Clone, Debug, Synchronous, SynchronousDQ)]
#[rhdl(dq_no_prefix)]
pub struct KnownKeyVerifyCore<const NV: usize, const NSHA: usize, const NSC: usize, const SH: usize> {
    /// Control state.
    regs: DFF<VerifyRegs<NV>>,
    /// Per-context memories, free list, bases.
    mems: VerifyMems,
    /// Verify machines, each with its own copy of all eight key tables.
    vm: [VerifyMachine; NV],
    /// SHA-512 rings for the challenge (idle when `SH = 1`).
    sha: [ShaRing; NSHA],
    /// Scalar rings for `k = H mod l`.
    sc: [ScalarRing; NSC],
    /// Batch point encoding of the verify machines' results.
    cps: CompressStage,
    /// Key loading and `R` decoding.
    aux: AuxMachine,
}

/// The aux machine's ROM and the entry of each program (decode, bases,
/// 16-entry digit table, 8-entry digit table).
///
/// Radix-32 digits need 16 table entries and radix-16 digits 8, hence two
/// builds of [`digit_table`]. The entries end up in `aux_entry`, in this
/// order.
pub fn aux_programs() -> (crate::microcode::Program, Vec<usize>) {
    let dec = decode_candidate();
    let bases = table_bases();
    let wide = digit_table(16);
    let narrow = digit_table(8);
    concat("aux", &[&dec, &bases, &wide, &narrow])
}

/// Reset state: all slots empty, all 2,048 contexts free, full compress
/// credits, every verify-machine context free.
impl<const NV: usize, const NSHA: usize, const NSC: usize, const SH: usize> Default for KnownKeyVerifyCore<NV, NSHA, NSC, SH> {
    /// Builds the reset state.
    fn default() -> Self {
        // Parameter limits; SH selects internal rings (0) or the shared
        // port (1).
        assert!(NV >= 1 && NV <= 64 && NSHA >= 1 && NSC >= 1 && SH <= 1);
        let (aux_rom, entries) = aux_programs();
        let zero = Fe17::zero();
        let regs = VerifyRegs::<NV> {
            key_enc: [[b64(0); 4]; KEY_SLOTS],
            key_state: [b2(KEY_EMPTY); KEY_SLOTS],
            key_small: [false; KEY_SLOTS],
            kl_state: b2(KL_IDLE),
            kl_slot: bits(0),
            kl_issued: false,
            kl_count: b8(0),
            kl_feed_digit: b6(0),
            kl_feed_step: b3(0),
            kl_feed_ctx: b5(0),
            kl_digits_done: b6(0),
            aux_busy: b32(0),
            kl_drain: b7(0),
            tbl: [idle_table_write(); NV],
            aux_entry: std::array::from_fn(|k| b12(entries[k] as u128)),
            dec_owner: b2(DEC_NONE),
            dec_feed: false,
            dec_y: zero,
            dec_sign: false,
            dec_ctx: b11(0),
            dec_vals: [zero; 4],
            dec_count: b4(0),
            dec_plain: false,
            dec_flipped: false,
            dec_neg: [[b85(0); 3]; 3],
            dec_neg_c: [false; 3],
            dec_running: false,
            adm_valid: false,
            adm_req: VerifyRequest {
                message: Words512 { words: [b64(0); 8] },
                public_key: ScalarWords { words: [b64(0); 4] },
                signature: Words512 { words: [b64(0); 8] },
                tag: b16(0),
            },
            adm_error: b8(0),
            adm_meta: Meta { tag: b16(0), slot: bits(0), a_small: false, r_small: false, canonical_s: false },
            free_head_valid: false,
            free_head: b11(0),
            free_fetching: false,
            free_rd: b11(0),
            free_wr: b11(0),
            // The free list starts full: its RAM is preloaded with 0..2048.
            free_count: b12(2048),
            credits: b12(COMPRESS_SLOTS as u128),
            sc: ScStage { valid: false, tag: b16(0), x: [b17(0); 31] },
            vm: VmStage { valid: false, has_mem: false, ctx: b11(0), k: [b6(0); 64], s: [b6(0); 64], slot: bits(0) },
            vm_free: [b32(0xffff_ffff); NV],
            vm_has_free: [true; NV],
            out_req: [false; NV],
            out_req_x: [false; NV],
            pt: PointIn { valid: false, tag: b16(0), index: b2(0), value: zero },
            pt_src: [false; NV],
            cmp: CmpStage { valid: false, ctx: b11(0), enc: [b64(0); 4] },
            slow_valid: false,
            slow_ctx: b11(0),
            slow_r: [b64(0); 4],
            slow_meta: Meta { tag: b16(0), slot: bits(0), a_small: false, r_small: false, canonical_s: false },
            dec_meta: Meta { tag: b16(0), slot: bits(0), a_small: false, r_small: false, canonical_s: false },
            fin: FinStage { valid: false, free_ctx: false, ctx: b11(0), tag: b16(0), error: b8(0) },
            error: false,
        };
        let program = verify_known_key();
        Self {
            regs: DFF::new(regs),
            mems: VerifyMems {
                r_enc: SyncBRAM::default(),
                s_words: SyncBRAM::default(),
                meta: SyncBRAM::default(),
                // Entry k holds context k.
                free_list: SyncBRAM::new((0..2048u128).map(|k| (b11(k), b11(k)))),
                bases: SyncBRAM::default(),
            },
            // As in the signer: the first `URAM_MACHINES` in UltraRAM, the
            // rest in the block-RAM fallback.
            vm: std::array::from_fn(|k| VerifyMachine::with_memory(&program, crate::machine::placement(k))),
            sha: std::array::from_fn(|_| ShaRing::default()),
            sc: std::array::from_fn(|_| ScalarRing::default()),
            cps: CompressStage::default(),
            aux: AuxMachine::new(&aux_rom),
        }
    }
}

/// Ports and kernel of [`KnownKeyVerifyCore`].
impl<const NV: usize, const NSHA: usize, const NSC: usize, const SH: usize> SynchronousIO for KnownKeyVerifyCore<NV, NSHA, NSC, SH> {
    /// Input port bundle.
    type I = VerifyIn;
    /// Output port bundle.
    type O = VerifyOut;
    /// Kernel computing the outputs and the next state.
    type Kernel = known_key_verify_kernel<NV, NSHA, NSC, SH>;
}

/// `y < p` for a 32-byte encoding (sign bit ignored).
///
/// `p = 2^255 - 19` has words `ffff..ffed, ffff..ffff, ffff..ffff,
/// 7fff..ffff` (least significant first), so `y < p` unless the top three
/// words are at their maximum and the low word is at least `ffff..ffed`.
#[kernel]
pub fn canonical_y(words: [b64; 4]) -> bool {
    // Top word without the sign bit.
    let high = words[3] & b64(0x7fff_ffff_ffff_ffff);
    high < b64(0x7fff_ffff_ffff_ffff)
        || (high == b64(0x7fff_ffff_ffff_ffff)
            && words[2] == b64(0xffff_ffff_ffff_ffff)
            && words[1] == b64(0xffff_ffff_ffff_ffff)
            && words[0] < b64(0xffff_ffff_ffff_ffed))
}

/// The lowest non-empty slot whose key encoding equals `key`, if any (slot
/// 0 otherwise). Two copies because a kernel's loop bound must be a literal.
#[cfg(not(feature = "key-slots-16"))]
#[kernel]
fn find_slot(key: [b64; 4], enc: [[b64; 4]; KEY_SLOTS], state: [b2; KEY_SLOTS]) -> (bool, KeySlot) {
    let mut found = false;
    let mut slot: KeySlot = bits(0);
    for k in 0..8 {
        if !found && (state[k] != b2(KEY_EMPTY)) && words_equal(key, enc[k]) {
            found = true;
            slot = bits(k as u128);
        }
    }
    (found, slot)
}

/// The lowest non-empty slot whose key encoding equals `key` (16 slots).
#[cfg(feature = "key-slots-16")]
#[kernel]
fn find_slot(key: [b64; 4], enc: [[b64; 4]; KEY_SLOTS], state: [b2; KEY_SLOTS]) -> (bool, KeySlot) {
    let mut found = false;
    let mut slot: KeySlot = bits(0);
    for k in 0..16 {
        if !found && (state[k] != b2(KEY_EMPTY)) && words_equal(key, enc[k]) {
            found = true;
            slot = bits(k as u128);
        }
    }
    (found, slot)
}

/// Equality of two 256-bit values.
#[kernel]
fn words_equal(a: [b64; 4], b: [b64; 4]) -> bool {
    a[0] == b[0] && a[1] == b[1] && a[2] == b[2] && a[3] == b[3]
}

/// The eight low-order encodings strict verification rejects.
///
/// The list is the one `rhdl_ed25519_fast_core` checks (see
/// `host::small_order_encoding`): the identity (`y = 1`), the point of order
/// 2 (`y = p - 1`), the two order-4 points (`y = 0`, both signs), and the
/// four order-8 points (two `y` values, both signs). The comparison is on
/// the encoding bytes, so it is only meaningful for canonical encodings.
#[kernel]
pub fn small_order_encoding(w: [b64; 4]) -> bool {
    words_equal(w, [b64(1), b64(0), b64(0), b64(0)])
        || words_equal(w, [b64(0x4fd84d3d706a17c7), b64(0x0f67100d760b3cba), b64(0xc6cc392cfa53202a), b64(0x7a03ac9277fdc74e)])
        || words_equal(w, [b64(0), b64(0), b64(0), b64(0x8000_0000_0000_0000)])
        || words_equal(w, [b64(0xb027b2c28f95e826), b64(0xf098eff289f4c345), b64(0x3933c6d305acdfd5), b64(0x05fc536d880238b1)])
        || words_equal(w, [b64(0xffff_ffff_ffff_ffec), b64(0xffff_ffff_ffff_ffff), b64(0xffff_ffff_ffff_ffff), b64(0x7fff_ffff_ffff_ffff)])
        || words_equal(w, [b64(0xb027b2c28f95e826), b64(0xf098eff289f4c345), b64(0x3933c6d305acdfd5), b64(0x85fc536d880238b1)])
        || words_equal(w, [b64(0), b64(0), b64(0), b64(0)])
        || words_equal(w, [b64(0x4fd84d3d706a17c7), b64(0x0f67100d760b3cba), b64(0xc6cc392cfa53202a), b64(0xfa03ac9277fdc74e)])
}

/// `S < l` for four little-endian words, `l = 2^252 +
/// 27742317777372353535851937790883648493` the group order.
#[kernel]
pub fn below_l(s: [b64; 4]) -> bool {
    let l = [b64(0x5812631a5cf5d3ed), b64(0x14def9dea2f79cd6), b64(0), b64(0x1000000000000000)];
    // Lexicographic compare from the most significant word: the first word
    // that differs decides; equal values are not below `l`.
    let mut less = false;
    let mut decided = false;
    for k in 0..4 {
        let w = 3 - k;
        if !decided && s[w] != l[w] {
            decided = true;
            less = s[w] < l[w];
        }
    }
    less
}

/// `y` of an encoding (sign bit cleared) as field digits.
///
/// The result is a canonical field element only if [`canonical_y`] holds.
#[kernel]
pub fn encoding_y(w: [b64; 4]) -> Fe17 {
    let mut y = w;
    y[3] = y[3] & b64(0x7fff_ffff_ffff_ffff);
    Fe17 { d: words_to_digits15(y) }
}

/// Equality of two field elements, digit by digit. Valid because the field
/// units always produce canonical values.
#[kernel]
fn fe_equal(a: Fe17, b: Fe17) -> bool {
    let mut eq = true;
    for k in 0..15 {
        if a.d[k] != b.d[k] {
            eq = false;
        }
    }
    eq
}

/// One limb of `p - v = !v + (2^255 - 18)` mod `2^255`: limb `step` of
/// the result from limb `step` of `v` and the carry out of the limbs below
/// (`c`); the other limbs of `acc` pass through. One 85-bit carry chain.
#[kernel]
pub fn p_minus_step(v: [b85; 3], acc: [b85; 3], c: bool, step: b2) -> ([b85; 3], bool) {
    let mut out = acc;
    let (s0, c0) = add_limb(!v[0], nb_const(true, true), false);
    let (s1, c1) = add_limb(!v[1], nb_const(true, false), c);
    let (s2, c2) = add_limb(!v[2], nb_const(true, false), c);
    let mut carry = c0;
    if step == b2(0) {
        out[0] = s0;
    } else if step == b2(1) {
        out[1] = s1;
        carry = c1;
    } else {
        out[2] = s2;
        carry = c2;
    }
    (out, carry)
}

/// `-v` from the limbs of `p - v` (as `field_neg`: zero stays zero).
#[kernel]
pub fn neg_from(v: Fe17, p_minus_v: [b85; 3]) -> Fe17 {
    if fe_is_zero(v) { v } else { Fe17 { d: digits_from_limbs(p_minus_v) } }
}

/// A canonical field element is zero.
#[kernel]
fn fe_is_zero(a: Fe17) -> bool {
    let mut z = true;
    for k in 0..15 {
        if a.d[k] != b17(0) {
            z = false;
        }
    }
    z
}

/// Row of digit `i`'s first entry pair within a key's 512-row table.
///
/// Rows hold two entries each. Radix-32 digits 0..39 have 16 entries (8
/// rows), so digit `i` starts at row `8i`; radix-16 digits 39..54 have 8
/// entries (4 rows) and start after the 312 rows of the radix-32 digits.
#[kernel]
pub fn digit_row(i: b6) -> b9 {
    if i < b6(39) {
        i.resize::<9>() << 3
    } else {
        b9(312) + ((i - b6(39)).resize::<9>() << 2)
    }
}

/// `(entry, coordinate)` of a digit-table output index `3e + c`.
///
/// `e` is the 0-based entry (the table point `(e + 1) * Q`), `c` the Niels
/// coordinate (0 `y+x`, 1 `y-x`, 2 `2dxy`). Computed by comparing `index`
/// with all 48 constant values rather than dividing by 3.
#[kernel]
pub fn split_entry(index: b6) -> (b5, b2) {
    let mut e = b5(0);
    let mut c = b2(0);
    for entry in 0..16 {
        for coord in 0..3 {
            if index == b6((3 * entry + coord) as u128) {
                e = b5(entry as u128);
                c = b2(coord as u128);
            }
        }
    }
    (e, c)
}

/// Machine inputs that do nothing: no start, no digits, no input or table
/// write, output not accepted. `filler` is the value carried by the idle
/// input write.
#[kernel]
fn idle_machine_in(filler: Fe17) -> MachineIn<5> {
    MachineIn::<5> {
        start: StartReq::<5> { valid: false, ctx: b5(0), tag: b16(0), key: bits(0), entry: b12(0) },
        digits: DigitsWrite::<5> { valid: false, ctx: b5(0), digits: [b6(0); 128] },
        input: InputWrite::<5> { valid: false, ctx: b5(0), index: b7(0), value: filler },
        table: idle_table_write(),
        out_accept: false,
    }
}

/// Key and base-point digits in the machine's 128 slots.
///
/// The verify program reads `S`'s digits (base-point table) from slots
/// 0..64 and `k`'s digits (key table) from slots 64..128.
#[kernel]
pub fn join_digits(s: [b6; 64], k: [b6; 64]) -> [b6; 128] {
    let mut out = [b6(0); 128];
    for j in 0..64 {
        out[j] = s[j];
        out[64 + j] = k[j];
    }
    out
}

/// State transition for [`KnownKeyVerifyCore`].
///
/// The sections below run every cycle, in dataflow order: free contexts, SHA
/// availability, admission, SHA -> scalar, scalar -> verify machine, verify
/// machine -> compress, compare, aux machine (slow path and key loading),
/// and the result register. A stage takes new work only when it is empty or
/// hands its content on in the same cycle.
#[kernel]
pub fn known_key_verify_kernel<const NV: usize, const NSHA: usize, const NSC: usize, const SH: usize>(
    _cr: ClockReset,
    i: VerifyIn,
    q: Q<NV, NSHA, NSC, SH>,
) -> (VerifyOut, D<NV, NSHA, NSC, SH>) {
    let r = q.regs;
    let mut n = r;
    let mut d = D::<NV, NSHA, NSC, SH>::dont_care();
    let zero = Fe17 { d: [b17(0); 15] };
    // The scalar 1 in 15 digits: multiplier Y = 1 makes the scalar ring
    // compute k = H mod l.
    let one15 = {
        let mut o = [b17(0); 15];
        o[0] = b17(1);
        o
    };
    // Default memory ports: no writes, read address 0. Stages below override
    // the read address when they need data next cycle. The free list always
    // reads at its read pointer.
    d.mems.r_enc = ram_read::<[b64; 4], 11>(b11(0), q.mems.r_enc);
    d.mems.s_words = ram_read::<[b64; 4], 11>(b11(0), q.mems.s_words);
    d.mems.meta = ram_read::<Meta, 11>(b11(0), q.mems.meta);
    d.mems.free_list = ram_read::<b11, 11>(r.free_rd, q.mems.free_list);
    d.mems.bases = ram_read::<Fe17, 8>(b8(0), q.mems.bases);
    for k in 0..NV {
        d.vm[k] = idle_machine_in(zero);
    }
    d.aux = idle_machine_in(zero);

    // ================= Free contexts =================
    // A one-entry prefetch (`free_head`) in front of the free-list RAM, so
    // admission can take a context without waiting for a read.
    // `free_count` collects this cycle's pops and pushes.
    let mut free_count = r.free_count;
    if r.free_fetching {
        n.free_head = q.mems.free_list;
        n.free_head_valid = true;
        n.free_fetching = false;
    }
    if !r.free_head_valid && !r.free_fetching && r.free_count != b12(0) {
        n.free_fetching = true;
        n.free_rd = r.free_rd + b11(1);
        free_count = free_count - b12(1);
    }
    // Contexts not in the RAM, the head or the read under way are in use.
    let in_flight = b12(2048) - r.free_count - if r.free_head_valid { b12(1) } else { b12(0) }
        - if r.free_fetching { b12(1) } else { b12(0) };
    // Compress credits after this cycle's admission; passed to the compress
    // stage, then updated with its spends and returns.
    let mut credits = r.credits;

    // ================= SHA availability =================
    // `sha_any`: some SHA resource can take a job; `sha_pick`: which ring,
    // or `NSHA` for the shared port. With SH = 1 the rings are never picked.
    let mut sha_any = false;
    let mut sha_pick = b8(0);
    let shared = SH == 1;
    for k in 0..NSHA {
        if q.sha[k].ready && !sha_any && !shared {
            sha_any = true;
            sha_pick = b8(k as u128);
        }
    }
    if shared && i.sha_ext.ready {
        sha_any = true;
        sha_pick = b8(NSHA as u128);
    }
    let mut sha_job = ShaJob { valid: false, tag: b16(0), block: [b64(0); 16], second_pad: false };

    // ================= Admission =================
    // Two steps. Accept: the request's key is matched and its immediate
    // result and facts are decided, into the admission register (`adm_*`).
    // Admit: the registered request takes a context, a credit and a SHA
    // resource (or reports its immediate result). Matching and admitting in
    // the same cycle ran eight 256-bit key compares into the memory writes
    // and SHA job: −2.13 ns in the routed L200 verifier (2026-10-19).
    let req = i.request;
    let in_r = [req.signature.words[0], req.signature.words[1], req.signature.words[2], req.signature.words[3]];
    let in_s = [req.signature.words[4], req.signature.words[5], req.signature.words[6], req.signature.words[7]];
    // Find the slot holding exactly this key encoding (the lowest one if a
    // key was loaded twice). Empty slots never match.
    let (slot_found, slot) = find_slot(req.public_key.words, r.key_enc, r.key_state);
    // Meaningful only when `slot_found`; slot 0 otherwise.
    let slot_state = r.key_state[slot];
    // No request can reach a LOADING slot: requests are accepted only with
    // no key load running or offered, and a key load starts only with no
    // request held or in flight.
    // Immediate results: unknown key, invalid key, non-canonical R.
    // UNKNOWN_KEY is outside the strict priority list and is checked first;
    // BAD_PUBLIC_KEY before BAD_R follows the strict priority.
    let in_error = if !slot_found {
        b8(VERIFY_UNKNOWN_KEY)
    } else if slot_state == b2(KEY_INVALID) {
        b8(VERIFY_BAD_PUBLIC_KEY)
    } else if !canonical_y(in_r) {
        b8(VERIFY_BAD_R)
    } else {
        b8(VERIFY_OK)
    };
    // The held request.
    let req = r.adm_req;
    let r_words = [req.signature.words[0], req.signature.words[1], req.signature.words[2], req.signature.words[3]];
    let s_words = [req.signature.words[4], req.signature.words[5], req.signature.words[6], req.signature.words[7]];
    let immediate_error = r.adm_error;
    let immediate = immediate_error != b8(VERIFY_OK);
    // It is admitted when it is rejected outright, or when a context, a
    // credit and a SHA resource are all available.
    let admit = r.adm_valid && (immediate || (r.free_head_valid && r.credits != b12(0) && sha_any));
    // `ready` never depends on the offered request (ready/valid): the
    // register must be empty or emptying, and no key load may be running or
    // offered this cycle.
    let ready = (!r.adm_valid || admit) && r.kl_state == b2(KL_IDLE) && !i.load_key_valid;
    if admit {
        n.adm_valid = false;
    }
    if i.valid && ready {
        n.adm_valid = true;
        n.adm_req = i.request;
        n.adm_error = in_error;
        n.adm_meta = Meta {
            tag: i.request.tag,
            slot: slot,
            a_small: r.key_small[slot],
            r_small: small_order_encoding(in_r),
            canonical_s: below_l(in_s),
        };
    }
    let accepted = admit;
    // Someone wrote the result register this cycle. Priority: admission
    // rejects, then the compare, then the slow-path decode; the later ones
    // wait a cycle.
    let mut fin_taken = false;
    if accepted && immediate {
        // Rejected at admission: no context, no credit, no hashing.
        n.fin = FinStage { valid: true, free_ctx: false, ctx: b11(0), tag: req.tag, error: immediate_error };
        fin_taken = true;
    }
    if accepted && !immediate {
        // Take the prefetched context and one compress credit (the point
        // this request will send), and record R, S and the facts the final
        // error decision needs.
        let ctx = r.free_head;
        n.free_head_valid = false;
        credits = credits - b12(1);
        d.mems.r_enc.write = RamWrite::<[b64; 4], 11> { addr: ctx, value: r_words, enable: true };
        d.mems.s_words.write = RamWrite::<[b64; 4], 11> { addr: ctx, value: s_words, enable: true };
        d.mems.meta.write = RamWrite::<Meta, 11> { addr: ctx, value: r.adm_meta, enable: true };
        // Tags carry the context and the key slot through the rings.
        // The challenge R || A || M is 128 bytes: one full block, then the
        // constant padding block (`second_pad`).
        let tag = (r.adm_meta.slot.resize::<16>() << 11) | ctx.resize::<16>();
        sha_job = ShaJob { valid: true, tag: tag, block: challenge_block(r_words, req.public_key.words, req.message), second_pad: challenge_second_pad() };
    }
    // The job goes to exactly one SHA resource: the picked ring, or the
    // shared port when `sha_pick == NSHA`. Results are taken further down.
    let mut sha_ext = ShaRingIn {
        job: ShaJob { valid: sha_job.valid && shared && sha_pick == b8(NSHA as u128), tag: sha_job.tag, block: sha_job.block, second_pad: sha_job.second_pad },
        out_ready: false,
    };
    for k in 0..NSHA {
        d.sha[k] = ShaRingIn {
            job: ShaJob { valid: sha_job.valid && b8(k as u128) == sha_pick, tag: sha_job.tag, block: sha_job.block, second_pad: sha_job.second_pad },
            out_ready: false,
        };
    }

    // ================= SHA done -> scalar stage =================
    // First hand the held digest to a free scalar ring: k = H * 1 + 0 mod l.
    let mut sc_any = false;
    let mut sc_pick = b8(0);
    for k in 0..NSC {
        if q.sc[k].ready && !sc_any {
            sc_any = true;
            sc_pick = b8(k as u128);
        }
    }
    let sc_dispatch = r.sc.valid && sc_any;
    for k in 0..NSC {
        d.sc[k] = ScalarRingIn {
            job: ScalarJob { valid: sc_dispatch && b8(k as u128) == sc_pick, tag: r.sc.tag, x: r.sc.x, y: one15, z: [b17(0); 15] },
            out_ready: false,
        };
    }
    if sc_dispatch {
        n.sc.valid = false;
    }
    // The stage can take a digest if it is empty or empties now. One digest
    // per cycle, lowest-numbered ring first; the shared port when SH = 1.
    let sc_free = !r.sc.valid || sc_dispatch;
    let mut sha_taken = false;
    for k in 0..NSHA {
        let done = q.sha[k].done;
        if done.valid && sc_free && !sha_taken {
            sha_taken = true;
            d.sha[k].out_ready = true;
            n.sc = ScStage { valid: true, tag: done.tag, x: words_to_digits31(digest_le(done.state)) };
        }
    }
    let ext_done: ShaDone = i.sha_ext.done;
    if shared && ext_done.valid && sc_free && !sha_taken {
        sha_ext.out_ready = true;
        n.sc = ScStage { valid: true, tag: ext_done.tag, x: words_to_digits31(digest_le(ext_done.state)) };
    }

    // ================= Scalar done -> verify-machine stage =================
    // The stage reads S and the slot, then dispatches to a free context.
    // Cycle 1 (below, at the take): k is recoded and S's read is issued.
    // Cycle 2 (here): S arrives from the RAM and is recoded.
    if r.vm.valid && !r.vm.has_mem {
        n.vm.s = recode_mixed(words_to_digits15(q.mems.s_words));
        n.vm.has_mem = true;
    }
    let (vm_any, vm_pick) = pick_machine::<NV>(r.vm_free, r.vm_has_free);
    let vm_dispatch = r.vm.valid && r.vm.has_mem && vm_any;
    for k in 0..NV {
        // Lowest free context of machine k.
        let mask = r.vm_free[k];
        let mut c = b5(0);
        let mut found = false;
        for j in 0..32 {
            if !found && ((mask >> (j as u128)) & b32(1)) != b32(0) {
                found = true;
                c = b5(j as u128);
            }
        }
        // Start the verify program (entry 0) with the digits loaded in the
        // same cycle. The machine's tag is the context; `key` selects the
        // slot's key table.
        let go = vm_dispatch && b8(k as u128) == vm_pick;
        d.vm[k].start = StartReq::<5> { valid: go, ctx: c, tag: r.vm.ctx.resize::<16>(), key: r.vm.slot, entry: b12(0) };
        d.vm[k].digits = DigitsWrite::<5> { valid: go, ctx: c, digits: join_digits(r.vm.s, r.vm.k) };
        if go {
            n.vm_free[k] = mask & !(b32(1) << (c.resize::<32>()));
        }
        // A retiring context is free again (its outputs have been emitted).
        if q.vm[k].done {
            n.vm_free[k] = n.vm_free[k] | (b32(1) << (q.vm[k].done_ctx.resize::<32>()));
        }
        n.vm_has_free[k] = n.vm_free[k] != b32(0);
    }
    if vm_dispatch {
        n.vm.valid = false;
    }
    // Take one scalar result per cycle into the stage when it is free.
    let vm_stage_free = !r.vm.valid || vm_dispatch;
    let mut sc_taken = false;
    for k in 0..NSC {
        let done = q.sc[k].done;
        if done.valid && vm_stage_free && !sc_taken {
            sc_taken = true;
            d.sc[k].out_ready = true;
            // Tag bits 0..11 are the context, bits 11..14 the key slot.
            let ctx = done.tag.resize::<11>();
            d.mems.s_words.read_addr = ctx;
            n.vm = VmStage { valid: true, has_mem: false, ctx: ctx, k: recode_mixed(done.s), s: [b6(0); 64], slot: (done.tag >> 11).resize() };
        }
    }

    // ================= Verify-machine outputs -> compress =================
    // One coordinate per two cycles at most, first machine first, picked
    // from registered offers into the registered `pt`. X opens a batch slot,
    // so it is picked only while `accept_x` holds; Y and Z are always taken.
    // The credit taken at admission guarantees the slot exists eventually.
    // The machine is told only when the stage takes `pt` (`pt_taken` is the
    // stage's own take condition, from its registered `accept_x`). An X
    // picked while `accept_x` held can still be refused the next cycle (a
    // partial-batch flush closes the filling context); it is then dropped
    // from `pt` and picked again later, the machine still holding it.
    // Why registered: picking straight from the machines chained the offers
    // of all machines into the stage's loader. With 24 machines spread over
    // the die that was the routed verifier's worst path (-5.76 ns,
    // 2026-10-18).
    let accept_x = q.cps.accept_x;
    let pt_taken = r.pt.valid && (r.pt.index != b2(0) || accept_x);
    let mut point = PointIn { valid: false, tag: b16(0), index: b2(0), value: zero };
    let mut vm_out_taken = r.pt.valid;
    for k in 0..NV {
        let o = q.vm[k].out;
        let accepted = r.pt_src[k] && pt_taken;
        if accepted {
            d.vm[k].out_accept = true;
        }
        let grant = r.out_req[k] && !vm_out_taken && (!r.out_req_x[k] || accept_x);
        if grant {
            vm_out_taken = true;
            point = PointIn { valid: true, tag: o.tag, index: o.index.resize::<2>(), value: o.value };
        }
        n.pt_src[k] = grant;
        n.out_req[k] = o.valid && !grant && !accepted;
        n.out_req_x[k] = o.index == b6(0);
    }
    n.pt = point;

    // ================= Compare =================
    // A compare stage reads R and the request facts, then either retires
    // (point encodes to R) or hands the context to the slow path.
    // The read for r.cmp.ctx was issued last cycle (on take, or while waiting).
    let mut cmp_done = false;
    if r.cmp.valid {
        let m = q.mems.meta;
        if words_equal(r.cmp.enc, q.mems.r_enc) {
            // Match: R is a valid point encoding (the compress stage only
            // produces those), so BAD_R cannot apply. Report the first
            // admission fact, in strict priority order, or OK.
            if !fin_taken {
                let error = if m.a_small {
                    b8(VERIFY_SMALL_PUBLIC_KEY)
                } else if m.r_small {
                    b8(VERIFY_SMALL_R)
                } else if !m.canonical_s {
                    b8(VERIFY_NONCANONICAL_S)
                } else {
                    b8(VERIFY_OK)
                };
                n.fin = FinStage { valid: true, free_ctx: true, ctx: r.cmp.ctx, tag: m.tag, error: error };
                fin_taken = true;
                cmp_done = true;
            }
        } else if !r.slow_valid {
            // Mismatch: BAD_R or EQUATION_FAILURE, which only an R decode
            // can tell apart. Park the request in the one-entry slow
            // register; the context stays in use until the decode reports.
            n.slow_valid = true;
            n.slow_ctx = r.cmp.ctx;
            n.slow_r = q.mems.r_enc;
            n.slow_meta = m;
            cmp_done = true;
        }
        // Still waiting (result register taken, or slow register full):
        // re-read so the data is valid again next cycle.
        if !cmp_done {
            d.mems.r_enc.read_addr = r.cmp.ctx;
            d.mems.meta.read_addr = r.cmp.ctx;
        }
    }
    if cmp_done {
        n.cmp.valid = false;
    }
    // Take an encoded point when the compare register is empty, issuing the
    // reads of R and the facts it will compare against next cycle.
    // Why only when empty (not also when it retires this cycle): retiring
    // depends on `fin_taken`, i.e. on the admission key match, and
    // `take_done` is the compress stage's `out_ready`, which fans out over
    // its output stage. That chain was the routed 6,0,1,1 worst path
    // (-0.190 ns, 2026-10-18). A point arrives every ~20 cycles, so the
    // extra cycle of a back-to-back take is not visible in throughput.
    let done = q.cps.done;
    let take_done = done.valid && !r.cmp.valid;
    if take_done {
        let ctx = done.tag.resize::<11>();
        d.mems.r_enc.read_addr = ctx;
        d.mems.meta.read_addr = ctx;
        n.cmp = CmpStage { valid: true, ctx: ctx, enc: done.enc };
    }
    // The stage sees the registered credits and keeps a margin for this
    // cycle's admission and the next, so it never flushes a partial batch
    // with credits already promised to a request.
    d.cps = CompressIn { point: r.pt, credits: r.credits, out_ready: take_done };
    credits = credits - q.cps.credit_spend + q.cps.credit_return;
    n.credits = credits;
    // Invariant: credits stay within the compress pool; an underflow would
    // wrap the 12-bit count above it. Sticky.
    if credits > b12(COMPRESS_SLOTS as u128) {
        n.error = true;
    }

    // ================= Aux machine =================
    // Context 0 decodes (a key or an R), context 1 computes table bases,
    // contexts 2..32 build digit tables.
    let mut aux_in = idle_machine_in(zero);
    // The aux machine has one input-write port; the first user this cycle
    // wins, in order: decode feed, bases feed, digit feed.
    let mut aux_input_used = false;
    // Slow path: start an R decode when the collector is free.
    // (Not between the key decode and the bases job, which still needs -A.)
    if r.slow_valid
        && r.dec_owner == b2(DEC_NONE)
        && !r.dec_running
        && (r.kl_state == b2(KL_IDLE) || r.kl_state == b2(KL_DIGITS))
    {
        n.dec_owner = b2(DEC_R);
        n.dec_ctx = r.slow_ctx;
        n.dec_y = encoding_y(r.slow_r);
        n.dec_sign = (r.slow_r[3] >> 63) != b64(0);
        n.dec_meta = r.slow_meta;
        n.dec_feed = true;
        n.dec_count = b4(0);
        n.slow_valid = false;
    }
    // Key load: start.
    // Needs the decode collector, so no R decode may be running or waiting.
    // The slot's encoding, state and low-order flag are set at once; the
    // slot is LOADING until the last digit job retires.
    // It also waits until no verification is in flight: loading rewrites the
    // slot's key tables, which a request in flight may still be reading.
    let load_key_ready =
        r.kl_state == b2(KL_IDLE) && r.dec_owner == b2(DEC_NONE) && !r.dec_running && !r.slow_valid && in_flight == b12(0) && !r.adm_valid;
    // A public key whose y is not below p is invalid (strict decoding):
    // marked at once, without running the decode program.
    let key_canonical = canonical_y(i.load_key.words);
    if i.load_key_valid && load_key_ready && !key_canonical {
        n.key_enc[i.load_slot] = i.load_key.words;
        n.key_state[i.load_slot] = b2(KEY_INVALID);
        n.key_small[i.load_slot] = false;
    }
    if i.load_key_valid && load_key_ready && key_canonical {
        n.kl_state = b2(KL_DECODE);
        n.kl_slot = i.load_slot;
        n.key_enc[i.load_slot] = i.load_key.words;
        n.key_state[i.load_slot] = b2(KEY_LOADING);
        n.key_small[i.load_slot] = small_order_encoding(i.load_key.words);
        n.dec_owner = b2(DEC_KEY);
        n.dec_y = encoding_y(i.load_key.words);
        n.dec_sign = (i.load_key.words[3] >> 63) != b64(0);
        n.dec_feed = true;
        n.dec_count = b4(0);
    }
    // Feed the decode input (y) and start the decode program on context 0.
    // The input write and the start go out in the same cycle.
    if r.dec_feed && !aux_input_used {
        aux_in.input = InputWrite::<5> { valid: true, ctx: b5(0), index: b7(0), value: r.dec_y };
        aux_in.start = StartReq::<5> { valid: true, ctx: b5(0), tag: b16(AUX_DECODE) << 8, key: bits(0), entry: r.aux_entry[0] };
        aux_input_used = true;
        n.dec_feed = false;
        n.dec_running = true;
    }
    // Key load: bases job on context 1 once the decode has produced -A.
    if r.kl_state == b2(KL_BASES) && !r.kl_issued && !aux_input_used {
        // Inputs 0 and 1 (x of -A, y) are written over two cycles.
        let value = if r.kl_feed_step == b3(0) { r.dec_vals[0] } else { r.dec_y };
        aux_in.input = InputWrite::<5> { valid: true, ctx: b5(1), index: r.kl_feed_step.resize::<7>(), value: value };
        aux_input_used = true;
        if r.kl_feed_step == b3(1) {
            aux_in.start = StartReq::<5> { valid: true, ctx: b5(1), tag: b16(AUX_BASES) << 8, key: bits(0), entry: r.aux_entry[1] };
            n.kl_issued = true;
            n.kl_feed_step = b3(0);
            n.kl_count = b8(0);
        } else {
            n.kl_feed_step = r.kl_feed_step + b3(1);
        }
    }
    // Key load: digit jobs, inputs read from the bases memory (1-cycle latency).
    // Per digit, five steps: step 0 picks a free context in 2..32 and reads
    // base word 0; steps 1..4 write input `step - 1` (X, Y, Z, T of the
    // digit's base, read the cycle before) while reading the next word; step
    // 4 also starts the job with the 16- or 8-entry table program.
    // While a digit's feed is between steps 1 and 4 its next input is the
    // bases word read last cycle. If the feed pauses this cycle (another
    // user took the input port), re-read that same word so it is still on
    // the read port when the feed resumes.
    if r.kl_state == b2(KL_DIGITS) && r.kl_feed_step != b3(0) {
        d.mems.bases.read_addr = (r.kl_feed_digit.resize::<8>() << 2) + (r.kl_feed_step - b3(1)).resize::<8>();
    }
    if r.kl_state == b2(KL_DIGITS) && r.kl_feed_digit != b6(54) && !aux_input_used {
        // After step 0 the context chosen then is kept.
        let mut free_ctx = r.kl_feed_ctx;
        let mut found = r.kl_feed_step != b3(0);
        if !found {
            for j in 2..32 {
                if !found && ((r.aux_busy >> (j as u128)) & b32(1)) == b32(0) {
                    found = true;
                    free_ctx = b5(j as u128);
                }
            }
        }
        n.kl_feed_ctx = free_ctx;
        // With no free context at step 0, the feed retries next cycle.
        if found {
            let step = r.kl_feed_step;
            // Word `step` of base `digit` (the step-4 read is not used).
            let base_addr = (r.kl_feed_digit.resize::<8>() << 2) + step.resize::<8>();
            d.mems.bases.read_addr = base_addr;
            if step != b3(0) {
                aux_in.input = InputWrite::<5> { valid: true, ctx: free_ctx, index: (step - b3(1)).resize::<7>(), value: q.mems.bases };
                aux_input_used = true;
            }
            if step == b3(4) {
                // Radix-32 digits get the 16-entry table, radix-16 the 8-entry
                // one. The tag carries the digit to route the outputs.
                let entry = if r.kl_feed_digit < b6(39) { r.aux_entry[2] } else { r.aux_entry[3] };
                aux_in.start = StartReq::<5> {
                    valid: true,
                    ctx: free_ctx,
                    tag: (b16(AUX_DIGIT) << 8) | r.kl_feed_digit.resize::<16>(),
                    key: bits(0),
                    entry: entry,
                };
                n.aux_busy = r.aux_busy | (b32(1) << (free_ctx.resize::<32>()));
                n.kl_feed_digit = r.kl_feed_digit + b6(1);
                n.kl_feed_step = b3(0);
            } else {
                n.kl_feed_step = step + b3(1);
            }
        }
    }
    // Aux outputs: always accepted.
    // Every consumer below can take a value each cycle, so the aux machine
    // never waits on the verification path.
    let ao = q.aux.out;
    let mut table_write = idle_table_write();
    if ao.valid {
        aux_in.out_accept = true;
        let kind = ao.tag >> 8;
        if kind == b16(AUX_DECODE) {
            // Decode outputs 0..4 go to the collector by index.
            n.dec_vals[ao.index.resize::<2>()] = ao.value;
            n.dec_count = r.dec_count + b4(1);
        } else if kind == b16(AUX_BASES) {
            // The bases program emits X, Y, Z, T per base in digit order, so
            // a running count is the address `4i + c`.
            d.mems.bases.write = RamWrite::<Fe17, 8> { addr: r.kl_count, value: ao.value, enable: true };
            n.kl_count = r.kl_count + b8(1);
        } else {
            // Digit-table output: entry e, coordinate c of digit `digit`.
            // Entries pair up by row: entry e sits in row `digit_row + e / 2`,
            // in the even image if e is even (multiple e + 1 odd) and in the
            // odd image otherwise.
            let digit = ao.tag.resize::<6>();
            let (e, c) = split_entry(ao.index);
            let row: Bits<KEY_TABLE_BITS> = ((r.kl_slot.resize::<16>() << 9) | (digit_row(digit) + (e >> 1).resize::<9>()).resize::<16>()).resize();
            table_write = TableWrite { valid: true, row: row, odd: (e & b5(1)) != b5(0), coord: c, value: ao.value };
        }
    }
    // Every verify machine holds its own copy of the key tables, so each
    // table write goes to all of them, one machine per cycle (`tbl`).
    for k in 0..NV {
        d.vm[k].table = r.tbl[k];
    }
    n.tbl[0] = table_write;
    for k in 1..NV {
        n.tbl[k] = r.tbl[k - 1];
    }
    // Key load: finish once the last table writes have drained.
    if r.kl_drain != b7(0) {
        n.kl_drain = r.kl_drain - b7(1);
        if r.kl_drain == b7(1) {
            n.key_state[r.kl_slot] = b2(KEY_READY);
            n.kl_state = b2(KL_IDLE);
        }
    }
    // Aux retirements.
    // A decode's result is complete when all four outputs have arrived
    // (`dec_count`); its retirement frees aux context 0 for the next decode.
    if q.aux.done {
        let kind = q.aux.done_tag >> 8;
        if kind == b16(AUX_DECODE) {
            n.dec_running = false;
        }
        if kind == b16(AUX_DIGIT) {
            // `n.aux_busy` may already include a start from this cycle.
            n.aux_busy = n.aux_busy & !(b32(1) << (q.aux.done_ctx.resize::<32>()));
            let done_digits = r.kl_digits_done + b6(1);
            n.kl_digits_done = done_digits;
            // A job's table writes precede its retirement, so the slot's
            // last write is in `tbl` here. It reaches machine NV - 1 after NV
            // more cycles and its memory one later.
            if done_digits == b6(54) {
                n.kl_drain = b7(NV as u128) + b7(2);
            }
        } else if kind == b16(AUX_BASES) {
            // All 216 bases are in memory: start feeding digit jobs.
            n.kl_state = b2(KL_DIGITS);
            n.kl_feed_digit = b6(0);
            n.kl_feed_step = b3(0);
            n.kl_digits_done = b6(0);
        }
    }
    // Decode result.
    // RFC 8032 decoding: x^2 = u / v with u = y^2 - 1, v = d y^2 + 1. The
    // program gives a candidate x with v x^2 = +-u; the choice is made here
    // so the program has no branches.
    // Steps 4-6 negate x, x sqrt(-1) and u one limb per cycle; step 7
    // compares and step 8 selects.
    let neg_src = [r.dec_vals[0], r.dec_vals[1], r.dec_vals[3]];
    if r.dec_owner != b2(DEC_NONE) && (r.dec_count == b4(4) || r.dec_count == b4(5) || r.dec_count == b4(6)) {
        let step = (r.dec_count - b4(4)).resize::<2>();
        for j in 0..3 {
            let (o, c) = p_minus_step(limbs_from_digits(neg_src[j].d), r.dec_neg[j], r.dec_neg_c[j], step);
            n.dec_neg[j] = o;
            n.dec_neg_c[j] = c;
        }
        n.dec_count = r.dec_count + b4(1);
    }
    if r.dec_count == b4(7) && r.dec_owner != b2(DEC_NONE) {
        // v x^2 = u: the candidate x is a root.
        n.dec_plain = fe_equal(r.dec_vals[2], r.dec_vals[3]);
        // v x^2 = -u: x * sqrt(-1) is a root.
        n.dec_flipped = fe_equal(r.dec_vals[2], neg_from(r.dec_vals[3], r.dec_neg[2]));
        n.dec_count = b4(8);
    }
    if r.dec_count == b4(8) && r.dec_owner != b2(DEC_NONE) {
        let x = r.dec_vals[0];
        let xi = r.dec_vals[1];
        let plain = r.dec_plain;
        let flipped = r.dec_flipped;
        let root = if plain { x } else { xi };
        let neg_root = if plain { neg_from(x, r.dec_neg[0]) } else { neg_from(xi, r.dec_neg[1]) };
        // Choose the root whose parity matches the encoding's sign bit: the
        // chosen root is `root` or `neg_root`, and -A's x the other one.
        let odd = (root.d[0] & b17(1)) != b17(0);
        let swap = odd != r.dec_sign;
        // Valid: some root exists, and not the encoding x = 0 with the sign
        // bit set (it has no matching root, since -0 = 0 is even).
        let valid = (plain || flipped) && !(fe_is_zero(root) && r.dec_sign);
        if r.dec_owner == b2(DEC_KEY) {
            n.dec_owner = b2(DEC_NONE);
            n.dec_count = b4(0);
            if valid {
                // -A = (-x, y): x of -A goes to dec_vals[0] for the bases job.
                n.dec_vals[0] = if swap { root } else { neg_root };
                n.kl_state = b2(KL_BASES);
                n.kl_issued = false;
                n.kl_feed_step = b3(0);
            } else {
                // The slot keeps the encoding so its requests get
                // BAD_PUBLIC_KEY instead of UNKNOWN_KEY.
                n.key_state[r.kl_slot] = b2(KEY_INVALID);
                n.kl_state = b2(KL_IDLE);
            }
        } else if !fin_taken {
            // R decode: report the slow-path result in strict priority
            // order. If the result register is taken this cycle, the
            // collector keeps the result and tries again next cycle.
            let m = r.dec_meta;
            let error = if !valid {
                b8(VERIFY_BAD_R)
            } else if m.a_small {
                b8(VERIFY_SMALL_PUBLIC_KEY)
            } else if m.r_small {
                b8(VERIFY_SMALL_R)
            } else if !m.canonical_s {
                b8(VERIFY_NONCANONICAL_S)
            } else {
                b8(VERIFY_EQUATION_FAILURE)
            };
            n.fin = FinStage { valid: true, free_ctx: true, ctx: r.dec_ctx, tag: m.tag, error: error };
            fin_taken = true;
            n.dec_owner = b2(DEC_NONE);
            n.dec_count = b4(0);
        }
    }
    d.aux = aux_in;

    // ================= Result =================
    // The register is shown on the ports for one cycle and then cleared,
    // unless a producer wrote a new result this cycle. A reported request
    // returns its context to the free list.
    let mut push_free = false;
    let mut push_ctx = b11(0);
    if r.fin.valid {
        if r.fin.free_ctx {
            push_free = true;
            push_ctx = r.fin.ctx;
        }
        if !fin_taken {
            n.fin.valid = false;
        }
    }
    if push_free {
        d.mems.free_list.write = RamWrite::<b11, 11> { addr: r.free_wr, value: push_ctx, enable: true };
        n.free_wr = r.free_wr + b11(1);
        free_count = free_count + b12(1);
    }
    n.free_count = free_count;
    d.regs = n;
    let out = VerifyOut {
        ready: ready,
        valid: r.fin.valid,
        ok: r.fin.error == b8(VERIFY_OK),
        error: r.fin.error,
        tag: r.fin.tag,
        in_flight: in_flight,
        load_key_ready: load_key_ready,
        key_state: r.key_state,
        internal_error: r.error,
        sha_ext: sha_ext,
    };
    (out, d)
}


/// The verify machine a new request goes to: the one with the most free
/// contexts, the lowest index on a tie (R-11). Fixed priority filled
/// machine 0's 32 contexts before any other, so with the uBFT service's 16
/// tags outstanding a second machine never received work.
#[kernel]
pub fn pick_machine<const NV: usize>(vm_free: [b32; NV], has_free: [bool; NV]) -> (bool, b8) {
    let mut any = false;
    let mut pick = b8(0);
    let mut best = b6(0);
    for k in 0..NV {
        let mut free = b6(0);
        for j in 0..32 {
            if ((vm_free[k] >> (j as u128)) & b32(1)) != b32(0) {
                free = free + b6(1);
            }
        }
        if has_free[k] && (!any || free > best) {
            any = true;
            pick = b8(k as u128);
            best = free;
        }
    }
    (any, pick)
}

#[cfg(test)]
mod pick_machine_tests {
    use super::*;

    /// Sixteen requests in a row with no completions: two machines share
    /// them 8/8, four machines 4/4/4/4 (R-11 regression).
    fn spread<const NV: usize>() -> [usize; NV] {
        let mut free = [b32(0xffff_ffff); NV];
        let mut counts = [0usize; NV];
        for _ in 0..16 {
            let has: [bool; NV] = std::array::from_fn(|k| free[k] != b32(0));
            let (any, pick) = pick_machine::<NV>(free, has);
            assert!(any);
            let k = pick.raw() as usize;
            counts[k] += 1;
            // Take its lowest free context.
            let f = free[k].raw() as u32;
            free[k] = b32((f & (f - 1)) as u128);
        }
        counts
    }

    #[test]
    fn sixteen_tags_reach_every_machine() {
        assert_eq!(spread::<1>(), [16]);
        assert_eq!(spread::<2>(), [8, 8]);
        assert_eq!(spread::<4>(), [4, 4, 4, 4]);
    }

    #[test]
    fn a_full_machine_is_skipped_and_none_free_is_reported() {
        let (any, pick) = pick_machine::<2>([b32(0), b32(1)], [false, true]);
        assert!(any && pick == b8(1));
        let (any, _) = pick_machine::<2>([b32(0), b32(0)], [false, false]);
        assert!(!any);
    }

    #[test]
    fn pick_machine_lowers() {
        use rhdl::core::{CompilationMode, compile_design};
        compile_design::<pick_machine<4>>(CompilationMode::Synchronous).map_err(|e| format!("{e:?}")).unwrap();
    }
}