rhdl_ubft/layout.rs
1//! Where ubft_new keeps its mailboxes in HBM, and what crosses the wire.
2//!
3//! A replica never sends a message in the usual sense. It RDMA-writes a few
4//! bytes into the same address of every peer's HBM (each card has the same
5//! layout), and every replica reads its own HBM to see what peers wrote.
6//! So the layout *is* the protocol. rhdl_ubft keeps ubft_new's layout word
7//! for word, which lets it run on ubft_new's RoCE stack and host unchanged.
8//!
9//! All addresses are 32-bit word indices into `network_ptr` (bytes = 4 ×
10//! word). The mailboxes start after ubft_new's heartbeat, log and deposit
11//! areas, at [`UBFT_BASE_PTR`].
12//!
13//! This table is ubft_new's layout ([`Mailbox::new`]). rhdl_ubft keeps the
14//! regions, their order and the base, and widens three of them so that a
15//! vote binds the value it is for ([`Mailbox::v2`]): a prepare is four words
16//! `(k, m, view, 0)`, and each vote and flag is two words `(tag(view, k),
17//! m)`. The host reads none of the widened regions.
18//!
19//! | Mailbox | Words | Index | What a peer writes there |
20//! |---|---|---|---|
21//! | `prepare` | `G_T · 2` | `slot·2 + {0: k, 1: m}` | the leader's proposal: sequence `k`, value `m` |
22//! | `will_cert` | `G_T · N` | `slot·N + node` | fast path, first vote |
23//! | `will_commit` | `G_T · N` | `slot·N + node` | fast path, second vote |
24//! | `commit` | `G_T · N` | `slot·N + node` | slow path commit vote |
25//! | `certify_valid` | `G_T · N` | `slot·N + node` | "my certify share is in place" |
26//! | `certify` | `G_T · N · 16 · SB` | `(slot·N + node)·16·SB + word` | slow path: the node's share (`SB` beats: Ed25519 1, DSig 26) |
27//! | `vc` | `N · 4` | `node·4 + {view, slot, k, m}` | view change: what the node last saw |
28//! | `vc_valid` | `N` | `node` | view change record present |
29//! | `newview` | 4 | `{view, slot, k, m}` | the new leader's decision |
30//! | `newview_valid` | 1 | | new view present |
31//! | `view` | 32 | | reserved |
32
33use crate::wire::{RDMA_WRITE, RdmaMeta};
34
35/// ubft_new's `NUM_NODES`: the largest cluster its constants are sized for.
36pub const UPSTREAM_NUM_NODES: u32 = 12;
37/// ubft_new's `G_T`: slots in its window.
38pub const UPSTREAM_G_T: u32 = 3;
39
40/// ubft_new's `UBFT_BASE_PTR` (`globals.hpp`): after `NUM_NODES` heartbeat
41/// words, one sync group's replication log (`2 + 11·5` proposals, 250,000
42/// slots, `2·11·5` queue words) and `NUM_NODES` deposit words.
43pub const UBFT_BASE_PTR: u32 = {
44 let n = UPSTREAM_NUM_NODES;
45 let fifo = 5;
46 let slots = 125_000 * 2;
47 let log = (2 + (n - 1) * fifo) + slots + 2 * (n - 1) * fifo;
48 let deposit = n + log;
49 deposit + n
50};
51
52/// The mailbox bases for a cluster of `n` replicas and a window of `g`
53/// slots, computed exactly as ubft_new's `ubft()` does at init.
54#[derive(Clone, Copy, Debug, PartialEq, Eq)]
55pub struct Mailbox {
56 /// Replicas in the cluster (`node_num`).
57 pub n: u32,
58 /// Slots in the window (`G_T`).
59 pub g: u32,
60 /// 64-byte beats per certify share (Ed25519 1, DSig 26).
61 pub sb: u32,
62 /// `prepare`.
63 pub prepare: u32,
64 /// `will_cert`.
65 pub will_cert: u32,
66 /// `will_commit`.
67 pub will_commit: u32,
68 /// `commit`.
69 pub commit: u32,
70 /// `certify_valid`.
71 pub certify_valid: u32,
72 /// `certify`.
73 pub certify: u32,
74 /// `vc`.
75 pub vc: u32,
76 /// `vc_valid`.
77 pub vc_valid: u32,
78 /// `newview`.
79 pub newview: u32,
80 /// `newview_valid`.
81 pub newview_valid: u32,
82 /// `view`.
83 pub view: u32,
84 /// One past the last mailbox word.
85 pub end: u32,
86}
87
88impl Mailbox {
89 /// The layout for `n` replicas, `g` slots and shares of `sb` beats
90 /// (ubft_new: `sb` = 1).
91 pub const fn new(n: u32, g: u32, sb: u32) -> Mailbox {
92 let prepare = UBFT_BASE_PTR;
93 let will_cert = prepare + g * 2;
94 let will_commit = will_cert + g * n;
95 let commit = will_commit + g * n;
96 let certify_valid = commit + g * n;
97 let certify = certify_valid + g * n;
98 let vc = certify + g * n * 16 * sb;
99 let vc_valid = vc + n * 4;
100 let newview = vc_valid + n;
101 let newview_valid = newview + 4;
102 let view = newview_valid + 1;
103 let end = view + 32;
104 Mailbox { n, g, sb, prepare, will_cert, will_commit, commit, certify_valid, certify, vc, vc_valid, newview, newview_valid, view, end }
105 }
106
107 /// rhdl_ubft's layout (v2): the same regions in the same order from the
108 /// same base, but a prepare is `(k, m, view, 0)`, a vote or flag is two
109 /// words `(tag(view, k), m)` so it binds the value, and a share is `sb`
110 /// beats. Records stay four words, `(view, slot | phase << 16, k, m)`.
111 pub const fn v2(n: u32, g: u32, sb: u32) -> Mailbox {
112 let prepare = UBFT_BASE_PTR;
113 let will_cert = prepare + g * 4;
114 let will_commit = will_cert + g * n * 2;
115 let commit = will_commit + g * n * 2;
116 let certify_valid = commit + g * n * 2;
117 let certify = certify_valid + g * n * 2;
118 let vc = certify + g * n * 16 * sb;
119 let vc_valid = vc + n * 4;
120 let newview = vc_valid + n;
121 let newview_valid = newview + 4;
122 let view = newview_valid + 1;
123 let end = view + 32;
124 Mailbox { n, g, sb, prepare, will_cert, will_commit, commit, certify_valid, certify, vc, vc_valid, newview, newview_valid, view, end }
125 }
126
127 /// The mailbox a word belongs to, for traces.
128 pub fn name(&self, word: u32) -> &'static str {
129 let bounds = [
130 (self.prepare, "prepare"),
131 (self.will_cert, "will_cert"),
132 (self.will_commit, "will_commit"),
133 (self.commit, "commit"),
134 (self.certify_valid, "certify_valid"),
135 (self.certify, "certify"),
136 (self.vc, "vc"),
137 (self.vc_valid, "vc_valid"),
138 (self.newview, "newview"),
139 (self.newview_valid, "newview_valid"),
140 (self.view, "view"),
141 (self.end, "other"),
142 ];
143 if word < self.prepare {
144 return "other";
145 }
146 let mut name = "other";
147 for (start, n) in bounds {
148 if word >= start {
149 name = n;
150 }
151 }
152 name
153 }
154
155 /// The RDMA write that puts `len` bytes at `word` of peer `dest`, sent
156 /// by replica `id`. The RoCE stack programs `N − 1` connections per
157 /// replica and none to itself: peer `j` is local queue pair `(N − 1)·id
158 /// + rank(j)`, rank `j` below `id` and `j − 1` above
159 /// (docs/rhdl_ubft-roce.md). `None` for `dest == id`.
160 pub fn write_to(&self, id: u32, dest: u32, word: u32, len: u32) -> Option<RdmaMeta> {
161 if dest == id {
162 return None;
163 }
164 let rank = if dest < id { dest } else { dest - 1 };
165 Some(RdmaMeta { op: RDMA_WRITE, qpn: (self.n - 1) * id + rank, laddr: 0, raddr: u64::from(word) * 4, len })
166 }
167
168 /// The peer a local queue pair of replica `id` reaches, if the RoCE
169 /// stack programmed it.
170 pub fn peer_of(&self, id: u32, qpn: u32) -> Option<u32> {
171 let lo = (self.n - 1) * id;
172 if qpn < lo || qpn >= lo + self.n - 1 {
173 return None;
174 }
175 let rank = qpn - lo;
176 Some(if rank < id { rank } else { rank + 1 })
177 }
178}
179
180#[cfg(test)]
181mod tests {
182 use super::*;
183
184 /// The base is ubft_new's: 250,191 words, byte 1,000,764 (not 64-byte
185 /// aligned, so mailbox fetches start mid-beat).
186 #[test]
187 fn base_matches_upstream() {
188 // protocol.rs uses this value as a literal.
189 assert_eq!(UBFT_BASE_PTR, 250_191);
190 assert_eq!(UBFT_BASE_PTR * 4 % 64, 60);
191 let m = Mailbox::new(3, 3, 1);
192 assert_eq!(m.will_cert - m.prepare, 6);
193 assert_eq!(m.certify - m.certify_valid, 9);
194 assert_eq!(m.vc - m.certify, 3 * 3 * 16);
195 assert_eq!(m.name(m.certify + 5), "certify");
196 assert_eq!(m.name(m.prepare), "prepare");
197 // No connection to self; peers by rank.
198 assert_eq!(m.write_to(1, 1, 0, 4), None);
199 assert_eq!(m.write_to(1, 0, 0, 4).unwrap().qpn, 2);
200 assert_eq!(m.write_to(1, 2, 0, 4).unwrap().qpn, 3);
201 assert_eq!(m.peer_of(1, 3), Some(2));
202 assert_eq!(m.peer_of(1, 4), None);
203 }
204}