From c82bb1102df6c3b1b1efe3e8d32dafd5f8b969da Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Sun, 4 Oct 2026 21:29:24 -0700 Subject: [PATCH 01/24] refactor(pbs): read the bid stream through a generic reader The get_header stream's handshake, TLS config, frame format and read loop move to bid_stream.rs, behind a reader that holds the last frame a parse closure accepts. get_header decodes and validates that frame at the deadline, as before, so the result is unchanged. Another bid stream can use the same reader with its own parse closure. RelayClient::get_header_request becomes get_header_stream_url, which returns the stream URL only for a relay that streams. test_decode_streamed_bid goes: every stream test decodes a streamed bid. --- crates/common/src/pbs/relay.rs | 50 +-- crates/pbs/src/bid_stream.rs | 351 ++++++++++++++++++++ crates/pbs/src/lib.rs | 1 + crates/pbs/src/mev_boost/get_header.rs | 11 +- crates/pbs/src/mev_boost/get_header_ws.rs | 377 ++-------------------- 5 files changed, 395 insertions(+), 395 deletions(-) create mode 100644 crates/pbs/src/bid_stream.rs diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index bb043df08..1a2768557 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -188,24 +188,17 @@ impl RelayClient { ) } - pub fn get_header_request( + /// The get_header stream, `None` when the relay does not stream + pub fn get_header_stream_url( &self, slot: u64, parent_hash: &B256, validator_pubkey: &BlsPublicKey, - ) -> Result { - Ok(match &self.stream_url { - None => { - GetHeaderRequest::Http(self.get_header_url(slot, parent_hash, validator_pubkey)?) - } - Some(base) => { - let mut url = base.clone(); - url.set_path(&format!("{}/{slot}/{parent_hash}/{validator_pubkey}", base.path())); - - self.append_get_params(&mut url); - GetHeaderRequest::Stream(url) - } - }) + ) -> Option { + let mut url = self.stream_url.clone()?; + url.set_path(&format!("{}/{slot}/{parent_hash}/{validator_pubkey}", url.path())); + self.append_get_params(&mut url); + Some(url) } pub fn get_status_url(&self) -> Result { @@ -234,7 +227,7 @@ mod tests { use alloy::primitives::B256; - use super::{GetHeaderRequest, RelayClient, RelayEntry, value_fingerprint}; + use super::{RelayClient, RelayEntry, value_fingerprint}; use crate::{ config::{GetHeaderTransport, RelayConfig, test_env::RELAY_URL}, utils::bls_pubkey_from_hex_unchecked, @@ -326,7 +319,7 @@ mod tests { } #[test] - fn test_get_header_request() { + fn test_get_header_stream_url() { let slot = 0; let parent_hash = B256::ZERO; let validator_pubkey = bls_pubkey_from_hex_unchecked( @@ -338,29 +331,16 @@ mod tests { }"#; let base_config = serde_json::from_str::(relay_config).unwrap(); - // Default transport: plain HTTP endpoint + // Default transport: no stream let relay = RelayClient::new(base_config.clone()).unwrap(); - let GetHeaderRequest::Http(url) = - relay.get_header_request(slot, &parent_hash, &validator_pubkey).unwrap() - else { - panic!("expected http request"); - }; - assert_eq!( - url, - relay.get_header_url(slot, &parent_hash, &validator_pubkey).unwrap(), - "http dispatch must match the plain url builder" - ); + assert!(relay.get_header_stream_url(slot, &parent_hash, &validator_pubkey).is_none()); // Streaming: the relay url over ws, at the fixed stream path, with the // pubkey credentials dropped let mut config = base_config.clone(); config.get_header = GetHeaderTransport::Stream; let relay = RelayClient::new(config).unwrap(); - let GetHeaderRequest::Stream(url) = - relay.get_header_request(slot, &parent_hash, &validator_pubkey).unwrap() - else { - panic!("expected stream request"); - }; + let url = relay.get_header_stream_url(slot, &parent_hash, &validator_pubkey).unwrap(); assert_eq!( url.to_string(), format!( @@ -375,11 +355,7 @@ mod tests { config.get_header = GetHeaderTransport::Stream; config.get_params = Some(HashMap::from([("token".to_string(), "abc".to_string())])); let relay = RelayClient::new(config).unwrap(); - let GetHeaderRequest::Stream(url) = - relay.get_header_request(slot, &parent_hash, &validator_pubkey).unwrap() - else { - panic!("expected stream request"); - }; + let url = relay.get_header_stream_url(slot, &parent_hash, &validator_pubkey).unwrap(); assert_eq!( url.to_string(), format!( diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs new file mode 100644 index 000000000..cd0eb58c8 --- /dev/null +++ b/crates/pbs/src/bid_stream.rs @@ -0,0 +1,351 @@ +//! The websocket bid stream of a relay configured with `get_header = "stream"`. +//! One connection per request, dropped at the deadline. The handshake carries +//! the request, and the relay replies with one binary frame per bid update: +//! +//! ```text +//! u8 message type +//! u8 fork +//! .. SSZ bid +//! ``` + +use std::{ + sync::{Arc, OnceLock}, + time::Duration, +}; + +use axum::http::{HeaderMap, HeaderValue, Request, header::USER_AGENT}; +use cb_common::{ + pbs::{ForkName, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, error::PbsError}, + utils::utcnow_ms, +}; +use futures::StreamExt; +use reqwest::StatusCode; +use rustls::{ClientConfig, RootCertStore, crypto::aws_lc_rs}; +use tokio::time::{Instant, sleep_until, timeout_at}; +use tokio_tungstenite::{ + Connector, connect_async_tls_with_config, + tungstenite::{ + Bytes, Error as WsError, Message, client::IntoClientRequest, protocol::WebSocketConfig, + }, +}; +use tracing::{debug, warn}; +use url::Url; + +use crate::{ + constants::{MAX_SIZE_GET_HEADER_RESPONSE, TIMEOUT_ERROR_STATUS, TRANSPORT_ERROR_STATUS}, + metrics::{ + RELAY_LATENCY, RELAY_STREAM_CONNECT_LATENCY, RELAY_STREAM_INVALID_FRAMES, + RELAY_STREAM_UPDATES, + }, +}; + +/// Frame prefix: message type + fork. +const FRAME_PREFIX_LEN: usize = 2; + +const MSG_BID: u8 = 0x01; + +/// One bid update, its bid still SSZ-encoded +pub(crate) struct Frame { + pub(crate) fork: ForkName, + pub(crate) bid: Bytes, +} + +/// What a stream held when its window closed +pub(crate) struct Held { + /// The last frame the parse closure accepted. None when the relay had no + /// bid. + pub(crate) latest: Option, + /// Frames the parse closure accepted + pub(crate) updates: usize, + pub(crate) connect_latency: Duration, + pub(crate) first_frame_latency: Option, + pub(crate) invalid_frames: usize, +} + +/// The handshake for `url`, with the same headers an HTTP request carries +pub(crate) fn handshake_request( + url: &Url, + relay: &RelayClient, + send_headers: &HeaderMap, + timeout_ms: u64, +) -> Result, PbsError> { + let mut request = url + .as_str() + .into_client_request() + .map_err(|err| PbsError::WebSocketConnect(format!("invalid ws url: {err}")))?; + + let headers = request.headers_mut(); + if let Some(user_agent) = send_headers.get(USER_AGENT) { + headers.insert(USER_AGENT, user_agent.clone()); + } + + for (key, value) in relay.stream_headers() { + headers.insert(key, value.clone()); + } + + headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); + headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); + + Ok(request) +} + +/// Opens the stream and holds the last frame `parse` accepts, until `deadline` +/// or the relay ends the stream. A frame `parse` rejects counts as invalid. On +/// failure, returns the status the failure records under. Time to the first +/// frame is recorded under `endpoint`. +pub(crate) async fn read_bid_stream( + request: Request<()>, + deadline: Instant, + relay: &RelayClient, + endpoint: &str, + mut parse: impl FnMut(Frame) -> Result, +) -> Result, (StatusCode, PbsError)> { + let config = WebSocketConfig::default() + .max_message_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)) + .max_frame_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)); + + let start_request = Instant::now(); + let connect = connect_async_tls_with_config( + request, + Some(config), + true, + Some(Connector::Rustls(tls_config().clone())), + ); + let (mut stream, _) = match timeout_at(deadline, connect).await { + Ok(Ok(connected)) => connected, + Ok(Err(err)) => return Err(connect_failed(&err)), + Err(_) => return Err((TIMEOUT_ERROR_STATUS, PbsError::WebSocketTimeout)), + }; + let connect_latency = start_request.elapsed(); + RELAY_STREAM_CONNECT_LATENCY + .with_label_values(&[relay.id.as_str()]) + .observe(connect_latency.as_secs_f64()); + debug!(relay_id = relay.id.as_ref(), ?connect_latency, "ws connected"); + + let timer = sleep_until(deadline); + tokio::pin!(timer); + + let mut latest = None; + let mut updates = 0usize; + let mut first_frame_latency = None; + let mut invalid_frames = 0usize; + let mut stream_error = None; + + loop { + let message = tokio::select! { + biased; + _ = &mut timer => break, + message = stream.next() => message, + }; + + let payload = match message { + Some(Ok(Message::Binary(payload))) => payload, + Some(Ok(Message::Close(_))) | None => break, + Some(Ok(_)) => continue, + Some(Err(err)) => { + warn!(relay_id = relay.id.as_ref(), %err, "ws stream error"); + stream_error = Some(PbsError::WebSocket(format!("stream error: {err}"))); + break; + } + }; + + match parse_frame(payload).and_then(&mut parse) { + Ok(frame) => { + first_frame_latency.get_or_insert_with(|| start_request.elapsed()); + updates += 1; + latest = Some(frame); + } + Err(err) => { + invalid_frames += 1; + if invalid_frames == 1 { + warn!(relay_id = relay.id.as_ref(), %err, "invalid ws frame, skipping"); + } + } + } + } + + drop(stream); + + RELAY_STREAM_UPDATES.with_label_values(&[relay.id.as_str()]).observe(updates as f64); + if invalid_frames > 0 { + RELAY_STREAM_INVALID_FRAMES + .with_label_values(&[relay.id.as_str()]) + .inc_by(invalid_frames as u64); + } + + if let Some(latency) = first_frame_latency { + RELAY_LATENCY.with_label_values(&[endpoint, &relay.id]).observe(latency.as_secs_f64()); + } else if let Some(err) = stream_error { + return Err((TRANSPORT_ERROR_STATUS, err)); + } + + Ok(Held { latest, updates, connect_latency, first_frame_latency, invalid_frames }) +} + +/// A relay rejecting the handshake puts the reason in the body, which +/// tungstenite's own `Display` drops. The body is only what arrived alongside +/// the headers, so it can be partial or empty. +fn connect_failed(err: &WsError) -> (StatusCode, PbsError) { + let WsError::Http(res) = err else { + return (TRANSPORT_ERROR_STATUS, PbsError::WebSocketConnect(err.to_string())); + }; + + let code = res.status(); + let body = res.body().as_deref().unwrap_or_default(); + let msg = if body.is_empty() { + format!("rejected with {code}") + } else { + format!("rejected with {code}: {}", String::from_utf8_lossy(body)) + }; + + // A 2xx handshake answer is a failed connect, not a delivered bid + let code = if code.is_success() { TRANSPORT_ERROR_STATUS } else { code }; + + (code, PbsError::WebSocketConnect(msg)) +} + +fn fork_from_wire(byte: u8) -> Option { + // TODO @nina: I don't see a point of extending a u8 for supporting older forks + // we could rotate these instead, i.e. 0 becomes Heze, etc + Some(match byte { + 0 => ForkName::Base, + 1 => ForkName::Altair, + 2 => ForkName::Bellatrix, + 3 => ForkName::Capella, + 4 => ForkName::Deneb, + 5 => ForkName::Electra, + 6 => ForkName::Fulu, + 7 => ForkName::Gloas, + _ => return None, + }) +} + +fn parse_frame(payload: Bytes) -> Result { + let &[msg_type, fork_byte] = payload + .first_chunk::() + .ok_or_else(|| PbsError::WebSocket(format!("frame too short: {} bytes", payload.len())))?; + + if msg_type != MSG_BID { + return Err(PbsError::WebSocket(format!("unknown message type: {msg_type}"))); + } + + let fork = fork_from_wire(fork_byte) + .ok_or_else(|| PbsError::WebSocket(format!("unknown fork: {fork_byte}")))?; + + Ok(Frame { fork, bid: payload.slice(FRAME_PREFIX_LEN..) }) +} + +/// One TLS config for every stream connection. Left to tokio-tungstenite it is +/// rebuilt per connect, which reparses the root store and, worse, gives each +/// connection its own session cache: every slot then pays a full handshake +/// instead of a resumed one. The provider is named explicitly because rustls is +/// built with both `ring` and `aws-lc-rs` here, so the default builder needs a +/// process-wide install to pick one. +fn tls_config() -> &'static Arc { + static CONFIG: OnceLock> = OnceLock::new(); + CONFIG.get_or_init(|| { + let mut roots = RootCertStore::empty(); + roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + + Arc::new( + ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .expect("aws-lc-rs supports tls 1.2 and 1.3") + .with_root_certificates(roots) + .with_no_client_auth(), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bid_frame(fork_byte: u8, bid: &[u8]) -> Bytes { + let mut frame = vec![MSG_BID, fork_byte]; + frame.extend_from_slice(bid); + Bytes::from(frame) + } + + #[test] + fn test_fork_from_wire_covers_all_forks() { + let forks = ForkName::list_all(); + + for (byte, fork) in forks.iter().enumerate() { + if *fork <= ForkName::Base { + continue; + } + assert_eq!(fork_from_wire(byte as u8), Some(*fork), "fork {fork} unmapped"); + } + + assert_eq!(fork_from_wire(forks.len() as u8), None); + } + + #[test] + fn test_parse_frame() { + assert!(matches!( + parse_frame(bid_frame(6, &[1, 2, 3])), + Ok(Frame { fork: ForkName::Fulu, bid }) if bid.as_ref() == [1, 2, 3] + )); + + // Empty bid payload is well-formed at this layer, SSZ decoding rejects it + assert!(matches!(parse_frame(bid_frame(6, &[])), Ok(Frame { fork: ForkName::Fulu, .. }))); + + for bad in [ + // Truncated prefix + Bytes::from_static(&[]), + Bytes::from_static(&[MSG_BID]), + // Unknown fork + bid_frame(0xff, &[1]), + // Unknown message type + Bytes::from_static(&[0xff, 6]), + ] { + assert!(matches!(parse_frame(bad), Err(PbsError::WebSocket(_)))); + } + } + + #[test] + fn test_connect_failed_carries_relay_reason() { + let rejected = axum::http::Response::builder() + .status(401) + .body(Some(b"api key not registered".to_vec())) + .unwrap(); + + let (status, err) = connect_failed(&WsError::Http(Box::new(rejected))); + assert_eq!(status, StatusCode::UNAUTHORIZED); + let PbsError::WebSocketConnect(msg) = err else { panic!("wrong outcome") }; + assert!(msg.contains("401"), "{msg}"); + assert!(msg.contains("api key not registered"), "{msg}"); + + // No body to add, and transport failures keep tungstenite's own message + let empty = axum::http::Response::builder().status(404).body(None).unwrap(); + let (status, _) = connect_failed(&WsError::Http(Box::new(empty))); + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, err) = connect_failed(&WsError::ConnectionClosed); + assert_eq!(status, TRANSPORT_ERROR_STATUS); + assert!(matches!(err, PbsError::WebSocketConnect(_))); + } + + // A url pointing at a plain http endpoint answers the handshake 200. That + // is the code the stream series uses for a delivered bid, so a failed + // handshake must never carry it. + #[test] + fn test_connect_failed_never_reports_a_success_code() { + for code in [200u16, 204, 299] { + let answered = axum::http::Response::builder().status(code).body(None).unwrap(); + let (status, err) = connect_failed(&WsError::Http(Box::new(answered))); + assert_eq!( + status, TRANSPORT_ERROR_STATUS, + "handshake answered {code} counted as a served stream" + ); + let PbsError::WebSocketConnect(msg) = err else { panic!("wrong outcome") }; + assert!(msg.contains(&code.to_string()), "{msg}"); + } + + // A relay's own rejection code still reaches the series unchanged + let moved = axum::http::Response::builder().status(302).body(None).unwrap(); + let (status, _) = connect_failed(&WsError::Http(Box::new(moved))); + assert_eq!(status, StatusCode::FOUND); + } +} diff --git a/crates/pbs/src/lib.rs b/crates/pbs/src/lib.rs index 8b4afdcfe..2c775f335 100644 --- a/crates/pbs/src/lib.rs +++ b/crates/pbs/src/lib.rs @@ -1,4 +1,5 @@ mod api; +mod bid_stream; mod constants; mod error; mod metrics; diff --git a/crates/pbs/src/mev_boost/get_header.rs b/crates/pbs/src/mev_boost/get_header.rs index 576ced51e..3427c8f77 100644 --- a/crates/pbs/src/mev_boost/get_header.rs +++ b/crates/pbs/src/mev_boost/get_header.rs @@ -251,8 +251,17 @@ async fn get_header_from_relay( timeout_left_ms: u64, ) -> Result, PbsError> { let params = &request_info.params; + let request = + match relay.get_header_stream_url(params.slot, ¶ms.parent_hash, ¶ms.pubkey) { + Some(url) => GetHeaderRequest::Stream(url), + None => GetHeaderRequest::Http(relay.get_header_url( + params.slot, + ¶ms.parent_hash, + ¶ms.pubkey, + )?), + }; - match relay.get_header_request(params.slot, ¶ms.parent_hash, ¶ms.pubkey)? { + match request { GetHeaderRequest::Stream(url) => { let started = Instant::now(); let err = match get_header_ws(&request_info, &relay, url, timeout_left_ms).await { diff --git a/crates/pbs/src/mev_boost/get_header_ws.rs b/crates/pbs/src/mev_boost/get_header_ws.rs index 3b209ce92..085aac45f 100644 --- a/crates/pbs/src/mev_boost/get_header_ws.rs +++ b/crates/pbs/src/mev_boost/get_header_ws.rs @@ -1,78 +1,24 @@ -//! Streaming get_header over a websocket, for relays configured with +//! get_header over the bid stream, for relays configured with //! `get_header = "stream"`. The stream URL is derived from the relay's own -//! `url`. One connection per get_header call, dropped when the call returns. -//! -//! The request is the handshake itself: slot / parent_hash / pubkey in the -//! path, deadline and timestamp in headers, same data the HTTP request carries. -//! The relay replies with one binary frame per bid update: -//! -//! ```text -//! u8 message type -//! u8 fork -//! .. SSZ SignedBuilderBid -//! ``` +//! `url`. The handshake carries the same data as the HTTP request: slot / +//! parent_hash / pubkey in the path, deadline and timestamp in headers. -use std::{ - sync::{Arc, OnceLock}, - time::Duration, -}; +use std::time::Duration; use alloy::primitives::utils::format_ether; -use axum::http::{HeaderValue, Request, header::USER_AGENT}; -use cb_common::{ - pbs::{ - ForkName, GetHeaderInfo, GetHeaderResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, - RelayClient, error::PbsError, - }, - utils::utcnow_ms, -}; -use futures::StreamExt; +use cb_common::pbs::{GetHeaderInfo, GetHeaderResponse, RelayClient, error::PbsError}; use reqwest::StatusCode; -use rustls::{ClientConfig, RootCertStore, crypto::aws_lc_rs}; -use tokio::time::{Instant, sleep_until, timeout_at}; -use tokio_tungstenite::{ - Connector, connect_async_tls_with_config, - tungstenite::{ - Bytes, Error as WsError, Message, client::IntoClientRequest, protocol::WebSocketConfig, - }, -}; -use tracing::{debug, info, warn}; +use tokio::time::Instant; +use tracing::{debug, info}; use url::Url; -use super::get_header::{RequestInfo, validate_get_header_response}; +use super::get_header::{RequestInfo, decode_ssz_payload, validate_get_header_response}; use crate::{ - constants::{ - GET_HEADER_STREAM_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, TIMEOUT_ERROR_STATUS, - TRANSPORT_ERROR_STATUS, - }, - metrics::{ - RELAY_LATENCY, RELAY_STATUS_CODE, RELAY_STREAM_CONNECT_LATENCY, - RELAY_STREAM_INVALID_FRAMES, RELAY_STREAM_UPDATES, - }, - mev_boost::get_header::decode_ssz_payload, + bid_stream::{Frame, Held, handshake_request, read_bid_stream}, + constants::{GET_HEADER_STREAM_ENDPOINT_TAG, TRANSPORT_ERROR_STATUS}, + metrics::RELAY_STATUS_CODE, }; -/// Frame prefix: message type + fork. -const FRAME_PREFIX_LEN: usize = 2; - -const MSG_BID: u8 = 0x01; - -fn fork_from_wire(byte: u8) -> Option { - // TODO @nina: I don't see a point of extending a u8 for supporting older forks - // we could rotate these instead, i.e. 0 becomes Heze, etc - Some(match byte { - 0 => ForkName::Base, - 1 => ForkName::Altair, - 2 => ForkName::Bellatrix, - 3 => ForkName::Capella, - 4 => ForkName::Deneb, - 5 => ForkName::Electra, - 6 => ForkName::Fulu, - 7 => ForkName::Gloas, - _ => return None, - }) -} - type StreamOutcome = (StatusCode, Result, PbsError>); /// Open a stream to the relay, keep the latest bid until the deadline, then @@ -97,109 +43,23 @@ async fn stream_header( timeout_ms: u64, ) -> StreamOutcome { let deadline = Instant::now() + Duration::from_millis(timeout_ms); - let request = match build_handshake_request(request_info, relay, &url, timeout_ms) { + let request = match handshake_request(&url, relay, &request_info.headers, timeout_ms) { Ok(request) => request, Err(err) => return (TRANSPORT_ERROR_STATUS, Err(err)), }; - let config = WebSocketConfig::default() - .max_message_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)) - .max_frame_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)); - - let start_request = Instant::now(); - let connect = connect_async_tls_with_config( - request, - Some(config), - true, - Some(Connector::Rustls(tls_config().clone())), - ); - let (mut stream, _) = match timeout_at(deadline, connect).await { - Ok(Ok(connected)) => connected, - Ok(Err(err)) => return connect_failed(&err), - Err(_) => { - return (TIMEOUT_ERROR_STATUS, Err(PbsError::WebSocketTimeout)); - } - }; - let connect_latency = start_request.elapsed(); - RELAY_STREAM_CONNECT_LATENCY - .with_label_values(&[relay.id.as_str()]) - .observe(connect_latency.as_secs_f64()); - debug!(relay_id = relay.id.as_ref(), ?connect_latency, "ws connected"); - - let timer = sleep_until(deadline); - tokio::pin!(timer); - - let mut latest: Option<(ForkName, Bytes)> = None; - let mut first_bid_latency: Option = None; - let mut updates = 0usize; - let mut invalid_frames = 0usize; - let mut stream_error = None; - - loop { - let message = tokio::select! { - biased; - _ = &mut timer => break, - message = stream.next() => message, - }; - - let message = match message { - Some(Ok(message)) => message, - Some(Err(err)) => { - warn!(relay_id = relay.id.as_ref(), %err, "ws stream error"); - stream_error = Some(PbsError::WebSocket(format!("stream error: {err}"))); - break; - } - None => break, + let Held { latest, updates, connect_latency, first_frame_latency, invalid_frames } = + match read_bid_stream(request, deadline, relay, GET_HEADER_STREAM_ENDPOINT_TAG, Ok).await { + Ok(held) => held, + Err((status, err)) => return (status, Err(err)), }; - let payload = match message { - Message::Binary(payload) => payload, - Message::Close(_) => break, - _ => continue, - }; - - match parse_frame(payload) { - Ok((fork, bid)) => { - updates += 1; - if first_bid_latency.is_none() { - first_bid_latency = Some(start_request.elapsed()); - } - latest = Some((fork, bid)); - } - Err(err) => { - invalid_frames += 1; - if invalid_frames == 1 { - warn!(relay_id = relay.id.as_ref(), %err, "invalid ws frame, skipping"); - } - } - } - } - - drop(stream); - - RELAY_STREAM_UPDATES.with_label_values(&[relay.id.as_str()]).observe(updates as f64); - if invalid_frames > 0 { - RELAY_STREAM_INVALID_FRAMES - .with_label_values(&[relay.id.as_str()]) - .inc_by(invalid_frames as u64); - } - - let Some((fork, bid_bytes)) = latest else { - if let Some(err) = stream_error { - return (TRANSPORT_ERROR_STATUS, Err(err)); - } - + let Some(Frame { fork, bid }) = latest else { debug!(relay_id = relay.id.as_ref(), ?connect_latency, invalid_frames, "no header"); return (StatusCode::NO_CONTENT, Ok(None)); }; - if let Some(first_bid_latency) = first_bid_latency { - RELAY_LATENCY - .with_label_values(&[GET_HEADER_STREAM_ENDPOINT_TAG, &relay.id]) - .observe(first_bid_latency.as_secs_f64()); - } - - let response = match decode_ssz_payload(&bid_bytes, fork) { + let response = match decode_ssz_payload(&bid, fork) { Ok(response) => response, Err(err) => return (StatusCode::OK, Err(err)), }; @@ -214,9 +74,9 @@ async fn stream_header( info!( relay_id = relay.id.as_ref(), - header_size_bytes = bid_bytes.len(), + header_size_bytes = bid.len(), ?connect_latency, - ?first_bid_latency, + first_bid_latency = ?first_frame_latency, ?validate_latency, version = ?fork, value_eth = format_ether(*response.value()), @@ -228,200 +88,3 @@ async fn stream_header( (StatusCode::OK, Ok(Some(response))) } - -/// A relay rejecting the handshake puts the reason in the body, which -/// tungstenite's own `Display` drops. The body is only what arrived alongside -/// the headers, so it can be partial or empty. -fn connect_failed(err: &WsError) -> StreamOutcome { - let WsError::Http(res) = err else { - return (TRANSPORT_ERROR_STATUS, Err(PbsError::WebSocketConnect(err.to_string()))); - }; - - let code = res.status(); - let body = res.body().as_deref().unwrap_or_default(); - let msg = if body.is_empty() { - format!("rejected with {code}") - } else { - format!("rejected with {code}: {}", String::from_utf8_lossy(body)) - }; - - // A 2xx handshake answer is a failed connect, not a delivered bid - let code = if code.is_success() { TRANSPORT_ERROR_STATUS } else { code }; - - (code, Err(PbsError::WebSocketConnect(msg))) -} - -fn build_handshake_request( - request_info: &RequestInfo, - relay: &RelayClient, - url: &Url, - timeout_ms: u64, -) -> Result, PbsError> { - let mut request = url - .as_str() - .into_client_request() - .map_err(|err| PbsError::WebSocketConnect(format!("invalid ws url: {err}")))?; - - let headers = request.headers_mut(); - if let Some(user_agent) = request_info.headers.get(USER_AGENT) { - headers.insert(USER_AGENT, user_agent.clone()); - } - - for (key, value) in relay.stream_headers() { - headers.insert(key, value.clone()); - } - - headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); - headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); - - Ok(request) -} - -fn parse_frame(payload: Bytes) -> Result<(ForkName, Bytes), PbsError> { - let &[msg_type, fork_byte] = payload - .first_chunk::() - .ok_or_else(|| PbsError::WebSocket(format!("frame too short: {} bytes", payload.len())))?; - - if msg_type != MSG_BID { - return Err(PbsError::WebSocket(format!("unknown message type: {msg_type}"))); - } - - let fork = fork_from_wire(fork_byte) - .ok_or_else(|| PbsError::WebSocket(format!("unknown fork: {fork_byte}")))?; - - Ok((fork, payload.slice(FRAME_PREFIX_LEN..))) -} - -/// One TLS config for every stream connection. Left to tokio-tungstenite it is -/// rebuilt per connect, which reparses the root store and, worse, gives each -/// connection its own session cache: every slot then pays a full handshake -/// instead of a resumed one. The provider is named explicitly because rustls is -/// built with both `ring` and `aws-lc-rs` here, so the default builder needs a -/// process-wide install to pick one. -fn tls_config() -> &'static Arc { - static CONFIG: OnceLock> = OnceLock::new(); - CONFIG.get_or_init(|| { - let mut roots = RootCertStore::empty(); - roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - - Arc::new( - ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) - .with_safe_default_protocol_versions() - .expect("aws-lc-rs supports tls 1.2 and 1.3") - .with_root_certificates(roots) - .with_no_client_auth(), - ) - }) -} - -#[cfg(test)] -mod tests { - use std::{fs, path::Path}; - - use ssz::Encode; - - use super::*; - - fn bid_frame(fork_byte: u8, bid: &[u8]) -> Bytes { - let mut frame = vec![MSG_BID, fork_byte]; - frame.extend_from_slice(bid); - Bytes::from(frame) - } - - #[test] - fn test_fork_from_wire_covers_all_forks() { - let forks = ForkName::list_all(); - - for (byte, fork) in forks.iter().enumerate() { - if *fork <= ForkName::Base { - continue; - } - assert_eq!(fork_from_wire(byte as u8), Some(*fork), "fork {fork} unmapped"); - } - - assert_eq!(fork_from_wire(forks.len() as u8), None); - } - - #[test] - fn test_parse_frame() { - assert!(matches!( - parse_frame(bid_frame(6, &[1, 2, 3])), - Ok((ForkName::Fulu, bid)) if bid.as_ref() == [1, 2, 3] - )); - - // Empty bid payload is well-formed at this layer, SSZ decoding rejects it - assert!(matches!(parse_frame(bid_frame(6, &[])), Ok((ForkName::Fulu, _)))); - - for bad in [ - // Truncated prefix - Bytes::from_static(&[]), - Bytes::from_static(&[MSG_BID]), - // Unknown fork - bid_frame(0xff, &[1]), - // Unknown message type - Bytes::from_static(&[0xff, 6]), - ] { - assert!(matches!(parse_frame(bad), Err(PbsError::WebSocket(_)))); - } - } - - #[test] - fn test_connect_failed_carries_relay_reason() { - let rejected = axum::http::Response::builder() - .status(401) - .body(Some(b"api key not registered".to_vec())) - .unwrap(); - - let (status, res) = connect_failed(&WsError::Http(Box::new(rejected))); - assert_eq!(status, StatusCode::UNAUTHORIZED); - let Err(PbsError::WebSocketConnect(msg)) = res else { panic!("wrong outcome") }; - assert!(msg.contains("401"), "{msg}"); - assert!(msg.contains("api key not registered"), "{msg}"); - - // No body to add, and transport failures keep tungstenite's own message - let empty = axum::http::Response::builder().status(404).body(None).unwrap(); - let (status, _) = connect_failed(&WsError::Http(Box::new(empty))); - assert_eq!(status, StatusCode::NOT_FOUND); - - let (status, res) = connect_failed(&WsError::ConnectionClosed); - assert_eq!(status, TRANSPORT_ERROR_STATUS); - assert!(matches!(res, Err(PbsError::WebSocketConnect(_)))); - } - - // A url pointing at a plain http endpoint answers the handshake 200. That - // is the code the stream series uses for a delivered bid, so a failed - // handshake must never carry it. - #[test] - fn test_connect_failed_never_reports_a_success_code() { - for code in [200u16, 204, 299] { - let answered = axum::http::Response::builder().status(code).body(None).unwrap(); - let (status, res) = connect_failed(&WsError::Http(Box::new(answered))); - assert_eq!( - status, TRANSPORT_ERROR_STATUS, - "handshake answered {code} counted as a served stream" - ); - let Err(PbsError::WebSocketConnect(msg)) = res else { panic!("wrong outcome") }; - assert!(msg.contains(&code.to_string()), "{msg}"); - } - - // A relay's own rejection code still reaches the series unchanged - let moved = axum::http::Response::builder().status(302).body(None).unwrap(); - let (status, _) = connect_failed(&WsError::Http(Box::new(moved))); - assert_eq!(status, StatusCode::FOUND); - } - - #[test] - fn test_decode_streamed_bid() { - let json_bytes = - fs::read(Path::new("../../tests/data/get_header/fulu.json")).expect("file not found"); - let expected: GetHeaderResponse = - serde_json::from_slice(&json_bytes).expect("failed to decode JSON"); - - let frame = bid_frame(6, &expected.data.as_ssz_bytes()); - let (fork, bid_bytes) = parse_frame(frame).unwrap(); - - let decoded = decode_ssz_payload(&bid_bytes, fork).unwrap(); - assert_eq!(fork, ForkName::Fulu); - assert_eq!(decoded.data, expected.data); - } -} From bb33556554a935ecd7b8c3aec28d3ca25189d012 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Sun, 4 Oct 2026 21:31:56 -0700 Subject: [PATCH 02/24] fix(pbs): a bid stream holds its latest valid bid The stream kept the latest frame that parsed and validated only that one at the deadline, so a last bid that failed validation (a bad signature, the wrong parent, below min_bid) discarded a valid bid the relay had sent before it. The reader now holds the newest 8 frames, validated newest first at the deadline, and the first that passes is returned. The usual case is still a single validation. When all 8 fail, the stream returns the newest one's error, as before. The cap bounds the memory a stream holds and the validation left for the deadline. A relay that sends 8 invalid bids after a valid one loses the valid one. --- crates/pbs/src/bid_stream.rs | 28 ++++++++----- crates/pbs/src/mev_boost/get_header_ws.rs | 48 ++++++++++++++--------- docs/docs/get_started/running/metrics.md | 2 +- tests/src/mock_ws_relay.rs | 21 ++++++---- tests/tests/pbs_get_header_ws.rs | 33 ++++++++++++++++ 5 files changed, 95 insertions(+), 37 deletions(-) diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs index cd0eb58c8..dde3929de 100644 --- a/crates/pbs/src/bid_stream.rs +++ b/crates/pbs/src/bid_stream.rs @@ -9,6 +9,7 @@ //! ``` use std::{ + collections::VecDeque, sync::{Arc, OnceLock}, time::Duration, }; @@ -44,6 +45,10 @@ const FRAME_PREFIX_LEN: usize = 2; const MSG_BID: u8 = 0x01; +/// A stream holds only its newest frames, which bounds its memory and the +/// validation a caller leaves until the deadline +const MAX_HELD_FRAMES: usize = 8; + /// One bid update, its bid still SSZ-encoded pub(crate) struct Frame { pub(crate) fork: ForkName, @@ -52,9 +57,9 @@ pub(crate) struct Frame { /// What a stream held when its window closed pub(crate) struct Held { - /// The last frame the parse closure accepted. None when the relay had no - /// bid. - pub(crate) latest: Option, + /// The newest `MAX_HELD_FRAMES` frames the parse closure accepted, oldest + /// first. Empty when the relay had no bid. + pub(crate) frames: VecDeque, /// Frames the parse closure accepted pub(crate) updates: usize, pub(crate) connect_latency: Duration, @@ -89,10 +94,10 @@ pub(crate) fn handshake_request( Ok(request) } -/// Opens the stream and holds the last frame `parse` accepts, until `deadline` -/// or the relay ends the stream. A frame `parse` rejects counts as invalid. On -/// failure, returns the status the failure records under. Time to the first -/// frame is recorded under `endpoint`. +/// Opens the stream and holds the newest frames `parse` accepts, until +/// `deadline` or the relay ends the stream. A frame `parse` rejects counts as +/// invalid. On failure, returns the status the failure records under. Time to +/// the first frame is recorded under `endpoint`. pub(crate) async fn read_bid_stream( request: Request<()>, deadline: Instant, @@ -125,7 +130,7 @@ pub(crate) async fn read_bid_stream( let timer = sleep_until(deadline); tokio::pin!(timer); - let mut latest = None; + let mut frames = VecDeque::with_capacity(MAX_HELD_FRAMES); let mut updates = 0usize; let mut first_frame_latency = None; let mut invalid_frames = 0usize; @@ -153,7 +158,10 @@ pub(crate) async fn read_bid_stream( Ok(frame) => { first_frame_latency.get_or_insert_with(|| start_request.elapsed()); updates += 1; - latest = Some(frame); + if frames.len() == MAX_HELD_FRAMES { + frames.pop_front(); + } + frames.push_back(frame); } Err(err) => { invalid_frames += 1; @@ -179,7 +187,7 @@ pub(crate) async fn read_bid_stream( return Err((TRANSPORT_ERROR_STATUS, err)); } - Ok(Held { latest, updates, connect_latency, first_frame_latency, invalid_frames }) + Ok(Held { frames, updates, connect_latency, first_frame_latency, invalid_frames }) } /// A relay rejecting the handshake puts the reason in the body, which diff --git a/crates/pbs/src/mev_boost/get_header_ws.rs b/crates/pbs/src/mev_boost/get_header_ws.rs index 085aac45f..5a8fcfd10 100644 --- a/crates/pbs/src/mev_boost/get_header_ws.rs +++ b/crates/pbs/src/mev_boost/get_header_ws.rs @@ -9,7 +9,7 @@ use alloy::primitives::utils::format_ether; use cb_common::pbs::{GetHeaderInfo, GetHeaderResponse, RelayClient, error::PbsError}; use reqwest::StatusCode; use tokio::time::Instant; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use url::Url; use super::get_header::{RequestInfo, decode_ssz_payload, validate_get_header_response}; @@ -21,8 +21,8 @@ use crate::{ type StreamOutcome = (StatusCode, Result, PbsError>); -/// Open a stream to the relay, keep the latest bid until the deadline, then -/// validate and return it. +/// Open a stream to the relay, hold its newest bids until the deadline, then +/// return the latest one that passes validation. pub(super) async fn get_header_ws( request_info: &RequestInfo, relay: &RelayClient, @@ -48,29 +48,41 @@ async fn stream_header( Err(err) => return (TRANSPORT_ERROR_STATUS, Err(err)), }; - let Held { latest, updates, connect_latency, first_frame_latency, invalid_frames } = + let Held { frames, updates, connect_latency, first_frame_latency, invalid_frames } = match read_bid_stream(request, deadline, relay, GET_HEADER_STREAM_ENDPOINT_TAG, Ok).await { Ok(held) => held, Err((status, err)) => return (status, Err(err)), }; - let Some(Frame { fork, bid }) = latest else { - debug!(relay_id = relay.id.as_ref(), ?connect_latency, invalid_frames, "no header"); - return (StatusCode::NO_CONTENT, Ok(None)); - }; - - let response = match decode_ssz_payload(&bid, fork) { - Ok(response) => response, - Err(err) => return (StatusCode::OK, Err(err)), - }; - + // The latest valid bid stands. Newest first, so the usual case validates once let start_validate = Instant::now(); - let validated = validate_get_header_response(request_info, relay, &response); + let mut newest_err = None; + let latest_valid = frames.iter().rev().find_map(|frame| { + let validated = decode_ssz_payload(&frame.bid, frame.fork).and_then(|response| { + validate_get_header_response(request_info, relay, &response).map(|()| response) + }); + match validated { + Ok(response) => Some((frame, response)), + Err(err) => { + newest_err.get_or_insert(err); + None + } + } + }); let validate_latency = start_validate.elapsed(); - if let Err(err) = validated { - return (StatusCode::OK, Err(err)); - } + let (Frame { fork, bid }, response) = match (latest_valid, newest_err) { + (Some(valid), None) => valid, + (Some(valid), Some(err)) => { + warn!(relay_id = relay.id.as_ref(), %err, "latest stream bid invalid, an earlier one stands"); + valid + } + (None, Some(err)) => return (StatusCode::OK, Err(err)), + (None, None) => { + debug!(relay_id = relay.id.as_ref(), ?connect_latency, invalid_frames, "no header"); + return (StatusCode::NO_CONTENT, Ok(None)); + } + }; info!( relay_id = relay.id.as_ref(), diff --git a/docs/docs/get_started/running/metrics.md b/docs/docs/get_started/running/metrics.md index 30ff9e606..5c1c7074c 100644 --- a/docs/docs/get_started/running/metrics.md +++ b/docs/docs/get_started/running/metrics.md @@ -106,4 +106,4 @@ For a relay with `get_header = "stream"`, every series below is labelled by `rel | Is it actually streaming? | `cb_pbs_relay_stream_updates`: bid updates received per window. A healthy relay sends several; windows that carry at most one update mean the stream connects but does not stream | | Are the frames usable? | `cb_pbs_relay_stream_invalid_frames_total`: frames that could not be parsed as a bid, absent while zero | -Two things the series do not tell apart. A bid that arrives but fails decoding or validation still counts as `200`, the same as over HTTP; the validation error is in the logs. And a request the beacon node abandons mid-window records no outcome at all. +Two things the series do not tell apart. The stream holds its newest 8 bids and returns the latest that passes validation, but a window in which every held bid fails decoding or validation still counts as `200`, the same as over HTTP; the validation error is in the logs. And a request the beacon node abandons mid-window records no outcome at all. diff --git a/tests/src/mock_ws_relay.rs b/tests/src/mock_ws_relay.rs index 2ef44dc0e..ea7364a17 100644 --- a/tests/src/mock_ws_relay.rs +++ b/tests/src/mock_ws_relay.rs @@ -59,6 +59,9 @@ pub struct MockWsRelayState { /// Precede each bid with frames PBS can't parse, which it must skip rather /// than treat as the end of the stream unknown_frames: bool, + /// Sign this many of the last bids for another parent hash, so they fail + /// validation + invalid_last_bids: usize, received_connections: AtomicU64, last_request: Mutex>, } @@ -72,6 +75,7 @@ impl MockWsRelayState { update_interval: Duration::ZERO, hold_open: false, unknown_frames: false, + invalid_last_bids: 0, received_connections: AtomicU64::new(0), last_request: Mutex::new(None), } @@ -93,6 +97,10 @@ impl MockWsRelayState { Self { unknown_frames: true, ..self } } + pub fn with_invalid_last_bids(self, invalid_last_bids: usize) -> Self { + Self { invalid_last_bids, ..self } + } + pub fn received_connections(&self) -> u64 { self.received_connections.load(Ordering::Relaxed) } @@ -131,7 +139,7 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: state.received_connections.fetch_add(1, Ordering::Relaxed); *state.last_request.lock().unwrap() = Some(request.clone()); - for value in &state.bid_values { + for (i, value) in state.bid_values.iter().enumerate() { if state.unknown_frames { // Unknown message type, unknown fork, truncated prefix for frame in [vec![0x7f, FORK_FULU, 1], vec![MSG_BID, 0xff, 1], vec![MSG_BID]] { @@ -139,13 +147,10 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: } } - let bid = mock_signed_builder_bid( - state.chain, - &state.signer, - request.slot, - request.parent_hash, - *value, - ); + let is_invalid = i + state.invalid_last_bids >= state.bid_values.len(); + let parent_hash = if is_invalid { B256::repeat_byte(0xee) } else { request.parent_hash }; + let bid = + mock_signed_builder_bid(state.chain, &state.signer, request.slot, parent_hash, *value); let mut frame = vec![MSG_BID, FORK_FULU]; frame.extend_from_slice(&bid.as_ssz_bytes()); diff --git a/tests/tests/pbs_get_header_ws.rs b/tests/tests/pbs_get_header_ws.rs index e132a70d1..ee958d7ab 100644 --- a/tests/tests/pbs_get_header_ws.rs +++ b/tests/tests/pbs_get_header_ws.rs @@ -132,6 +132,39 @@ async fn test_get_header_ws_returns_latest_bid() -> Result<()> { Ok(()) } +/// Bids that fail validation do not displace the valid one before them, while +/// it is among the newest 8 the stream holds +#[tokio::test] +async fn test_get_header_ws_returns_latest_valid_bid() -> Result<()> { + setup_test_env(); + let signer = random_secret(); + let chain = Chain::Hoodi; + + for (invalid_bids, served) in [(7, Some(50)), (8, None)] { + let mut bid_values = vec![U256::from(50)]; + bid_values.resize(invalid_bids + 1, U256::from(60)); + let (_relay_state, relay) = start_stream_relay( + MockWsRelayState::new(chain, signer.clone()) + .with_bid_values(bid_values) + .with_invalid_last_bids(invalid_bids), + signer.public_key(), + ) + .await?; + + let validator = start_pbs(chain, vec![relay], 1_000).await?; + + let (code, res) = get_header_json(&validator).await?; + match served { + Some(value) => { + assert_eq!(code, StatusCode::OK, "{invalid_bids} invalid bids"); + assert_bid(&res.unwrap(), chain, &signer, U256::from(value)); + } + None => assert_eq!(code, StatusCode::NO_CONTENT, "{invalid_bids} invalid bids"), + } + } + Ok(()) +} + /// Frames PBS can't parse are skipped, not treated as the end of the stream: /// the updates after them still count. #[tokio::test] From b7d5e90f3a02f78e545d663a76a2f9c945f9ef73 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Sun, 4 Oct 2026 21:33:56 -0700 Subject: [PATCH 03/24] fix(pbs): a 204 answer to the bid stream handshake is no bid A relay with no bid for the slot can answer the stream handshake 204, as it answers get_header. The stream treated every 2xx without an upgrade as a failed connect, recorded it as 556 and fell back to HTTP. A 204 is now no bid: recorded as 204, no fallback. Any other 2xx without an upgrade is still a failed connect. --- crates/pbs/src/bid_stream.rs | 12 ++- crates/pbs/src/mev_boost/get_header.rs | 93 +++++++++++++++++++++++- docs/docs/get_started/running/metrics.md | 2 +- 3 files changed, 104 insertions(+), 3 deletions(-) diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs index dde3929de..4def5ad0a 100644 --- a/crates/pbs/src/bid_stream.rs +++ b/crates/pbs/src/bid_stream.rs @@ -118,6 +118,16 @@ pub(crate) async fn read_bid_stream( ); let (mut stream, _) = match timeout_at(deadline, connect).await { Ok(Ok(connected)) => connected, + // A relay with no bid can answer the handshake as it answers get_header + Ok(Err(WsError::Http(res))) if res.status() == StatusCode::NO_CONTENT => { + return Ok(Held { + frames: VecDeque::new(), + updates: 0, + connect_latency: start_request.elapsed(), + first_frame_latency: None, + invalid_frames: 0, + }); + } Ok(Err(err)) => return Err(connect_failed(&err)), Err(_) => return Err((TIMEOUT_ERROR_STATUS, PbsError::WebSocketTimeout)), }; @@ -340,7 +350,7 @@ mod tests { // handshake must never carry it. #[test] fn test_connect_failed_never_reports_a_success_code() { - for code in [200u16, 204, 299] { + for code in [200u16, 299] { let answered = axum::http::Response::builder().status(code).body(None).unwrap(); let (status, err) = connect_failed(&WsError::Http(Box::new(answered))); assert_eq!( diff --git a/crates/pbs/src/mev_boost/get_header.rs b/crates/pbs/src/mev_boost/get_header.rs index 3427c8f77..c21f1ce95 100644 --- a/crates/pbs/src/mev_boost/get_header.rs +++ b/crates/pbs/src/mev_boost/get_header.rs @@ -742,10 +742,11 @@ fn extra_validation( #[cfg(test)] mod tests { - use std::{fs, path::Path}; + use std::{fs, net::SocketAddr, path::Path}; use alloy::primitives::{B256, U256}; use cb_common::{ + config::{GetHeaderTransport, PbsConfig, PbsModuleConfig, RelayConfig}, pbs::*, signature::sign_builder_message, ssz::get_bid_value_from_signed_builder_bid_ssz, @@ -753,8 +754,13 @@ mod tests { utils::{TestRandomSeed, timestamp_of_slot_start_sec}, }; use ssz::Encode; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; use super::{validate_header_data, *}; + use crate::constants::GET_HEADER_STREAM_ENDPOINT_TAG; #[test] fn test_validate_header() { @@ -890,4 +896,89 @@ mod tests { assert_eq!(*decoded.value(), bid_value); } } + + /// A relay that answers every request, the stream handshake included, with + /// `status` and no body + async fn start_status_relay(status: StatusCode) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + tokio::spawn(async move { + let mut request = [0u8; 4096]; + let _ = socket.read(&mut request).await; + let response = format!("HTTP/1.1 {status}\r\ncontent-length: 0\r\n\r\n"); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + addr + } + + /// get_header against one stream relay at `addr`. Returns the relay's id, + /// which labels its metrics, and the bid. + async fn get_header_from_stream_relay(addr: SocketAddr) -> (String, Option) { + let pubkey = BlsSecretKey::test_random().public_key(); + let relay_id = format!("stream_relay_{}", addr.port()); + let relay = RelayClient::new(RelayConfig { + entry: RelayEntry { + id: relay_id.clone(), + pubkey: pubkey.clone(), + url: format!("http://{pubkey}@{addr}").parse().unwrap(), + }, + id: None, + headers: None, + get_params: None, + get_header: GetHeaderTransport::Stream, + enable_timing_games: false, + target_first_request_ms: None, + frequency_get_header_ms: None, + validator_registration_batch_size: None, + }) + .unwrap(); + let mut pbs_config: PbsConfig = serde_json::from_str("{}").unwrap(); + pbs_config.late_in_slot_time_ms = u64::MAX; + let state = PbsState::new( + PbsModuleConfig { + chain: Chain::Hoodi, + endpoint: addr, + pbs_config: Arc::new(pbs_config), + relays: vec![relay.clone()], + all_relays: vec![relay], + signer_client: None, + registry_muxes: None, + mux_lookup: None, + }, + Default::default(), + ); + let params = GetHeaderParams { slot: 1, parent_hash: B256::ZERO, pubkey }; + + let bid = get_header(params, HeaderMap::new(), state).await.unwrap(); + (relay_id, bid) + } + + #[tokio::test] + async fn test_stream_fallback_counts_a_refused_handshake_not_a_204() { + for (status, fallbacks) in [(StatusCode::NO_CONTENT, 0), (StatusCode::NOT_FOUND, 1)] { + let addr = start_status_relay(status).await; + let (relay_id, bid) = get_header_from_stream_relay(addr).await; + assert!(bid.is_none()); + assert_eq!( + RELAY_STATUS_CODE + .with_label_values(&[ + status.as_str(), + GET_HEADER_STREAM_ENDPOINT_TAG, + &relay_id + ]) + .get(), + 1, + "handshake answered {status}" + ); + assert_eq!( + RELAY_STREAM_FALLBACK.with_label_values(&[&relay_id]).get(), + fallbacks, + "handshake answered {status}" + ); + } + } } diff --git a/docs/docs/get_started/running/metrics.md b/docs/docs/get_started/running/metrics.md index 5c1c7074c..abea50abc 100644 --- a/docs/docs/get_started/running/metrics.md +++ b/docs/docs/get_started/running/metrics.md @@ -99,7 +99,7 @@ For a relay with `get_header = "stream"`, every series below is labelled by `rel | Question | Series | |---|---| -| Is the stream serving bids? | `cb_pbs_relay_status_code_total{endpoint="get_header_stream"}`: `200` a bid was delivered, `204` connected but no bid before the deadline, `555` the bid window ran out during the handshake, `556` a transport error (connect failed, stream broke mid-window, or the handshake was answered with anything but `101`), any other code the relay's own refusal of the upgrade | +| Is the stream serving bids? | `cb_pbs_relay_status_code_total{endpoint="get_header_stream"}`: `200` a bid was delivered, `204` no bid (the relay answered the handshake `204`, or sent no bid before the deadline), `555` the bid window ran out during the handshake, `556` a transport error (connect failed, stream broke mid-window, or the handshake was answered with a `2xx` other than `204`), any other code the relay's own refusal of the upgrade | | How fast does the first bid arrive? | `cb_pbs_relay_latency{endpoint="get_header_stream"}` | | Is it falling back to HTTP? | `cb_pbs_relay_stream_fallback_total`: handshake failures that had bid window left to retry over HTTP. One at startup is the registration race; a steady rate means the relay is refusing the stream. The fallback's own results are under `endpoint="get_header"` | | Is the handshake slow? | `cb_pbs_relay_stream_connect_latency` | From a02add957e2708288cf985101f56fd88ef938158 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Sun, 4 Oct 2026 21:39:42 -0700 Subject: [PATCH 04/24] feat(pbs): race the relay's HTTP get_header alongside every bid stream A relay with get_header = "stream" now also gets the relay's normal HTTP get_header, timing games included, sent at the same time as the stream handshake. The two are separate candidates in the bid selection: the stream's latest valid bid and the HTTP bid compete like bids from any two relays. A stream whose handshake times out or is refused, or that breaks before a bid, leaves the HTTP bid in the auction, and an operator with a slow link to the relay no longer needs a second HTTP entry for it. This replaces the HTTP fallback that ran only when the handshake failed with bid window left. cb_pbs_relay_stream_fallback_total keeps its name and now counts every stream that failed while the HTTP request ran alongside, including a window whose held bids all fail validation. relay_header_value reports the relay's best bid across both transports, and relay_last_slot is set when either delivers one. The four relay_stream metrics gain an endpoint label (get_header_stream) and help text that holds for any bid stream, and the auction winner log names the transport. --- config.example.toml | 5 +- crates/common/src/config/pbs.rs | 9 ++ crates/common/src/pbs/relay.rs | 9 ++ crates/pbs/src/bid_stream.rs | 20 ++- crates/pbs/src/metrics.rs | 24 +-- crates/pbs/src/mev_boost/get_header.rs | 188 ++++++++++++++--------- docs/docs/get_started/running/metrics.md | 12 +- tests/src/mock_ws_relay.rs | 66 ++++++++ tests/tests/pbs_get_header_ws.rs | 133 ++++++++++++---- 9 files changed, 334 insertions(+), 132 deletions(-) diff --git a/config.example.toml b/config.example.toml index 64b6d66a4..efd3e5db7 100644 --- a/config.example.toml +++ b/config.example.toml @@ -100,7 +100,10 @@ get_params = { param1 = "value1", param2 = "value2" } # How to fetch headers from this relay. # "http" -> one request per get_header, at the relay url above # "stream" -> websocket stream of bid updates, only for relays that support it. Connects to -# ws(s):///eth/v1/builder/header_stream/{slot}/{parent_hash}/{pubkey}. +# ws(s):///eth/v1/builder/header_stream/{slot}/{parent_hash}/{pubkey}, +# and sends the relay's normal HTTP request alongside, timing games included. The +# stream's latest valid bid and the HTTP bid are separate candidates in the +# auction, so a stream that fails leaves the HTTP bid in the auction. # Requires a UUID api key, set it in `headers` field above under the `X-Api-Key` # name; it is sent on the websocket handshake. Can use # https://www.uuidgenerator.net/version4 to generate one. diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index cdade80ed..7dcf5613f 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -49,6 +49,15 @@ pub enum GetHeaderTransport { Stream, } +impl GetHeaderTransport { + pub fn as_str(&self) -> &'static str { + match self { + Self::Http => "http", + Self::Stream => "stream", + } + } +} + /// A custom relay header value: a literal, or a secret read from a file or an /// environment variable when the relay client is built (at startup and on every /// reload), so an API key never has to sit in plaintext in the config file. diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index 1a2768557..6464241ec 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -62,6 +62,15 @@ pub enum GetHeaderRequest { Stream(Url), } +impl GetHeaderRequest { + pub fn transport(&self) -> GetHeaderTransport { + match self { + Self::Http(_) => GetHeaderTransport::Http, + Self::Stream(_) => GetHeaderTransport::Stream, + } + } +} + fn stream_url(entry: &Url) -> eyre::Result { let scheme = match entry.scheme() { "http" | "ws" => "ws", diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs index 4def5ad0a..4c961a0a6 100644 --- a/crates/pbs/src/bid_stream.rs +++ b/crates/pbs/src/bid_stream.rs @@ -35,8 +35,8 @@ use url::Url; use crate::{ constants::{MAX_SIZE_GET_HEADER_RESPONSE, TIMEOUT_ERROR_STATUS, TRANSPORT_ERROR_STATUS}, metrics::{ - RELAY_LATENCY, RELAY_STREAM_CONNECT_LATENCY, RELAY_STREAM_INVALID_FRAMES, - RELAY_STREAM_UPDATES, + RELAY_LATENCY, RELAY_STREAM_CONNECT_LATENCY, RELAY_STREAM_FALLBACK, + RELAY_STREAM_INVALID_FRAMES, RELAY_STREAM_UPDATES, }, }; @@ -96,8 +96,8 @@ pub(crate) fn handshake_request( /// Opens the stream and holds the newest frames `parse` accepts, until /// `deadline` or the relay ends the stream. A frame `parse` rejects counts as -/// invalid. On failure, returns the status the failure records under. Time to -/// the first frame is recorded under `endpoint`. +/// invalid. On failure, returns the status the failure records under. The +/// stream metrics are recorded under `endpoint`. pub(crate) async fn read_bid_stream( request: Request<()>, deadline: Instant, @@ -133,9 +133,9 @@ pub(crate) async fn read_bid_stream( }; let connect_latency = start_request.elapsed(); RELAY_STREAM_CONNECT_LATENCY - .with_label_values(&[relay.id.as_str()]) + .with_label_values(&[endpoint, relay.id.as_str()]) .observe(connect_latency.as_secs_f64()); - debug!(relay_id = relay.id.as_ref(), ?connect_latency, "ws connected"); + debug!(relay_id = relay.id.as_ref(), endpoint, ?connect_latency, "ws connected"); let timer = sleep_until(deadline); tokio::pin!(timer); @@ -184,10 +184,10 @@ pub(crate) async fn read_bid_stream( drop(stream); - RELAY_STREAM_UPDATES.with_label_values(&[relay.id.as_str()]).observe(updates as f64); + RELAY_STREAM_UPDATES.with_label_values(&[endpoint, relay.id.as_str()]).observe(updates as f64); if invalid_frames > 0 { RELAY_STREAM_INVALID_FRAMES - .with_label_values(&[relay.id.as_str()]) + .with_label_values(&[endpoint, relay.id.as_str()]) .inc_by(invalid_frames as u64); } @@ -200,6 +200,10 @@ pub(crate) async fn read_bid_stream( Ok(Held { frames, updates, connect_latency, first_frame_latency, invalid_frames }) } +pub(crate) fn record_stream_fallback(endpoint: &str, relay_id: &str) { + RELAY_STREAM_FALLBACK.with_label_values(&[endpoint, relay_id]).inc(); +} + /// A relay rejecting the handshake puts the reason in the body, which /// tungstenite's own `Display` drops. The body is only what arrived alongside /// the headers, so it can be partial or empty. diff --git a/crates/pbs/src/metrics.rs b/crates/pbs/src/metrics.rs index 51846f946..47e93e36b 100644 --- a/crates/pbs/src/metrics.rs +++ b/crates/pbs/src/metrics.rs @@ -60,38 +60,38 @@ lazy_static! { pub static ref RELAY_STREAM_CONNECT_LATENCY: HistogramVec = register_histogram_vec_with_registry!( "relay_stream_connect_latency", "Websocket handshake latency by relay", - &["relay_id"], + &["endpoint", "relay_id"], vec![0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 1.0], PBS_METRICS_REGISTRY ) .unwrap(); - /// Bid updates received per stream window by relay + /// Stream frames accepted on arrival per window, by relay + // What a frame must pass on arrival is the endpoint's: get_header_stream + // reads only its prefix, and validates the bids it holds at the deadline pub static ref RELAY_STREAM_UPDATES: HistogramVec = register_histogram_vec_with_registry!( "relay_stream_updates", - "Bid updates received per stream window by relay", - &["relay_id"], + "Stream frames accepted on arrival per window, by relay", + &["endpoint", "relay_id"], vec![0.0, 1.0, 2.0, 3.0, 5.0, 10.0, 20.0, 50.0, 100.0], PBS_METRICS_REGISTRY ) .unwrap(); - /// Websocket frames that could not be parsed as a bid, by relay + /// Stream frames rejected as invalid on arrival, by relay pub static ref RELAY_STREAM_INVALID_FRAMES: IntCounterVec = register_int_counter_vec_with_registry!( "relay_stream_invalid_frames_total", - "Websocket frames that could not be parsed as a bid, by relay", - &["relay_id"], + "Stream frames rejected as invalid on arrival, by relay", + &["endpoint", "relay_id"], PBS_METRICS_REGISTRY ) .unwrap(); - /// Stream attempts that fell back to HTTP, by relay - // Only a handshake failure with bid window left retries; one at the - // deadline shows on the status series alone + /// Bid streams that failed while the relay's HTTP request ran alongside pub static ref RELAY_STREAM_FALLBACK: IntCounterVec = register_int_counter_vec_with_registry!( "relay_stream_fallback_total", - "get_header stream attempts that fell back to HTTP, by relay", - &["relay_id"], + "Bid streams that failed while the relay's HTTP request ran alongside, by relay", + &["endpoint", "relay_id"], PBS_METRICS_REGISTRY ) .unwrap(); diff --git a/crates/pbs/src/mev_boost/get_header.rs b/crates/pbs/src/mev_boost/get_header.rs index c21f1ce95..a829e78cf 100644 --- a/crates/pbs/src/mev_boost/get_header.rs +++ b/crates/pbs/src/mev_boost/get_header.rs @@ -1,4 +1,6 @@ use std::{ + collections::HashMap, + iter::once, sync::Arc, time::{Duration, Instant}, }; @@ -10,6 +12,7 @@ use alloy::{ }; use axum::http::{HeaderMap, HeaderValue}; use cb_common::{ + config::GetHeaderTransport, constants::APPLICATION_BUILDER_DOMAIN, pbs::{ EMPTY_TX_ROOT_HASH, ExecutionPayloadHeaderRef, ForkName, ForkVersionDecode, GetHeaderInfo, @@ -25,7 +28,7 @@ use cb_common::{ parse_response_encoding_and_fork, safe_read_http_response, }, }; -use futures::future::join_all; +use futures::{FutureExt, future::join_all}; use parking_lot::RwLock; use reqwest::{ StatusCode, @@ -38,14 +41,12 @@ use url::Url; use super::get_header_ws::get_header_ws; use crate::{ + bid_stream::record_stream_fallback, constants::{ - GET_HEADER_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, TIMEOUT_ERROR_CODE, - TIMEOUT_ERROR_CODE_STR, - }, - metrics::{ - RELAY_HEADER_VALUE, RELAY_LAST_SLOT, RELAY_LATENCY, RELAY_STATUS_CODE, - RELAY_STREAM_FALLBACK, + GET_HEADER_ENDPOINT_TAG, GET_HEADER_STREAM_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, + TIMEOUT_ERROR_CODE, TIMEOUT_ERROR_CODE_STR, }, + metrics::{RELAY_HEADER_VALUE, RELAY_LAST_SLOT, RELAY_LATENCY, RELAY_STATUS_CODE}, state::{BuilderApiState, PbsState}, utils::check_gas_limit, }; @@ -171,52 +172,80 @@ pub async fn get_header( parent_block, }, }); + // A stream relay races its normal HTTP request; each is a separate candidate + // below + let params = &request_info.params; let mut handles = Vec::with_capacity(relays.len()); for relay in relays.iter() { - handles.push( - get_header_from_relay( - request_info.clone(), - relay.clone(), - ms_into_slot, - max_timeout_ms, - ) - .in_current_span(), - ); + let relay_id = relay.id.as_str(); + let http_url = match relay.get_header_url(params.slot, ¶ms.parent_hash, ¶ms.pubkey) + { + Ok(url) => url, + Err(err) => { + error!(%err, relay_id); + continue; + } + }; + let stream_url = + relay.get_header_stream_url(params.slot, ¶ms.parent_hash, ¶ms.pubkey); + + let requests = + once(GetHeaderRequest::Http(http_url)).chain(stream_url.map(GetHeaderRequest::Stream)); + for request in requests { + let transport = request.transport(); + handles.push( + get_header_from_relay( + request_info.clone(), + relay.clone(), + ms_into_slot, + max_timeout_ms, + request, + ) + .map(move |res| (relay_id, transport, res)) + .in_current_span(), + ); + } } - let results = join_all(handles).await; - let mut relay_bids = Vec::with_capacity(relays.len()); - for (i, res) in results.into_iter().enumerate() { - let relay_id = relays[i].id.as_str(); - + let mut relay_bids = Vec::with_capacity(handles.len()); + let mut best_per_relay: HashMap<&str, U256> = HashMap::new(); + for (relay_id, transport, res) in join_all(handles).await { match res { Ok(Some(res)) => { - RELAY_LAST_SLOT.with_label_values(&[relay_id]).set(slot); - let value_gwei = (res.data.message.value() / U256::from(1_000_000_000)) - .try_into() - .unwrap_or_default(); - RELAY_HEADER_VALUE.with_label_values(&[relay_id]).set(value_gwei); - - relay_bids.push((relay_id, res)) + let best = best_per_relay.entry(relay_id).or_default(); + *best = (*best).max(*res.value()); + relay_bids.push((relay_id, transport, res)) + } + Ok(None) => {} + Err(err) if transport == GetHeaderTransport::Stream => { + warn!(%err, relay_id, "stream failed"); + record_stream_fallback(GET_HEADER_STREAM_ENDPOINT_TAG, relay_id); } - Ok(_) => {} - Err(err) if err.is_timeout() => error!(err = "Timed Out", relay_id), - Err(err) => error!(%err, relay_id), + Err(err) if err.is_timeout() => { + error!(err = "Timed Out", relay_id, transport = transport.as_str()) + } + Err(err) => error!(%err, relay_id, transport = transport.as_str()), } } + for (relay_id, value) in best_per_relay { + RELAY_LAST_SLOT.with_label_values(&[relay_id]).set(slot); + let value_gwei = (value / U256::from(1_000_000_000)).try_into().unwrap_or_default(); + RELAY_HEADER_VALUE.with_label_values(&[relay_id]).set(value_gwei); + } - let max_bid = relay_bids.into_iter().max_by_key(|(_, bid)| *bid.value()); + let max_bid = relay_bids.into_iter().max_by_key(|(_, _, bid)| *bid.value()); - if let Some((winning_relay_id, ref bid)) = max_bid { + if let Some((winning_relay_id, transport, ref bid)) = max_bid { info!( relay_id = winning_relay_id, + transport = transport.as_str(), value_eth = format_ether(*bid.value()), block_hash = %bid.block_hash(), "auction winner" ); } - Ok(max_bid.map(|(_, bid)| bid)) + Ok(max_bid.map(|(_, _, bid)| bid)) } /// Fetch the parent block from the RPC URL for extra validation of the header. @@ -249,44 +278,11 @@ async fn get_header_from_relay( relay: RelayClient, ms_into_slot: u64, timeout_left_ms: u64, + request: GetHeaderRequest, ) -> Result, PbsError> { - let params = &request_info.params; - let request = - match relay.get_header_stream_url(params.slot, ¶ms.parent_hash, ¶ms.pubkey) { - Some(url) => GetHeaderRequest::Stream(url), - None => GetHeaderRequest::Http(relay.get_header_url( - params.slot, - ¶ms.parent_hash, - ¶ms.pubkey, - )?), - }; - match request { GetHeaderRequest::Stream(url) => { - let started = Instant::now(); - let err = match get_header_ws(&request_info, &relay, url, timeout_left_ms).await { - Err(PbsError::WebSocketConnect(err)) => err, - res => return res, - }; - - let elapsed_ms = started.elapsed().as_millis() as u64; - let timeout_left_ms = timeout_left_ms.saturating_sub(elapsed_ms); - if timeout_left_ms == 0 { - return Err(PbsError::WebSocketConnect(err)); - } - - RELAY_STREAM_FALLBACK.with_label_values(&[relay.id.as_str()]).inc(); - warn!(relay_id = relay.id.as_ref(), %err, timeout_left_ms, "stream failed, falling back to http get_header"); - - let url = relay.get_header_url(params.slot, ¶ms.parent_hash, ¶ms.pubkey)?; - send_timed_get_header( - request_info, - relay, - ms_into_slot + elapsed_ms, - url, - timeout_left_ms, - ) - .await + get_header_ws(&request_info, &relay, url, timeout_left_ms).await } GetHeaderRequest::Http(url) => { send_timed_get_header(request_info, relay, ms_into_slot, url, timeout_left_ms).await @@ -746,21 +742,23 @@ mod tests { use alloy::primitives::{B256, U256}; use cb_common::{ - config::{GetHeaderTransport, PbsConfig, PbsModuleConfig, RelayConfig}, + config::{PbsConfig, PbsModuleConfig, RelayConfig}, pbs::*, signature::sign_builder_message, ssz::get_bid_value_from_signed_builder_bid_ssz, types::{BlsPublicKeyBytes, BlsSecretKey, BlsSignature, Chain}, utils::{TestRandomSeed, timestamp_of_slot_start_sec}, }; + use futures::SinkExt; use ssz::Encode; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, }; + use tokio_tungstenite::{accept_async, tungstenite::Message}; use super::{validate_header_data, *}; - use crate::constants::GET_HEADER_STREAM_ENDPOINT_TAG; + use crate::metrics::{RELAY_STREAM_FALLBACK, RELAY_STREAM_UPDATES}; #[test] fn test_validate_header() { @@ -915,6 +913,31 @@ mod tests { addr } + /// A relay whose stream sends `frames` copies of a bid for another parent, + /// then closes. Its HTTP get_header fails. + async fn start_invalid_bid_stream_relay(frames: usize) -> SocketAddr { + let json = fs::read("../../tests/data/get_header/fulu.json").unwrap(); + // Message type bid, fork fulu + let mut frame = vec![0x01, 6]; + frame.extend(decode_json_payload(&json).unwrap().data.as_ssz_bytes()); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((socket, _)) = listener.accept().await { + let frame = frame.clone(); + tokio::spawn(async move { + let Ok(mut stream) = accept_async(socket).await else { return }; + for _ in 0..frames { + let _ = stream.send(Message::Binary(frame.clone().into())).await; + } + let _ = stream.close(None).await; + }); + } + }); + addr + } + /// get_header against one stream relay at `addr`. Returns the relay's id, /// which labels its metrics, and the bid. async fn get_header_from_stream_relay(addr: SocketAddr) -> (String, Option) { @@ -975,10 +998,31 @@ mod tests { "handshake answered {status}" ); assert_eq!( - RELAY_STREAM_FALLBACK.with_label_values(&[&relay_id]).get(), + RELAY_STREAM_FALLBACK + .with_label_values(&[GET_HEADER_STREAM_ENDPOINT_TAG, &relay_id]) + .get(), fallbacks, "handshake answered {status}" ); } } + + // A window whose every bid fails validation records 200 and one failed + // stream. Its updates count every bid it accepted, not only those it held. + #[tokio::test] + async fn test_stream_fallback_counts_a_window_with_no_valid_bid() { + let addr = start_invalid_bid_stream_relay(10).await; + let (relay_id, bid) = get_header_from_stream_relay(addr).await; + assert!(bid.is_none()); + + let status = RELAY_STATUS_CODE.with_label_values(&[ + StatusCode::OK.as_str(), + GET_HEADER_STREAM_ENDPOINT_TAG, + &relay_id, + ]); + assert_eq!(status.get(), 1); + let stream = [GET_HEADER_STREAM_ENDPOINT_TAG, relay_id.as_str()]; + assert_eq!(RELAY_STREAM_FALLBACK.with_label_values(&stream).get(), 1); + assert_eq!(RELAY_STREAM_UPDATES.with_label_values(&stream).get_sample_sum(), 10.0); + } } diff --git a/docs/docs/get_started/running/metrics.md b/docs/docs/get_started/running/metrics.md index abea50abc..ca6de6c62 100644 --- a/docs/docs/get_started/running/metrics.md +++ b/docs/docs/get_started/running/metrics.md @@ -95,15 +95,15 @@ Once Grafana is running, you can [import](https://grafana.com/docs/grafana/lates ## Bid stream -For a relay with `get_header = "stream"`, every series below is labelled by `relay_id`. The stream's own outcome and time-to-first-bid share the two general relay series under `endpoint="get_header_stream"`; the HTTP fallback keeps `endpoint="get_header"`, so the two divide slots served by the stream from slots served by the fallback. +For a relay with `get_header = "stream"`, Commit-Boost sends the relay's normal HTTP get_header alongside the stream, timing games included, and the two are separate candidates in the auction; the `auction winner` log line names the winning `transport`. Every series below is labelled by `relay_id`, and the stream-only series also by `endpoint`. The stream's own outcome and time-to-first-bid share the two general relay series under `endpoint="get_header_stream"`; the HTTP request keeps `endpoint="get_header"`. | Question | Series | |---|---| -| Is the stream serving bids? | `cb_pbs_relay_status_code_total{endpoint="get_header_stream"}`: `200` a bid was delivered, `204` no bid (the relay answered the handshake `204`, or sent no bid before the deadline), `555` the bid window ran out during the handshake, `556` a transport error (connect failed, stream broke mid-window, or the handshake was answered with a `2xx` other than `204`), any other code the relay's own refusal of the upgrade | +| Is the stream serving bids? | `cb_pbs_relay_status_code_total{endpoint="get_header_stream"}`: `200` a bid was delivered, `204` no bid (the relay answered the handshake `204`, or sent no bid before the deadline), `555` the bid window ran out during the handshake, `556` a transport error (connect failed, stream broke before a bid, or the handshake was answered with a `2xx` other than `204`), any other code the relay's own refusal of the upgrade | | How fast does the first bid arrive? | `cb_pbs_relay_latency{endpoint="get_header_stream"}` | -| Is it falling back to HTTP? | `cb_pbs_relay_stream_fallback_total`: handshake failures that had bid window left to retry over HTTP. One at startup is the registration race; a steady rate means the relay is refusing the stream. The fallback's own results are under `endpoint="get_header"` | +| Is the stream failing? | `cb_pbs_relay_stream_fallback_total{endpoint="get_header_stream"}`: streams that failed while the relay's HTTP request ran alongside: a handshake that could not connect, timed out or was refused, a stream that broke before a bid, or a window in which every held bid failed decoding or validation. A `204` is no bid, not a failure. It counts every failed stream, whether or not the HTTP request returned a bid; the HTTP results are under `endpoint="get_header"`. One at startup is the registration race; for a steady rate, the status series above gives the reason | | Is the handshake slow? | `cb_pbs_relay_stream_connect_latency` | -| Is it actually streaming? | `cb_pbs_relay_stream_updates`: bid updates received per window. A healthy relay sends several; windows that carry at most one update mean the stream connects but does not stream | -| Are the frames usable? | `cb_pbs_relay_stream_invalid_frames_total`: frames that could not be parsed as a bid, absent while zero | +| Is it actually streaming? | `cb_pbs_relay_stream_updates`: frames accepted per window, which is every frame whose message type and fork byte can be read; their bids are decoded and validated only when the window closes. A healthy relay sends several; windows that carry at most one update mean the stream connects but does not stream | +| Are the frames usable? | `cb_pbs_relay_stream_invalid_frames_total`: frames dropped on arrival because their message type or fork byte cannot be read, absent while zero. A bid that fails decoding or validation is not counted here | -Two things the series do not tell apart. The stream holds its newest 8 bids and returns the latest that passes validation, but a window in which every held bid fails decoding or validation still counts as `200`, the same as over HTTP; the validation error is in the logs. And a request the beacon node abandons mid-window records no outcome at all. +When the window closes, the stream serves the latest bid that passes decoding and validation, from the newest 8 it holds. A failing bid newer than the one served is not counted as a failure; the error is in the logs. A window in which every held bid fails counts as `200` on the status series, the same as an invalid bid over HTTP, and as one failed stream. A request the beacon node abandons mid-window records no outcome at all. diff --git a/tests/src/mock_ws_relay.rs b/tests/src/mock_ws_relay.rs index ea7364a17..8ab6b75ca 100644 --- a/tests/src/mock_ws_relay.rs +++ b/tests/src/mock_ws_relay.rs @@ -62,6 +62,13 @@ pub struct MockWsRelayState { /// Sign this many of the last bids for another parent hash, so they fail /// validation invalid_last_bids: usize, + /// Wait this long before answering the handshake, as a relay whose + /// handshake outlasts PBS's budget + handshake_delay: Duration, + /// Drop the connection after the last bid, with no close frame, as a + /// stream that breaks + abort_after_bids: bool, + handshake_attempts: AtomicU64, received_connections: AtomicU64, last_request: Mutex>, } @@ -76,6 +83,9 @@ impl MockWsRelayState { hold_open: false, unknown_frames: false, invalid_last_bids: 0, + handshake_delay: Duration::ZERO, + abort_after_bids: false, + handshake_attempts: AtomicU64::new(0), received_connections: AtomicU64::new(0), last_request: Mutex::new(None), } @@ -101,6 +111,20 @@ impl MockWsRelayState { Self { invalid_last_bids, ..self } } + pub fn with_handshake_delay(self, handshake_delay: Duration) -> Self { + Self { handshake_delay, ..self } + } + + pub fn abort_after_bids(self) -> Self { + Self { abort_after_bids: true, ..self } + } + + /// Connections that reached the stream, counted before the handshake, so + /// one that never completes still shows + pub fn handshake_attempts(&self) -> u64 { + self.handshake_attempts.load(Ordering::Relaxed) + } + pub fn received_connections(&self) -> u64 { self.received_connections.load(Ordering::Relaxed) } @@ -128,6 +152,10 @@ pub async fn start_mock_ws_relay_service( // The handshake callback's Err type is fixed by tungstenite #[allow(clippy::result_large_err)] async fn serve_stream(state: Arc, stream: TcpStream) -> eyre::Result<()> { + state.handshake_attempts.fetch_add(1, Ordering::Relaxed); + if !state.handshake_delay.is_zero() { + tokio::time::sleep(state.handshake_delay).await; + } let mut request = None; let mut ws = accept_hdr_async(stream, |req: &Request, res: Response| { request = parse_request(req); @@ -161,6 +189,10 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: } } + if state.abort_after_bids { + return Ok(()); + } + if state.hold_open { // Never resolves: PBS has to cut the stream off at its own deadline std::future::pending::<()>().await; @@ -195,3 +227,37 @@ fn parse_request(req: &Request) -> Option { fn header(req: &Request, name: &str) -> Option { req.headers().get(name)?.to_str().ok().map(ToOwned::to_owned) } + +/// Serves a get_header stream relay and an HTTP relay on one listener, as a +/// real relay does: a connection whose request asks for a websocket upgrade +/// goes to the stream, any other is forwarded byte for byte to the HTTP relay +/// at `http_addr`. +pub async fn start_mock_dual_relay_service( + ws_state: Arc, + listener: TcpListener, + http_addr: std::net::SocketAddr, +) -> eyre::Result<()> { + loop { + let (stream, addr) = listener.accept().await?; + let ws_state = ws_state.clone(); + tokio::spawn(async move { + let res = async { + let mut head = [0u8; 2048]; + let n = stream.peek(&mut head).await?; + let head = String::from_utf8_lossy(&head[..n]).to_ascii_lowercase(); + if head.contains("upgrade: websocket") { + serve_stream(ws_state, stream).await + } else { + let mut stream = stream; + let mut upstream = TcpStream::connect(http_addr).await?; + tokio::io::copy_bidirectional(&mut stream, &mut upstream).await?; + Ok(()) + } + } + .await; + if let Err(err) = res { + debug!(%addr, %err, "mock dual relay connection ended"); + } + }); + } +} diff --git a/tests/tests/pbs_get_header_ws.rs b/tests/tests/pbs_get_header_ws.rs index ee958d7ab..22315e44a 100644 --- a/tests/tests/pbs_get_header_ws.rs +++ b/tests/tests/pbs_get_header_ws.rs @@ -17,7 +17,7 @@ use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use cb_tests::{ mock_relay::{MockRelayState, start_mock_relay_service_with_listener}, mock_validator::MockValidator, - mock_ws_relay::{MockWsRelayState, start_mock_ws_relay_service}, + mock_ws_relay::{MockWsRelayState, start_mock_dual_relay_service, start_mock_ws_relay_service}, utils::{ API_KEY, generate_mock_relay, generate_mock_stream_relay, generate_mock_stream_relay_with_timing_games, get_free_listener, get_pbs_config, @@ -324,38 +324,10 @@ async fn test_get_header_ws_unreachable_relay_returns_204() -> Result<()> { Ok(()) } -/// A relay configured to stream that rejects the handshake โ€” here by not -/// serving the stream route at all โ€” is still asked over HTTP, so its bid stays -/// in the auction instead of the relay silently dropping out. +/// The HTTP request raced alongside the stream is the relay's normal +/// get_header, timing games included. #[tokio::test] -async fn test_get_header_ws_falls_back_to_http() -> Result<()> { - setup_test_env(); - let signer = random_secret(); - let pubkey = signer.public_key(); - let chain = Chain::Hoodi; - - let listener = get_free_listener().await; - let port = listener.local_addr()?.port(); - let relay_state = - Arc::new(MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(42))); - tokio::spawn(start_mock_relay_service_with_listener(relay_state.clone(), listener)); - - let relay = generate_mock_stream_relay(port, pubkey)?; - let validator = start_pbs(chain, vec![relay], 1_000).await?; - - let (code, res) = get_header_json(&validator).await?; - assert_eq!(code, StatusCode::OK); - assert_bid(&res.unwrap(), chain, &signer, U256::from(42)); - - // Timing games are off for this relay, so the fallback is a single request - assert_eq!(relay_state.received_get_header(), 1); - Ok(()) -} - -/// The handshake can fail early in the slot, so the fallback is a normal http -/// get_header and still plays this relay's timing games. -#[tokio::test] -async fn test_get_header_ws_fallback_runs_timing_games() -> Result<()> { +async fn test_get_header_ws_race_http_runs_timing_games() -> Result<()> { setup_test_env(); let signer = random_secret(); let pubkey = signer.public_key(); @@ -376,7 +348,7 @@ async fn test_get_header_ws_fallback_runs_timing_games() -> Result<()> { assert_bid(&res.unwrap(), chain, &signer, U256::from(10)); let n_requests = relay_state.received_get_header(); - assert!(n_requests > 1, "fallback skipped the timing games loop: {n_requests} requests"); + assert!(n_requests > 1, "HTTP skipped the timing games loop: {n_requests} requests"); Ok(()) } @@ -412,3 +384,98 @@ async fn test_get_header_ws_wins_auction_against_http() -> Result<()> { assert_eq!(stream_state.received_connections(), 1); Ok(()) } + +/// One relay serving both the HTTP `get_header` and the stream on the same +/// port, as a real relay does. Returns the two mock states and the stream-mode +/// relay client. +async fn start_dual_relay( + http_state: MockRelayState, + ws_state: MockWsRelayState, + pubkey: cb_common::types::BlsPublicKey, +) -> Result<(Arc, Arc, cb_common::pbs::RelayClient)> { + let http_listener = get_free_listener().await; + let http_addr = http_listener.local_addr()?; + let http_state = Arc::new(http_state); + tokio::spawn(start_mock_relay_service_with_listener(http_state.clone(), http_listener)); + + let listener = get_free_listener().await; + let port = listener.local_addr()?.port(); + let ws_state = Arc::new(ws_state); + tokio::spawn(start_mock_dual_relay_service(ws_state.clone(), listener, http_addr)); + + Ok((http_state, ws_state, generate_mock_stream_relay(port, pubkey)?)) +} + +/// A handshake that outlasts the budget: the HTTP request raced alongside it +/// supplies the bid. +#[tokio::test] +async fn test_get_header_ws_race_handshake_timeout_http_bid_serves() -> Result<()> { + setup_test_env(); + let signer = random_secret(); + let chain = Chain::Hoodi; + let (http_state, ws_state, relay) = start_dual_relay( + MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(42)), + MockWsRelayState::new(chain, signer.clone()).with_handshake_delay(Duration::from_secs(2)), + signer.public_key(), + ) + .await?; + let validator = start_pbs(chain, vec![relay], 300).await?; + + let (code, res) = get_header_json(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert_bid(&res.unwrap(), chain, &signer, U256::from(42)); + assert_eq!(http_state.received_get_header(), 1); + assert_eq!(ws_state.handshake_attempts(), 1); + assert_eq!(ws_state.received_connections(), 0, "the handshake never completed"); + Ok(()) +} + +/// A stream that breaks after a bid keeps that bid, here above the HTTP one. +#[tokio::test] +async fn test_get_header_ws_race_stream_break_keeps_held_bid() -> Result<()> { + setup_test_env(); + let signer = random_secret(); + let chain = Chain::Hoodi; + let (_, ws_state, relay) = start_dual_relay( + MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(10)), + MockWsRelayState::new(chain, signer.clone()) + .with_bid_values(vec![U256::from(50)]) + .abort_after_bids(), + signer.public_key(), + ) + .await?; + let validator = start_pbs(chain, vec![relay], 1_000).await?; + + let (code, res) = get_header_json(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert_bid(&res.unwrap(), chain, &signer, U256::from(50)); + assert_eq!(ws_state.received_connections(), 1); + Ok(()) +} + +/// The stream and the HTTP request are separate candidates: the higher bid +/// wins, and the stream's candidate is its latest bid, not its highest. +#[tokio::test] +async fn test_get_header_ws_race_higher_bid_wins() -> Result<()> { + setup_test_env(); + let signer = random_secret(); + let chain = Chain::Hoodi; + + for (http_bid, stream_bids, served) in [(10, [20, 50], 50), (60, [70, 50], 60)] { + let (http_state, ws_state, relay) = start_dual_relay( + MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(http_bid)), + MockWsRelayState::new(chain, signer.clone()) + .with_bid_values(stream_bids.map(U256::from).to_vec()), + signer.public_key(), + ) + .await?; + let validator = start_pbs(chain, vec![relay], 1_000).await?; + + let (code, res) = get_header_json(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert_bid(&res.unwrap(), chain, &signer, U256::from(served)); + assert_eq!(http_state.received_get_header(), 1); + assert_eq!(ws_state.received_connections(), 1); + } + Ok(()) +} From eaa9d6044a674c87f9a81105058e12f0f3003876 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Sun, 4 Oct 2026 22:10:20 -0700 Subject: [PATCH 05/24] fix(pbs): ask the bid stream to end one downlink trip before the deadline A relay honouring the timing headers ends the stream at Date-Milliseconds + X-Timeout-Ms, which CB set to its own read deadline, so every frame sent in the last one-way latency arrived after CB stopped reading. Measured against Helix with 7 rising frames: at 100 ms one-way 2 frames were lost, at 200 ms 4, and at 300 ms or more all 7, so HTTP served a 1.1 s old bid. The stream now opens TCP first and takes the connect time as the round trip (SYN to SYN-ACK, no clock skew), stamps Date-Milliseconds then, and sends X-Timeout-Ms as the time left to CB's deadline less half that round trip, before completing TLS and the upgrade on the same connection. CB's own deadline is unchanged; with nothing left after the connect, the stream times out as before. The reader adds the timing headers itself, so the ePBS stream gets the same behaviour. --- crates/pbs/src/bid_stream.rs | 113 +++++++++++++++++----- crates/pbs/src/mev_boost/get_header_ws.rs | 2 +- tests/src/mock_ws_relay.rs | 79 ++++++++++++++- tests/tests/pbs_get_header_ws.rs | 66 ++++++++++++- 4 files changed, 228 insertions(+), 32 deletions(-) diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs index 4c961a0a6..4550598ff 100644 --- a/crates/pbs/src/bid_stream.rs +++ b/crates/pbs/src/bid_stream.rs @@ -10,11 +10,12 @@ use std::{ collections::VecDeque, + io, sync::{Arc, OnceLock}, time::Duration, }; -use axum::http::{HeaderMap, HeaderValue, Request, header::USER_AGENT}; +use axum::http::{HeaderMap, HeaderValue, Request, Uri, header::USER_AGENT}; use cb_common::{ pbs::{ForkName, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, error::PbsError}, utils::utcnow_ms, @@ -22,11 +23,15 @@ use cb_common::{ use futures::StreamExt; use reqwest::StatusCode; use rustls::{ClientConfig, RootCertStore, crypto::aws_lc_rs}; -use tokio::time::{Instant, sleep_until, timeout_at}; +use tokio::{ + net::{TcpStream, lookup_host}, + time::{Instant, sleep_until, timeout_at}, +}; use tokio_tungstenite::{ - Connector, connect_async_tls_with_config, + Connector, MaybeTlsStream, WebSocketStream, client_async_tls_with_config, tungstenite::{ - Bytes, Error as WsError, Message, client::IntoClientRequest, protocol::WebSocketConfig, + Bytes, Error as WsError, Message, client::IntoClientRequest, error::UrlError, + protocol::WebSocketConfig, }, }; use tracing::{debug, warn}; @@ -67,12 +72,12 @@ pub(crate) struct Held { pub(crate) invalid_frames: usize, } -/// The handshake for `url`, with the same headers an HTTP request carries +/// The handshake for `url`, with the same headers an HTTP request carries. +/// `read_bid_stream` adds the timing headers once connected. pub(crate) fn handshake_request( url: &Url, relay: &RelayClient, send_headers: &HeaderMap, - timeout_ms: u64, ) -> Result, PbsError> { let mut request = url .as_str() @@ -88,9 +93,6 @@ pub(crate) fn handshake_request( headers.insert(key, value.clone()); } - headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); - headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); - Ok(request) } @@ -105,19 +107,9 @@ pub(crate) async fn read_bid_stream( endpoint: &str, mut parse: impl FnMut(Frame) -> Result, ) -> Result, (StatusCode, PbsError)> { - let config = WebSocketConfig::default() - .max_message_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)) - .max_frame_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)); - let start_request = Instant::now(); - let connect = connect_async_tls_with_config( - request, - Some(config), - true, - Some(Connector::Rustls(tls_config().clone())), - ); - let (mut stream, _) = match timeout_at(deadline, connect).await { - Ok(Ok(connected)) => connected, + let mut stream = match timeout_at(deadline, connect(request, deadline)).await { + Ok(Ok(Some(stream))) => stream, // A relay with no bid can answer the handshake as it answers get_header Ok(Err(WsError::Http(res))) if res.status() == StatusCode::NO_CONTENT => { return Ok(Held { @@ -129,7 +121,7 @@ pub(crate) async fn read_bid_stream( }); } Ok(Err(err)) => return Err(connect_failed(&err)), - Err(_) => return Err((TIMEOUT_ERROR_STATUS, PbsError::WebSocketTimeout)), + Ok(Ok(None)) | Err(_) => return Err((TIMEOUT_ERROR_STATUS, PbsError::WebSocketTimeout)), }; let connect_latency = start_request.elapsed(); RELAY_STREAM_CONNECT_LATENCY @@ -200,6 +192,73 @@ pub(crate) async fn read_bid_stream( Ok(Held { frames, updates, connect_latency, first_frame_latency, invalid_frames }) } +/// Opens TCP first, so the handshake can ask the relay to end its stream one +/// downlink trip before `deadline`: a relay honouring the timing headers ends +/// it at Date-Milliseconds + X-Timeout-Ms, and a frame sent then still has to +/// travel. None when no time would be left. +async fn connect( + mut request: Request<()>, + deadline: Instant, +) -> Result>>, WsError> { + let (socket, rtt) = connect_tcp(request.uri()).await?; + + let Some(timeout_ms) = + relay_timeout_ms(deadline.saturating_duration_since(Instant::now()), rtt) + else { + return Ok(None); + }; + let headers = request.headers_mut(); + headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); + headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); + + let config = WebSocketConfig::default() + .max_message_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)) + .max_frame_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)); + let connector = Connector::Rustls(tls_config().clone()); + let (stream, _) = + client_async_tls_with_config(request, socket, Some(config), Some(connector)).await?; + + Ok(Some(stream)) +} + +/// The TCP handshake's duration is one round trip, free of clock skew. Name +/// resolution is left out of it. +async fn connect_tcp(uri: &Uri) -> Result<(TcpStream, Duration), WsError> { + let host = uri.host().ok_or(WsError::Url(UrlError::NoHostName))?; + let host = host.strip_prefix('[').and_then(|host| host.strip_suffix(']')).unwrap_or(host); + let port = uri + .port_u16() + .or(match uri.scheme_str() { + Some("wss") => Some(443), + Some("ws") => Some(80), + _ => None, + }) + .ok_or(WsError::Url(UrlError::UnsupportedUrlScheme))?; + + let mut last_err = None; + for addr in lookup_host((host, port)).await? { + let start = Instant::now(); + match TcpStream::connect(addr).await { + Ok(socket) => { + let rtt = start.elapsed(); + socket.set_nodelay(true)?; + return Ok((socket, rtt)); + } + Err(err) => last_err = Some(err), + } + } + + Err(WsError::Io(last_err.unwrap_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "could not resolve to any address") + }))) +} + +/// The time left less half the round trip, the estimate of one downlink trip +fn relay_timeout_ms(time_left: Duration, rtt: Duration) -> Option { + let timeout_ms = time_left.saturating_sub(rtt / 2).as_millis() as u64; + (timeout_ms > 0).then_some(timeout_ms) +} + pub(crate) fn record_stream_fallback(endpoint: &str, relay_id: &str) { RELAY_STREAM_FALLBACK.with_label_values(&[endpoint, relay_id]).inc(); } @@ -303,6 +362,16 @@ mod tests { assert_eq!(fork_from_wire(forks.len() as u8), None); } + #[test] + fn test_relay_timeout_leaves_the_downlink_trip() { + let ms = Duration::from_millis; + assert_eq!(relay_timeout_ms(ms(1_000), ms(100)), Some(950)); + assert_eq!(relay_timeout_ms(ms(1_000), Duration::ZERO), Some(1_000)); + // Nothing left: no frame could land before the deadline + assert_eq!(relay_timeout_ms(ms(50), ms(100)), None); + assert_eq!(relay_timeout_ms(ms(40), ms(100)), None); + } + #[test] fn test_parse_frame() { assert!(matches!( diff --git a/crates/pbs/src/mev_boost/get_header_ws.rs b/crates/pbs/src/mev_boost/get_header_ws.rs index 5a8fcfd10..52a7b5f25 100644 --- a/crates/pbs/src/mev_boost/get_header_ws.rs +++ b/crates/pbs/src/mev_boost/get_header_ws.rs @@ -43,7 +43,7 @@ async fn stream_header( timeout_ms: u64, ) -> StreamOutcome { let deadline = Instant::now() + Duration::from_millis(timeout_ms); - let request = match handshake_request(&url, relay, &request_info.headers, timeout_ms) { + let request = match handshake_request(&url, relay, &request_info.headers) { Ok(request) => request, Err(err) => return (TRANSPORT_ERROR_STATUS, Err(err)), }; diff --git a/tests/src/mock_ws_relay.rs b/tests/src/mock_ws_relay.rs index 8ab6b75ca..eeec147fb 100644 --- a/tests/src/mock_ws_relay.rs +++ b/tests/src/mock_ws_relay.rs @@ -14,10 +14,15 @@ use cb_common::{ HEADER_VERSION_KEY, }, types::{BlsSecretKey, Chain}, + utils::utcnow_ms, }; use futures::SinkExt; use ssz::Encode; -use tokio::net::{TcpListener, TcpStream}; +use tokio::{ + net::{TcpListener, TcpSocket, TcpStream}, + sync::oneshot, + time::timeout, +}; use tokio_tungstenite::{ accept_hdr_async, tungstenite::{ @@ -43,6 +48,8 @@ pub struct StreamRequest { pub user_agent: Option, pub cb_version: Option, pub api_key: Option, + /// When the relay accepted the connection, unix ms + pub accepted_at_ms: u64, } pub struct MockWsRelayState { @@ -68,6 +75,10 @@ pub struct MockWsRelayState { /// Drop the connection after the last bid, with no close frame, as a /// stream that breaks abort_after_bids: bool, + /// End the stream at Date-Milliseconds + X-Timeout-Ms, as a relay honouring + /// the timing headers does: the last bid goes out then, the earlier ones + /// `update_interval` apart before it, and each reaches PBS this much later + ends_at_timeout: Option, handshake_attempts: AtomicU64, received_connections: AtomicU64, last_request: Mutex>, @@ -85,6 +96,7 @@ impl MockWsRelayState { invalid_last_bids: 0, handshake_delay: Duration::ZERO, abort_after_bids: false, + ends_at_timeout: None, handshake_attempts: AtomicU64::new(0), received_connections: AtomicU64::new(0), last_request: Mutex::new(None), @@ -119,6 +131,10 @@ impl MockWsRelayState { Self { abort_after_bids: true, ..self } } + pub fn ends_at_timeout(self, downlink_latency: Duration) -> Self { + Self { ends_at_timeout: Some(downlink_latency), ..self } + } + /// Connections that reached the stream, counted before the handshake, so /// one that never completes still shows pub fn handshake_attempts(&self) -> u64 { @@ -152,13 +168,14 @@ pub async fn start_mock_ws_relay_service( // The handshake callback's Err type is fixed by tungstenite #[allow(clippy::result_large_err)] async fn serve_stream(state: Arc, stream: TcpStream) -> eyre::Result<()> { + let accepted_at_ms = utcnow_ms(); state.handshake_attempts.fetch_add(1, Ordering::Relaxed); if !state.handshake_delay.is_zero() { tokio::time::sleep(state.handshake_delay).await; } let mut request = None; let mut ws = accept_hdr_async(stream, |req: &Request, res: Response| { - request = parse_request(req); + request = parse_request(req, accepted_at_ms); Ok(res) }) .await?; @@ -167,7 +184,26 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: state.received_connections.fetch_add(1, Ordering::Relaxed); *state.last_request.lock().unwrap() = Some(request.clone()); + let stream_end_ms = match state.ends_at_timeout { + Some(_) => Some( + request + .start_time_ms + .zip(request.timeout_ms) + .map(|(start, timeout)| start + timeout) + .ok_or_else(|| eyre::eyre!("no timing headers"))?, + ), + None => None, + }; + for (i, value) in state.bid_values.iter().enumerate() { + if let (Some(end_ms), Some(latency)) = (stream_end_ms, state.ends_at_timeout) { + let sent_ms = end_ms - + state.update_interval.as_millis() as u64 * + (state.bid_values.len() - 1 - i) as u64; + let arrives_ms = sent_ms + latency.as_millis() as u64; + tokio::time::sleep(Duration::from_millis(arrives_ms.saturating_sub(utcnow_ms()))).await; + } + if state.unknown_frames { // Unknown message type, unknown fork, truncated prefix for frame in [vec![0x7f, FORK_FULU, 1], vec![MSG_BID, 0xff, 1], vec![MSG_BID]] { @@ -184,7 +220,7 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: frame.extend_from_slice(&bid.as_ssz_bytes()); ws.send(Message::Binary(frame.into())).await?; - if !state.update_interval.is_zero() { + if stream_end_ms.is_none() && !state.update_interval.is_zero() { tokio::time::sleep(state.update_interval).await; } } @@ -205,7 +241,7 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: /// The request is the handshake: the fixed stream path followed by /// `/{slot}/{parent_hash}/{pubkey}`, plus the same headers the HTTP path sends. -fn parse_request(req: &Request) -> Option { +fn parse_request(req: &Request, accepted_at_ms: u64) -> Option { let prefix = format!("{}{GET_HEADER_STREAM_PATH}/", BuilderApiVersion::V1.path()); let mut segments = req.uri().path().strip_prefix(&prefix)?.split('/'); let slot = segments.next()?.parse().ok()?; @@ -221,6 +257,7 @@ fn parse_request(req: &Request) -> Option { user_agent: header(req, "user-agent"), cb_version: header(req, HEADER_VERSION_KEY), api_key: header(req, HEADER_API_KEY), + accepted_at_ms, }) } @@ -228,6 +265,40 @@ fn header(req: &Request, name: &str) -> Option { req.headers().get(name)?.to_str().ok().map(ToOwned::to_owned) } +/// Starts the stream relay with its accept queue full until `drain` fires. A +/// client's SYN is dropped meanwhile, so its connect completes on the SYN +/// retransmit, about a second after it began. On loopback that is the only way +/// to slow a TCP connect from user space: a delayed accept or a proxy hop does +/// not delay the SYN-ACK. Returns the port. +pub async fn start_mock_ws_relay_with_full_backlog( + state: Arc, + drain: oneshot::Receiver<()>, +) -> eyre::Result { + let socket = TcpSocket::new_v4()?; + socket.bind(([127, 0, 0, 1], 0).into())?; + let listener = socket.listen(1)?; + let addr = listener.local_addr()?; + + // How many connections the queue admits is up to the OS: fill it until one + // does not complete + let mut fillers = Vec::new(); + while let Ok(filler) = timeout(Duration::from_millis(200), TcpStream::connect(addr)).await { + fillers.push(filler?); + eyre::ensure!(fillers.len() < 16, "accept queue never filled"); + } + + tokio::spawn(async move { + let _ = drain.await; + for _ in &fillers { + listener.accept().await?; + } + drop(fillers); + start_mock_ws_relay_service(state, listener).await + }); + + Ok(addr.port()) +} + /// Serves a get_header stream relay and an HTTP relay on one listener, as a /// real relay does: a connection whose request asks for a websocket upgrade /// goes to the stream, any other is forwarded byte for byte to the HTTP relay diff --git a/tests/tests/pbs_get_header_ws.rs b/tests/tests/pbs_get_header_ws.rs index 22315e44a..858f4daa4 100644 --- a/tests/tests/pbs_get_header_ws.rs +++ b/tests/tests/pbs_get_header_ws.rs @@ -10,14 +10,17 @@ use cb_common::{ signature::sign_builder_root, signer::random_secret, types::{BlsPublicKeyBytes, BlsSecretKey, Chain, KnownChain}, - utils::timestamp_of_slot_start_sec, + utils::{timestamp_of_slot_start_sec, utcnow_ms}, wire::EncodingType, }; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use cb_tests::{ mock_relay::{MockRelayState, start_mock_relay_service_with_listener}, mock_validator::MockValidator, - mock_ws_relay::{MockWsRelayState, start_mock_dual_relay_service, start_mock_ws_relay_service}, + mock_ws_relay::{ + MockWsRelayState, start_mock_dual_relay_service, start_mock_ws_relay_service, + start_mock_ws_relay_with_full_backlog, + }, utils::{ API_KEY, generate_mock_relay, generate_mock_stream_relay, generate_mock_stream_relay_with_timing_games, get_free_listener, get_pbs_config, @@ -27,6 +30,7 @@ use cb_tests::{ use eyre::Result; use lh_types::ForkName; use reqwest::StatusCode; +use tokio::sync::oneshot; use tree_hash::TreeHash; fn request_slot() -> u64 { @@ -227,7 +231,7 @@ async fn test_get_header_ws_handshake_carries_request() -> Result<()> { start_stream_relay(MockWsRelayState::new(chain, signer.clone()), pubkey).await?; let validator = start_pbs(chain, vec![relay], timeout_ms).await?; - let sent_at = cb_common::utils::utcnow_ms(); + let sent_at = utcnow_ms(); let (code, _) = get_header_json(&validator).await?; assert_eq!(code, StatusCode::OK); @@ -236,8 +240,13 @@ async fn test_get_header_ws_handshake_carries_request() -> Result<()> { assert_eq!(request.parent_hash, B256::ZERO); assert!(request.validator_pubkey.starts_with("0x")); - // No timeout header from the caller, so PBS passes its own budget through - assert_eq!(request.timeout_ms, Some(timeout_ms)); + // No timeout header from the caller, so PBS asks for its own budget, less + // what the connect took + let relay_timeout_ms = request.timeout_ms.expect("missing timeout header"); + assert!( + (timeout_ms - 100..=timeout_ms).contains(&relay_timeout_ms), + "timeout header {relay_timeout_ms}ms" + ); let start_time_ms = request.start_time_ms.expect("missing start time header"); assert!((sent_at..sent_at + timeout_ms).contains(&start_time_ms)); @@ -280,6 +289,53 @@ async fn test_get_header_ws_returns_at_deadline() -> Result<()> { Ok(()) } +/// A relay that ends its stream at Date-Milliseconds + X-Timeout-Ms sends its +/// last bid then, and each frame takes one downlink trip to arrive. PBS asks it +/// to end half the TCP connect time before PBS's own deadline, so that bid +/// still lands. +#[tokio::test] +async fn test_get_header_ws_last_bid_lands_before_deadline() -> Result<()> { + setup_test_env(); + let signer = random_secret(); + let chain = Chain::Hoodi; + let timeout_ms = 3_000; + + let relay_state = Arc::new( + MockWsRelayState::new(chain, signer.clone()) + .with_bid_values(vec![U256::from(10), U256::from(20)]) + .with_update_interval(Duration::from_millis(400)) + .ends_at_timeout(Duration::from_millis(200)), + ); + let (drain, drained) = oneshot::channel(); + let port = start_mock_ws_relay_with_full_backlog(relay_state.clone(), drained).await?; + let relay = generate_mock_stream_relay(port, signer.public_key())?; + let validator = start_pbs(chain, vec![relay], timeout_ms).await?; + + let requested_at_ms = utcnow_ms(); + let (res, ()) = tokio::join!(get_header_json(&validator), async { + // After PBS's first SYN, before its retransmit + tokio::time::sleep(Duration::from_millis(500)).await; + let _ = drain.send(()); + }); + let (code, res) = res?; + + assert_eq!(code, StatusCode::OK); + assert_bid(&res.unwrap(), chain, &signer, U256::from(20)); + + let request = relay_state.last_request().expect("relay saw no request"); + let connect_ms = request.accepted_at_ms - requested_at_ms; + assert!(connect_ms >= 500, "the connect was not delayed: {connect_ms}ms"); + + let relay_end_ms = request.start_time_ms.unwrap() + request.timeout_ms.unwrap(); + let early_ms = (requested_at_ms + timeout_ms) as i64 - relay_end_ms as i64; + let half_connect_ms = connect_ms as i64 / 2; + assert!( + (half_connect_ms - 100..=half_connect_ms + 20).contains(&early_ms), + "relay asked to end {early_ms}ms before the deadline, connect took {connect_ms}ms" + ); + Ok(()) +} + /// A stream that never delivers a bid is a 204, same as an HTTP relay with no /// header for the slot. #[tokio::test] From 2edf6617eca0d92555d228a8a072876c61cd1fb5 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 5 Oct 2026 10:55:45 -0700 Subject: [PATCH 06/24] chore(deps): bump lighthouse to rev 31d8cfd for the gloas types Pin the lighthouse crates to sigp/lighthouse unstable at 31d8cfd, whose gloas containers hash as EIP-7495 progressive containers, matching the consensus specs. Mirror lighthouse's own [patch.crates-io] entries for the progressive ssz stack (a consumer does not inherit a dependency's patches) and pin the blstrs_plus patch to an explicit rev. lighthouse dropped `TestRandom` for an `arbitrary` generator, so `TestRandomSeed` now draws from `arbitrary::Arbitrary`. Validated BLS points do not randomize under `arbitrary`, so tests that need a real key use `BlsSecretKey::random()`. Also adapt to `ForkName::Heze` and to `ExecutionRequests` no longer implementing `ssz::Decode`. --- Cargo.lock | 687 ++++++++++++---------- Cargo.toml | 22 +- crates/common/Cargo.toml | 1 + crates/common/src/config/utils.rs | 7 +- crates/common/src/signature.rs | 9 +- crates/common/src/ssz.rs | 13 +- crates/common/src/utils.rs | 17 +- crates/pbs/src/mev_boost/get_header.rs | 4 +- crates/pbs/src/mev_boost/get_header_ws.rs | 1 + tests/src/mock_relay.rs | 9 +- 10 files changed, 436 insertions(+), 334 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d797a53b2..0214e688b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,8 +245,6 @@ dependencies = [ "c-kzg", "derive_more", "either", - "ethereum_ssz 0.9.1", - "ethereum_ssz_derive 0.9.1", "serde", "serde_with", "sha2", @@ -367,6 +365,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5e9dbd49258ac3ab893a481d46be29b58be7f734dcac46cd80b6b13ee36566c" dependencies = [ "alloy-rlp", + "arbitrary", "bytes", "cfg-if", "const-hex", @@ -381,6 +380,7 @@ dependencies = [ "keccak-asm", "paste", "proptest", + "proptest-derive", "rand 0.9.5", "rapidhash", "ruint", @@ -557,14 +557,10 @@ dependencies = [ "alloy-primitives 1.7.3", "alloy-rpc-types-engine", "derive_more", - "ethereum_ssz 0.9.1", - "ethereum_ssz_derive 0.9.1", "serde", "serde_json", "serde_with", "thiserror 2.0.20", - "tree_hash 0.10.0", - "tree_hash_derive 0.10.0", ] [[package]] @@ -591,8 +587,6 @@ dependencies = [ "alloy-rlp", "alloy-serde", "derive_more", - "ethereum_ssz 0.9.1", - "ethereum_ssz_derive 0.9.1", "rand 0.8.8", "serde", "strum", @@ -1509,7 +1503,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.9", "object", "rustc-demangle", "windows-link", @@ -1644,20 +1638,20 @@ dependencies = [ [[package]] name = "bls" version = "0.2.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "alloy-primitives 1.7.3", "arbitrary", "blst", - "ethereum_hashing 0.8.0", + "ethereum_hashing", "ethereum_serde_utils 0.8.1", - "ethereum_ssz 0.10.4", + "ethereum_ssz", "fixed_bytes", "hex", "rand 0.9.5", "safe_arith", "serde", - "tree_hash 0.12.1", + "tree_hash", "zeroize", ] @@ -1719,7 +1713,7 @@ dependencies = [ [[package]] name = "blstrs_plus" version = "0.8.18" -source = "git+https://github.com/Commit-Boost/blstrs#c4ea6b21193886ee9849867397a62e9c243c1fbb" +source = "git+https://github.com/Commit-Boost/blstrs?rev=c4ea6b21193886ee9849867397a62e9c243c1fbb#c4ea6b21193886ee9849867397a62e9c243c1fbb" dependencies = [ "arrayref", "blst", @@ -1777,6 +1771,25 @@ dependencies = [ "serde_core", ] +[[package]] +name = "builder_types" +version = "0.1.0" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" +dependencies = [ + "bls", + "context_deserialize", + "ethereum_serde_utils 0.8.1", + "ethereum_ssz", + "ethereum_ssz_derive", + "sensitive_url", + "serde", + "ssz_types", + "tree_hash", + "tree_hash_derive", + "typenum", + "types", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -1819,6 +1832,38 @@ dependencies = [ "serde", ] +[[package]] +name = "camino" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbad30e4b4c14a39e3cc8aed085a12a327257c316619c93581e017bc52be591" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.28", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "cast" version = "0.3.0" @@ -1875,6 +1920,7 @@ version = "0.11.0" dependencies = [ "aes", "alloy", + "arbitrary", "async-trait", "axum 0.8.9", "base64 0.22.1", @@ -1889,8 +1935,8 @@ dependencies = [ "eth2", "eth2_keystore", "ethereum_serde_utils 0.7.0", - "ethereum_ssz 0.10.4", - "ethereum_ssz_derive 0.10.4", + "ethereum_ssz", + "ethereum_ssz_derive", "eyre", "futures", "headers-accept", @@ -1907,7 +1953,7 @@ dependencies = [ "serde_json", "serde_yaml", "sha2", - "ssz_types 0.11.0", + "ssz_types", "tempfile", "thiserror 2.0.20", "tokio", @@ -1916,12 +1962,12 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", - "tree_hash 0.12.1", - "tree_hash_derive 0.12.1", + "tree_hash", + "tree_hash_derive", "types", "unicode-normalization", "url", - "uuid 1.26.1", + "uuid", ] [[package]] @@ -1931,7 +1977,7 @@ dependencies = [ "axum 0.8.9", "cb-common", "eyre", - "prometheus", + "prometheus 0.14.0", "thiserror 2.0.20", "tokio", "tracing", @@ -1948,14 +1994,14 @@ dependencies = [ "cb-common", "cb-metrics", "ethereum_serde_utils 0.7.0", - "ethereum_ssz 0.10.4", + "ethereum_ssz", "eyre", "futures", "headers", "lazy_static", "notify", "parking_lot", - "prometheus", + "prometheus 0.14.0", "reqwest 0.13.5", "rustls", "serde", @@ -1965,10 +2011,10 @@ dependencies = [ "tokio-tungstenite", "tower-http", "tracing", - "tree_hash 0.12.1", + "tree_hash", "types", "url", - "uuid 1.26.1", + "uuid", "webpki-roots 1.0.9", ] @@ -1990,7 +2036,7 @@ dependencies = [ "jsonwebtoken", "lazy_static", "parking_lot", - "prometheus", + "prometheus 0.14.0", "prost", "rand 0.9.5", "rustls", @@ -1999,8 +2045,8 @@ dependencies = [ "tonic", "tonic-build", "tracing", - "tree_hash 0.12.1", - "uuid 1.26.1", + "tree_hash", + "uuid", ] [[package]] @@ -2013,7 +2059,7 @@ dependencies = [ "cb-pbs", "cb-signer", "eth2", - "ethereum_ssz 0.10.4", + "ethereum_ssz", "eyre", "futures", "jsonwebtoken", @@ -2028,7 +2074,7 @@ dependencies = [ "tracing", "tracing-subscriber", "tracing-test", - "tree_hash 0.12.1", + "tree_hash", "types", "url", ] @@ -2075,8 +2121,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -2238,8 +2286,8 @@ dependencies = [ "tempfile", "tokio", "tracing", - "tree_hash 0.12.1", - "tree_hash_derive 0.12.1", + "tree_hash", + "tree_hash_derive", ] [[package]] @@ -2395,6 +2443,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -2475,7 +2532,7 @@ dependencies = [ "crossterm_winapi", "document-features", "parking_lot", - "rustix", + "rustix 1.1.4", "winapi", ] @@ -2543,7 +2600,7 @@ dependencies = [ "commit-boost", "eyre", "lazy_static", - "prometheus", + "prometheus 0.14.0", "serde", "serde_json", "tokio", @@ -3113,14 +3170,16 @@ dependencies = [ [[package]] name = "eth2" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "bls", + "builder_types", "context_deserialize", "educe", "ethereum_serde_utils 0.8.1", - "ethereum_ssz 0.10.4", - "ethereum_ssz_derive 0.10.4", + "ethereum_ssz", + "ethereum_ssz_derive", + "fork_choice", "futures", "futures-util", "mediatype 0.19.20", @@ -3130,7 +3189,7 @@ dependencies = [ "sensitive_url", "serde", "serde_json", - "ssz_types 0.14.1", + "ssz_types", "superstruct", "types", ] @@ -3138,20 +3197,20 @@ dependencies = [ [[package]] name = "eth2_interop_keypairs" version = "0.2.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "bls", - "ethereum_hashing 0.8.0", + "ethereum_hashing", "hex", "num-bigint", "serde", - "serde_yaml", + "yaml_serde", ] [[package]] name = "eth2_key_derivation" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "bls", "num-bigint-dig", @@ -3163,7 +3222,7 @@ dependencies = [ [[package]] name = "eth2_keystore" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "aes", "bls", @@ -3180,21 +3239,10 @@ dependencies = [ "serde_repr", "sha2", "unicode-normalization", - "uuid 0.8.2", + "uuid", "zeroize", ] -[[package]] -name = "ethereum_hashing" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c853bd72c9e5787f8aafc3df2907c2ed03cff3150c3acd94e2e53a98ab70a8ab" -dependencies = [ - "cpufeatures 0.2.17", - "ring", - "sha2", -] - [[package]] name = "ethereum_hashing" version = "0.8.0" @@ -3232,28 +3280,13 @@ dependencies = [ "serde_json", ] -[[package]] -name = "ethereum_ssz" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dcddb2554d19cde19b099fadddde576929d7a4d0c1cd3512d1fd95cf174375c" -dependencies = [ - "alloy-primitives 1.7.3", - "ethereum_serde_utils 0.8.1", - "itertools 0.13.0", - "serde", - "serde_derive", - "smallvec", - "typenum", -] - [[package]] name = "ethereum_ssz" version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e462875ad8693755ea8913d6e905715c76ea4836e2254e18c9cf0f7a8f8c2a13" +source = "git+https://github.com/sigp/ethereum_ssz?rev=2059c21ba52cd3a7e39a8ad537012b761812a393#2059c21ba52cd3a7e39a8ad537012b761812a393" dependencies = [ "alloy-primitives 1.7.3", + "arbitrary", "context_deserialize", "ethereum_serde_utils 0.8.1", "itertools 0.14.0", @@ -3263,23 +3296,10 @@ dependencies = [ "typenum", ] -[[package]] -name = "ethereum_ssz_derive" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a657b6b3b7e153637dc6bdc6566ad9279d9ee11a15b12cfb24a2e04360637e9f" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "ethereum_ssz_derive" version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf022360bdbe9456eda5f35718a50476d5b2a0d51a97ed4eae27420737a6fba" +source = "git+https://github.com/sigp/ethereum_ssz?rev=2059c21ba52cd3a7e39a8ad537012b761812a393#2059c21ba52cd3a7e39a8ad537012b761812a393" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -3379,7 +3399,7 @@ dependencies = [ [[package]] name = "fixed_bytes" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "alloy-primitives 1.7.3", "safe_arith", @@ -3391,6 +3411,17 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + [[package]] name = "float-cmp" version = "0.10.0" @@ -3419,20 +3450,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +name = "fork_choice" +version = "0.1.0" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ - "foreign-types-shared", + "ethereum_ssz", + "ethereum_ssz_derive", + "fixed_bytes", + "logging", + "metrics", + "proto_array", + "state_processing", + "superstruct", + "tracing", + "types", ] -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -3935,22 +3968,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.20" @@ -4153,6 +4170,7 @@ version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ + "arbitrary", "equivalent", "hashbrown 0.17.1", "serde", @@ -4191,11 +4209,20 @@ dependencies = [ [[package]] name = "int_to_bytes" version = "0.2.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "bytes", ] +[[package]] +name = "integer-sqrt" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "276ec31bcb4a9ee45f58bec6f9ec700ae4cf4f4f8f2fa7e06cb406bd5ffdd770" +dependencies = [ + "num-traits", +] + [[package]] name = "interprocess" version = "2.4.4" @@ -4485,22 +4512,21 @@ dependencies = [ [[package]] name = "kzg" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "arbitrary", - "c-kzg", "educe", - "ethereum_hashing 0.8.0", + "ethereum_hashing", "ethereum_serde_utils 0.8.1", - "ethereum_ssz 0.10.4", - "ethereum_ssz_derive 0.10.4", + "ethereum_ssz", + "ethereum_ssz_derive", "hex", "rayon", "rust_eth_kzg", "serde", "serde_json", "tracing", - "tree_hash 0.12.1", + "tree_hash", ] [[package]] @@ -4524,6 +4550,18 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libyaml-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -4557,6 +4595,37 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "logging" +version = "0.2.0" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" +dependencies = [ + "chrono", + "logroller", + "metrics", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-appender", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "workspace_members", +] + +[[package]] +name = "logroller" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7525242cbd0624fe9b76dfebc1923a2d20c003e8c5d1a7eab1324bd8c0f615ac" +dependencies = [ + "chrono", + "flate2", + "regex", + "thiserror 1.0.69", +] + [[package]] name = "lru" version = "0.16.4" @@ -4631,10 +4700,10 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "merkle_proof" version = "0.2.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "alloy-primitives 1.7.3", - "ethereum_hashing 0.8.0", + "ethereum_hashing", "fixed_bytes", "safe_arith", ] @@ -4674,24 +4743,32 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "metrics" +version = "0.2.0" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" +dependencies = [ + "prometheus 0.13.4", +] + [[package]] name = "milhouse" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "259dd9da2ae5e0278b95da0b7ecef9c18c309d0a2d9e6db57ed33b9e8910c5e7" +source = "git+https://github.com/sigp/milhouse?rev=c70f128976ac0d60ea65a978dabd117a921c36ee#c70f128976ac0d60ea65a978dabd117a921c36ee" dependencies = [ "alloy-primitives 1.7.3", + "arbitrary", "context_deserialize", "educe", - "ethereum_hashing 0.8.0", - "ethereum_ssz 0.10.4", - "ethereum_ssz_derive 0.10.4", + "ethereum_hashing", + "ethereum_ssz", + "ethereum_ssz_derive", "itertools 0.13.0", "parking_lot", "rayon", "serde", "smallvec", - "tree_hash 0.12.1", + "tree_hash", "triomphe", "typenum", "vec_map", @@ -4718,6 +4795,16 @@ dependencies = [ "adler2", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.3" @@ -4763,23 +4850,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" -[[package]] -name = "native-tls" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nom" version = "7.1.3" @@ -4966,59 +5036,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" -[[package]] -name = "openssl" -version = "0.10.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" -dependencies = [ - "bitflags 2.13.2", - "cfg-if", - "foreign-types", - "libc", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "openssl-src" -version = "300.6.1+3.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" -dependencies = [ - "cc", -] - -[[package]] -name = "openssl-sys" -version = "0.9.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" -dependencies = [ - "cc", - "libc", - "openssl-src", - "pkg-config", - "vcpkg", -] - [[package]] name = "owo-colors" version = "4.4.0" @@ -5313,7 +5336,7 @@ dependencies = [ [[package]] name = "pretty_reqwest_error" version = "0.1.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "reqwest 0.12.28", "sensitive_url", @@ -5380,6 +5403,45 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +dependencies = [ + "bitflags 2.13.2", + "hex", + "lazy_static", + "procfs-core", + "rustix 0.38.44", +] + +[[package]] +name = "procfs-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +dependencies = [ + "bitflags 2.13.2", + "hex", +] + +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs", + "thiserror 1.0.69", +] + [[package]] name = "prometheus" version = "0.14.0" @@ -5414,6 +5476,17 @@ dependencies = [ "unarray", ] +[[package]] +name = "proptest-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57924a81864dddafba92e1bf92f9bf82f97096c44489548a60e888e1547549b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "prost" version = "0.13.5" @@ -5466,6 +5539,23 @@ dependencies = [ "prost", ] +[[package]] +name = "proto_array" +version = "0.2.0" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" +dependencies = [ + "ethereum_ssz", + "ethereum_ssz_derive", + "fixed_bytes", + "safe_arith", + "serde", + "smallvec", + "superstruct", + "typenum", + "types", + "yaml_serde", +] + [[package]] name = "protobuf" version = "3.7.2" @@ -5804,11 +5894,9 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", - "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -5819,7 +5907,6 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower 0.5.3", @@ -5942,6 +6029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5e99bff0393163bb25029a6af25d3d8d202ba5b5438a74d1bd8789f5c822970" dependencies = [ "alloy-rlp", + "arbitrary", "ark-ff 0.3.0", "ark-ff 0.4.2", "ark-ff 0.5.0", @@ -6023,6 +6111,19 @@ dependencies = [ "semver 1.0.28", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -6032,7 +6133,7 @@ dependencies = [ "bitflags 2.13.2", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -6313,6 +6414,10 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "semver-parser" @@ -6577,6 +6682,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simd_cesu8" version = "1.2.0" @@ -6611,7 +6722,6 @@ version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" dependencies = [ - "arbitrary", "serde", ] @@ -6651,37 +6761,21 @@ dependencies = [ "der", ] -[[package]] -name = "ssz_types" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b55bedc9a18ed2860a46d6beb4f4082416ee1d60be0cc364cebdcdddc7afd4" -dependencies = [ - "ethereum_serde_utils 0.8.1", - "ethereum_ssz 0.9.1", - "itertools 0.13.0", - "serde", - "serde_derive", - "smallvec", - "tree_hash 0.10.0", - "typenum", -] - [[package]] name = "ssz_types" version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d625e4de8e0057eefe7e0b1510ba1dd7adf10cd375fad6cc7fcceac7c39623c9" +source = "git+https://github.com/sigp/ssz_types?rev=9203d56ad2d7bc5f12133d043e085843f81edcd6#9203d56ad2d7bc5f12133d043e085843f81edcd6" dependencies = [ + "arbitrary", "context_deserialize", "educe", "ethereum_serde_utils 0.8.1", - "ethereum_ssz 0.10.4", + "ethereum_ssz", "itertools 0.14.0", "serde", "serde_derive", "smallvec", - "tree_hash 0.12.1", + "tree_hash", "typenum", ] @@ -6691,6 +6785,34 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "state_processing" +version = "0.2.0" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" +dependencies = [ + "bls", + "educe", + "ethereum_hashing", + "ethereum_ssz", + "ethereum_ssz_derive", + "fixed_bytes", + "int_to_bytes", + "integer-sqrt", + "itertools 0.14.0", + "merkle_proof", + "metrics", + "milhouse", + "rand 0.9.5", + "rayon", + "safe_arith", + "smallvec", + "ssz_types", + "tracing", + "tree_hash", + "typenum", + "types", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -6707,7 +6829,7 @@ dependencies = [ "commit-boost", "eyre", "lazy_static", - "prometheus", + "prometheus 0.14.0", "reqwest 0.13.5", "serde", "tokio", @@ -6764,10 +6886,10 @@ dependencies = [ [[package]] name = "swap_or_not_shuffle" version = "0.2.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "alloy-primitives 1.7.3", - "ethereum_hashing 0.8.0", + "ethereum_hashing", "fixed_bytes", ] @@ -6878,7 +7000,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -6888,15 +7010,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" -[[package]] -name = "test_random_derive" -version = "0.2.0" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" -dependencies = [ - "quote", - "syn 2.0.119", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -7068,16 +7181,6 @@ dependencies = [ "syn 3.0.5", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.5" @@ -7433,49 +7536,22 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "tree_hash" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee44f4cef85f88b4dea21c0b1f58320bdf35715cf56d840969487cff00613321" -dependencies = [ - "alloy-primitives 1.7.3", - "ethereum_hashing 0.7.0", - "ethereum_ssz 0.9.1", - "smallvec", - "typenum", -] - [[package]] name = "tree_hash" version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fd51aa83d2eb83b04570808430808b5d24fdbf479a4d5ac5dee4a2e2dd2be4" +source = "git+https://github.com/sigp/tree_hash?rev=03d9fa474586c125f306dd7a00cf46284575a01f#03d9fa474586c125f306dd7a00cf46284575a01f" dependencies = [ "alloy-primitives 1.7.3", - "ethereum_hashing 0.8.0", - "ethereum_ssz 0.10.4", + "ethereum_hashing", + "ethereum_ssz", "smallvec", "typenum", ] -[[package]] -name = "tree_hash_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bee2ea1551f90040ab0e34b6fb7f2fa3bad8acc925837ac654f2c78a13e3089" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "tree_hash_derive" version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8840ad4d852e325d3afa7fde8a50b2412f89dce47d7eb291c0cc7f87cd040f38" +source = "git+https://github.com/sigp/tree_hash?rev=03d9fa474586c125f306dd7a00cf46284575a01f#03d9fa474586c125f306dd7a00cf46284575a01f" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -7527,29 +7603,31 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "types" version = "0.2.1" -source = "git+https://github.com/sigp/lighthouse?tag=v8.1.3#176cce585c1ba979a6210ed79b6b6528596cdb8c" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" dependencies = [ "alloy-primitives 1.7.3", "alloy-rlp", + "arbitrary", "bls", "compare_fields", "context_deserialize", "educe", "eth2_interop_keypairs", - "ethereum_hashing 0.8.0", + "ethereum_hashing", "ethereum_serde_utils 0.8.1", - "ethereum_ssz 0.10.4", - "ethereum_ssz_derive 0.10.4", + "ethereum_ssz", + "ethereum_ssz_derive", "fixed_bytes", "hex", "int_to_bytes", - "itertools 0.10.5", + "itertools 0.14.0", "kzg", "maplit", "merkle_proof", "metastruct", "milhouse", "parking_lot", + "paste", "rand 0.9.5", "rand_xorshift 0.4.0", "rayon", @@ -7558,17 +7636,16 @@ dependencies = [ "safe_arith", "serde", "serde_json", - "serde_yaml", "smallvec", - "ssz_types 0.14.1", + "ssz_types", "superstruct", "swap_or_not_shuffle", "tempfile", - "test_random_derive", "tracing", - "tree_hash 0.12.1", - "tree_hash_derive 0.12.1", + "tree_hash", + "tree_hash_derive", "typenum", + "yaml_serde", ] [[package]] @@ -7677,16 +7754,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom 0.2.17", - "serde", -] - [[package]] name = "uuid" version = "1.26.1" @@ -7706,12 +7773,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "vec_map" version = "0.8.2" @@ -8212,6 +8273,15 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "workspace_members" +version = "0.1.0" +source = "git+https://github.com/sigp/lighthouse?rev=31d8cfd40d228c2dd902a67b5142ff6abf00c058#31d8cfd40d228c2dd902a67b5142ff6abf00c058" +dependencies = [ + "cargo_metadata", + "quote", +] + [[package]] name = "writeable" version = "0.6.4" @@ -8246,6 +8316,19 @@ dependencies = [ "tap", ] +[[package]] +name = "yaml_serde" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b729a08a9a6be689bbad3e2bf8015926db54b6622cc89c3a5f7dc174b9e918" +dependencies = [ + "indexmap 2.14.2", + "itoa", + "libyaml-rs", + "ryu", + "serde", +] + [[package]] name = "yasna" version = "0.5.2" @@ -8373,6 +8456,12 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 3d5b23b14..b0281eae9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,9 +16,9 @@ alloy = { version = "^1.0.35", features = [ "rpc-types-beacon", "serde", "signer-local", - "ssz", ] } alloy-primitives = "^1.3.1" +arbitrary = "1.4" assert_cmd = "2.1.2" async-trait = "0.1.80" axum = { version = "0.8.1", features = ["macros"] } @@ -54,10 +54,10 @@ indexmap = "2.2.6" jsonwebtoken = { version = "9.3.1", default-features = false } lazy_static = "1.5.0" mediatype = "0.20.0" -lh_eth2 = { package = "eth2", git = "https://github.com/sigp/lighthouse", tag = "v8.1.3", features = ["events"] } -lh_eth2_keystore = { package = "eth2_keystore", git = "https://github.com/sigp/lighthouse", tag = "v8.1.3" } -lh_bls = { package = "bls", git = "https://github.com/sigp/lighthouse", tag = "v8.1.3" } -lh_types = { package = "types", git = "https://github.com/sigp/lighthouse", tag = "v8.1.3" } +lh_eth2 = { package = "eth2", git = "https://github.com/sigp/lighthouse", rev = "31d8cfd40d228c2dd902a67b5142ff6abf00c058", features = ["events"] } +lh_eth2_keystore = { package = "eth2_keystore", git = "https://github.com/sigp/lighthouse", rev = "31d8cfd40d228c2dd902a67b5142ff6abf00c058" } +lh_bls = { package = "bls", git = "https://github.com/sigp/lighthouse", rev = "31d8cfd40d228c2dd902a67b5142ff6abf00c058", features = ["arbitrary"] } +lh_types = { package = "types", git = "https://github.com/sigp/lighthouse", rev = "31d8cfd40d228c2dd902a67b5142ff6abf00c058", features = ["arbitrary"] } notify = "8.2.0" parking_lot = "0.12.3" pbkdf2 = "0.12.2" @@ -74,7 +74,7 @@ serde = { version = "1.0.202", features = ["derive"] } serde_json = "1.0.117" serde_yaml = "0.9.33" sha2 = "0.10.8" -ssz_types = "0.11" +ssz_types = "0.14.1" subtle = "2.5" tempfile = "3.20.0" thiserror = "2.0.12" @@ -97,4 +97,12 @@ uuid = { version = "1.8.0", features = ["fast-rng", "serde", "v4"] } webpki-roots = "1.0" [patch.crates-io] -blstrs_plus = { git = "https://github.com/Commit-Boost/blstrs" } +blstrs_plus = { git = "https://github.com/Commit-Boost/blstrs", rev = "c4ea6b21193886ee9849867397a62e9c243c1fbb" } +# Progressive-container ssz stack required by lighthouse-unstable (EIP-7495/7916). +# A consumer does NOT inherit a dependency's [patch]; mirror lighthouse's here. +ssz_types = { git = "https://github.com/sigp/ssz_types", rev = "9203d56ad2d7bc5f12133d043e085843f81edcd6" } +milhouse = { git = "https://github.com/sigp/milhouse", rev = "c70f128976ac0d60ea65a978dabd117a921c36ee" } +ethereum_ssz = { git = "https://github.com/sigp/ethereum_ssz", rev = "2059c21ba52cd3a7e39a8ad537012b761812a393" } +ethereum_ssz_derive = { git = "https://github.com/sigp/ethereum_ssz", rev = "2059c21ba52cd3a7e39a8ad537012b761812a393" } +tree_hash = { git = "https://github.com/sigp/tree_hash", rev = "03d9fa474586c125f306dd7a00cf46284575a01f" } +tree_hash_derive = { git = "https://github.com/sigp/tree_hash", rev = "03d9fa474586c125f306dd7a00cf46284575a01f" } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 7350b85cc..23aa9a0c4 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -11,6 +11,7 @@ testing-flags = [] [dependencies] aes.workspace = true alloy.workspace = true +arbitrary.workspace = true async-trait.workspace = true axum.workspace = true base64.workspace = true diff --git a/crates/common/src/config/utils.rs b/crates/common/src/config/utils.rs index 2e47eb941..de3777fe1 100644 --- a/crates/common/src/config/utils.rs +++ b/crates/common/src/config/utils.rs @@ -107,7 +107,6 @@ pub(crate) mod test_env { #[cfg(test)] mod tests { use super::{test_env::with_env, *}; - use crate::utils::TestRandomSeed; // Minimal TOML-deserializable type used by load_from_file / load_file_from_env // tests. @@ -140,8 +139,10 @@ mod tests { #[test] fn test_remove_duplicate_keys() { - let key1 = BlsPublicKey::test_random(); - let key2 = BlsPublicKey::test_random(); + // Real, distinct keys: `arbitrary` for a validated point falls back to + // the (invalid) all-zeros pubkey, so derive from random secret keys. + let key1 = crate::types::BlsSecretKey::random().public_key(); + let key2 = crate::types::BlsSecretKey::random().public_key(); let keys = vec![key1.clone(), key2.clone(), key1.clone()]; let unique_keys = remove_duplicate_keys(keys); diff --git a/crates/common/src/signature.rs b/crates/common/src/signature.rs index 41631e33a..70bd5c61c 100644 --- a/crates/common/src/signature.rs +++ b/crates/common/src/signature.rs @@ -171,8 +171,7 @@ mod tests { use crate::{ constants::APPLICATION_BUILDER_DOMAIN, pbs::{ - BlindedBeaconBlockElectra, BuilderBid, BuilderBidElectra, - ExecutionPayloadHeaderElectra, ExecutionRequests, + BlindedBeaconBlockElectra, BuilderBid, BuilderBidElectra, ExecutionPayloadHeaderElectra, }, types::{BlsSecretKey, Chain}, utils::TestRandomSeed, @@ -189,13 +188,13 @@ mod tests { #[test] fn test_builder_bid_sign_and_verify() { - let secret_key = BlsSecretKey::test_random(); + let secret_key = BlsSecretKey::random(); let pubkey = secret_key.public_key(); let message = BuilderBid::Electra(BuilderBidElectra { header: ExecutionPayloadHeaderElectra::test_random(), blob_kzg_commitments: Default::default(), - execution_requests: ExecutionRequests::default(), + execution_requests: Default::default(), value: U256::from(10), pubkey: pubkey.clone().into(), }); @@ -214,7 +213,7 @@ mod tests { #[test] fn test_blinded_block_sign_and_verify() { - let secret_key = BlsSecretKey::test_random(); + let secret_key = BlsSecretKey::random(); let pubkey = secret_key.public_key(); let block = BlindedBeaconBlockElectra::test_random(); diff --git a/crates/common/src/ssz.rs b/crates/common/src/ssz.rs index a62ac1993..18f7f7cf5 100644 --- a/crates/common/src/ssz.rs +++ b/crates/common/src/ssz.rs @@ -1,11 +1,10 @@ use alloy::primitives::U256; use lh_bls::Signature; -use lh_types::ForkName; +use lh_types::{ExecutionRequestsElectra, ForkName, MainnetEthSpec}; use ssz::BYTES_PER_LENGTH_OFFSET; use crate::pbs::{ - BuilderBidFulu, ExecutionPayloadHeaderFulu, ExecutionRequests, KzgCommitments, - error::SszValueError, + BuilderBidFulu, ExecutionPayloadHeaderFulu, KzgCommitments, error::SszValueError, }; // Get the offset of the message in a SignedBuilderBid SSZ structure @@ -17,7 +16,7 @@ fn get_ssz_value_offset_for_fork(fork: ForkName) -> Result Ok(get_message_offset::() + ::ssz_fixed_len() + ::ssz_fixed_len() + - ::ssz_fixed_len()) + as ssz::Decode>::ssz_fixed_len()) } _ => Err(SszValueError::UnsupportedFork { name: fork }), @@ -76,8 +75,8 @@ mod test { use super::get_bid_value_from_signed_builder_bid_ssz; use crate::{ pbs::{ - BuilderBid, BuilderBidFulu, ExecutionPayloadHeaderFulu, ExecutionRequests, - SignedBuilderBid, error::SszValueError, + BuilderBid, BuilderBidFulu, ExecutionPayloadHeaderFulu, SignedBuilderBid, + error::SszValueError, }, types::{BlsPublicKeyBytes, BlsSignature}, utils::TestRandomSeed, @@ -116,7 +115,7 @@ mod test { let message = BuilderBid::Fulu(BuilderBidFulu { header: ExecutionPayloadHeaderFulu::test_random(), blob_kzg_commitments: Default::default(), - execution_requests: ExecutionRequests::default(), + execution_requests: Default::default(), value: known_value, pubkey, }); diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index c0b53b4c0..c026849db 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -7,8 +7,7 @@ use alloy::{ hex, primitives::{U256, keccak256}, }; -use lh_types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; -use rand::{Rng, distr::Alphanumeric}; +use rand::{Rng, RngCore, distr::Alphanumeric}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::Value; use tracing::Level; @@ -428,17 +427,23 @@ pub async fn wait_for_signal() -> eyre::Result<()> { Ok(()) } -pub trait TestRandomSeed: TestRandom { +// lighthouse (rev 31d8cfd) replaced `TestRandom` with an `arbitrary` generator. +// Validated BLS types do NOT randomize: `impl_arbitrary!` yields the all-zeros +// point, which is an invalid pubkey. Use `BlsSecretKey::random().public_key()` +// for keys; `BlsSignature::test_random()` is a constant placeholder signature. +pub trait TestRandomSeed: for<'a> arbitrary::Arbitrary<'a> { fn test_random() -> Self where Self: Sized, { - let mut rng = XorShiftRng::from_os_rng(); - Self::random_for_test(&mut rng) + let mut bytes = vec![0u8; 256 * 1024]; + rand::rng().fill_bytes(&mut bytes); + let mut u = arbitrary::Unstructured::new(&bytes); + Self::arbitrary(&mut u).expect("enough entropy for an arbitrary test instance") } } -impl TestRandomSeed for T {} +impl arbitrary::Arbitrary<'a>> TestRandomSeed for T {} pub fn bls_pubkey_from_hex(hex: &str) -> eyre::Result { let Ok(bytes) = hex::decode(hex) else { diff --git a/crates/pbs/src/mev_boost/get_header.rs b/crates/pbs/src/mev_boost/get_header.rs index 576ced51e..670e80b16 100644 --- a/crates/pbs/src/mev_boost/get_header.rs +++ b/crates/pbs/src/mev_boost/get_header.rs @@ -806,7 +806,7 @@ mod tests { #[test] fn test_validate_signature() { - let secret_key = BlsSecretKey::test_random(); + let secret_key = BlsSecretKey::random(); let pubkey = secret_key.public_key(); let wrong_pubkey = BlsPublicKeyBytes::test_random(); let wrong_signature = BlsSignature::test_random(); @@ -862,7 +862,7 @@ mod tests { ForkName::Fulu => {} // Skip future forks - ForkName::Gloas => continue, + ForkName::Gloas | ForkName::Heze => continue, } // Load get_header JSON from test data diff --git a/crates/pbs/src/mev_boost/get_header_ws.rs b/crates/pbs/src/mev_boost/get_header_ws.rs index 3b209ce92..a91017ec8 100644 --- a/crates/pbs/src/mev_boost/get_header_ws.rs +++ b/crates/pbs/src/mev_boost/get_header_ws.rs @@ -69,6 +69,7 @@ fn fork_from_wire(byte: u8) -> Option { 5 => ForkName::Electra, 6 => ForkName::Fulu, 7 => ForkName::Gloas, + 8 => ForkName::Heze, _ => return None, }) } diff --git a/tests/src/mock_relay.rs b/tests/src/mock_relay.rs index becdc1f04..336c91f1a 100644 --- a/tests/src/mock_relay.rs +++ b/tests/src/mock_relay.rs @@ -23,10 +23,9 @@ use axum::{ use cb_common::{ pbs::{ BUILDER_V1_API_PATH, BUILDER_V2_API_PATH, BlobsBundle, BuilderBid, BuilderBidFulu, - ExecutionPayloadElectra, ExecutionPayloadHeaderFulu, ExecutionRequests, ForkName, - GET_HEADER_PATH, GET_STATUS_PATH, GetHeaderParams, GetHeaderResponse, GetPayloadInfo, - PayloadAndBlobs, REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH, SignedBuilderBid, - SubmitBlindedBlockResponse, + ExecutionPayloadElectra, ExecutionPayloadHeaderFulu, ForkName, GET_HEADER_PATH, + GET_STATUS_PATH, GetHeaderParams, GetHeaderResponse, GetPayloadInfo, PayloadAndBlobs, + REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH, SignedBuilderBid, SubmitBlindedBlockResponse, }, signature::sign_builder_root, types::{BlsSecretKey, Chain}, @@ -256,7 +255,7 @@ pub fn mock_signed_builder_bid( let message = BuilderBid::Fulu(BuilderBidFulu { header, blob_kzg_commitments: Default::default(), - execution_requests: ExecutionRequests::default(), + execution_requests: Default::default(), value, pubkey: signer.public_key().into(), }); From 8683257a82485004ba931f4954803c547d3127e4 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 5 Oct 2026 10:57:36 -0700 Subject: [PATCH 07/24] test: bind test listeners once instead of dropping and rebinding Tests discovered a free port, dropped the listener and let the server rebind it, leaving a window where another process could take the port. PbsService and SigningService gain `run_with_listener`, the mock SSV servers take a bound listener, and the legacy suites hand their listeners straight to the server. Also add `wait_for_ready`, which polls /status instead of sleeping a fixed 100 ms. --- crates/pbs/src/service.rs | 13 ++++++-- crates/signer/src/service.rs | 15 +++++++-- tests/src/mock_ssv_node.rs | 8 ++--- tests/src/mock_ssv_public.rs | 8 ++--- tests/src/signer_service.rs | 8 +++-- tests/src/utils.rs | 19 +++++++++++ tests/tests/pbs_cfg_file_update.rs | 3 +- tests/tests/pbs_get_header.rs | 15 +++------ tests/tests/pbs_get_status.rs | 30 +++++++++-------- tests/tests/pbs_mux.rs | 40 +++++++++-------------- tests/tests/pbs_mux_refresh.rs | 8 ++--- tests/tests/pbs_post_blinded_blocks.rs | 12 +++---- tests/tests/pbs_post_validators.rs | 45 ++++++++++++++++++-------- tests/tests/signer_jwt_auth.rs | 20 ++++++++---- tests/tests/signer_jwt_auth_cleanup.rs | 5 +-- tests/tests/signer_request_sig.rs | 14 +++++--- tests/tests/signer_tls.rs | 5 +-- 17 files changed, 158 insertions(+), 110 deletions(-) diff --git a/crates/pbs/src/service.rs b/crates/pbs/src/service.rs index ded5b7db9..ed3d179c2 100644 --- a/crates/pbs/src/service.rs +++ b/crates/pbs/src/service.rs @@ -30,7 +30,17 @@ pub struct PbsService; impl PbsService { pub async fn run>(state: PbsState) -> Result<()> { - let addr = state.config.endpoint; + let listener = TcpListener::bind(state.config.endpoint).await?; + Self::run_with_listener::(state, listener).await + } + + /// Serve from an already-bound listener. Prefer this in tests: rebinding a + /// freed port races other processes for it. + pub async fn run_with_listener>( + state: PbsState, + listener: TcpListener, + ) -> Result<()> { + let addr = listener.local_addr()?; info!(version = COMMIT_BOOST_VERSION, commit_hash = COMMIT_BOOST_COMMIT, ?addr, chain =? state.config.chain, "starting PBS service"); // Check if refreshing registry muxes is required @@ -44,7 +54,6 @@ impl PbsService { let config_path = state.config_path.clone(); let state: Arc>> = RwLock::new(state).into(); let app = create_app_router::(state.clone()); - let listener = TcpListener::bind(addr).await?; let task = tokio::spawn( diff --git a/crates/signer/src/service.rs b/crates/signer/src/service.rs index 710d7ea6e..b6023a485 100644 --- a/crates/signer/src/service.rs +++ b/crates/signer/src/service.rs @@ -94,6 +94,16 @@ impl SigningService { return Ok(()); } + let listener = tokio::net::TcpListener::bind(config.endpoint).await?; + Self::run_with_listener(config, listener).await + } + + /// Serve from an already-bound listener. Prefer this in tests: rebinding a + /// freed port races other processes for it. + pub async fn run_with_listener( + config: StartSignerConfig, + listener: tokio::net::TcpListener, + ) -> eyre::Result<()> { let module_ids: Vec = config.mod_signing_configs.keys().cloned().map(Into::into).collect(); @@ -169,16 +179,17 @@ impl SigningService { } }); + let std_listener = listener.into_std()?; let server_result = if let Some(tls_config) = config.tls_certificates { let tls_config = RustlsConfig::from_pem(tls_config.0, tls_config.1).await?; - axum_server::bind_rustls(config.endpoint, tls_config) + axum_server::tls_rustls::from_tcp_rustls(std_listener, tls_config) .serve( signer_app.merge(admin_app).into_make_service_with_connect_info::(), ) .await } else { warn!("Running in insecure HTTP mode, no TLS certificates provided"); - axum_server::bind(config.endpoint) + axum_server::from_tcp(std_listener) .serve( signer_app.merge(admin_app).into_make_service_with_connect_info::(), ) diff --git a/tests/src/mock_ssv_node.rs b/tests/src/mock_ssv_node.rs index 7f24569d4..fe0d77675 100644 --- a/tests/src/mock_ssv_node.rs +++ b/tests/src/mock_ssv_node.rs @@ -1,4 +1,4 @@ -use std::{net::SocketAddr, sync::Arc}; +use std::sync::Arc; use alloy::primitives::U256; use axum::{ @@ -34,7 +34,7 @@ struct SsvNodeValidatorsRequestBody { /// Creates a simple mock server to simulate the SSV API endpoint under /// various conditions for testing. Note this ignores pub async fn create_mock_ssv_node_server( - port: u16, + listener: TcpListener, state: Option, ) -> Result, axum::Error> { let data = include_str!("../../tests/data/ssv_valid_node.json"); @@ -50,8 +50,6 @@ pub async fn create_mock_ssv_node_server( .with_state(state) .into_make_service(); - let address = SocketAddr::from(([127, 0, 0, 1], port)); - let listener = TcpListener::bind(address).await.map_err(axum::Error::new)?; let server = axum::serve(listener, router).with_graceful_shutdown(async { tokio::signal::ctrl_c().await.expect("Failed to listen for shutdown signal"); }); @@ -60,7 +58,7 @@ pub async fn create_mock_ssv_node_server( eprintln!("Server error: {e}"); } })); - info!("Mock server started on http://localhost:{port}/"); + info!("mock SSV server started"); result } diff --git a/tests/src/mock_ssv_public.rs b/tests/src/mock_ssv_public.rs index a014db42e..4ead5c7be 100644 --- a/tests/src/mock_ssv_public.rs +++ b/tests/src/mock_ssv_public.rs @@ -1,4 +1,4 @@ -use std::{net::SocketAddr, sync::Arc}; +use std::sync::Arc; use axum::{ extract::{Path, State}, @@ -27,7 +27,7 @@ pub struct PublicSsvMockState { /// Creates a simple mock server to simulate the SSV API endpoint under /// various conditions for testing. Note this ignores pub async fn create_mock_public_ssv_server( - port: u16, + listener: TcpListener, state: Option, ) -> Result, axum::Error> { let data = include_str!("../../tests/data/ssv_valid_public.json"); @@ -46,8 +46,6 @@ pub async fn create_mock_public_ssv_server( .with_state(state) .into_make_service(); - let address = SocketAddr::from(([127, 0, 0, 1], port)); - let listener = TcpListener::bind(address).await.map_err(axum::Error::new)?; let server = axum::serve(listener, router).with_graceful_shutdown(async { tokio::signal::ctrl_c().await.expect("Failed to listen for shutdown signal"); }); @@ -56,7 +54,7 @@ pub async fn create_mock_public_ssv_server( eprintln!("Server error: {e}"); } })); - info!("Mock server started on http://localhost:{port}/"); + info!("mock SSV server started"); result } diff --git a/tests/src/signer_service.rs b/tests/src/signer_service.rs index 550ac4ce7..8338a5829 100644 --- a/tests/src/signer_service.rs +++ b/tests/src/signer_service.rs @@ -10,6 +10,7 @@ use cb_common::{ use cb_signer::service::SigningService; use eyre::Result; use reqwest::{Certificate, Response, StatusCode}; +use tokio::net::TcpListener; use tracing::info; use crate::utils::{get_signer_config, get_start_signer_config}; @@ -17,7 +18,7 @@ use crate::utils::{get_signer_config, get_start_signer_config}; // Starts the signer moduler server on a separate task and returns its // configuration pub async fn start_server( - port: u16, + listener: TcpListener, mod_signing_configs: &HashMap, admin_secret: String, use_tls: bool, @@ -31,13 +32,14 @@ pub async fn start_server( format: ValidatorKeysFormat::Lighthouse, }; let mut config = get_signer_config(loader, use_tls); - config.port = port; + config.port = listener.local_addr()?.port(); config.jwt_auth_fail_limit = 3; // Set a low fail limit for testing config.jwt_auth_fail_timeout_seconds = 3; // Set a short timeout for testing let start_config = get_start_signer_config(config, chain, mod_signing_configs, admin_secret); // Run the Signer - let server_handle = tokio::spawn(SigningService::run(start_config.clone())); + let server_handle = + tokio::spawn(SigningService::run_with_listener(start_config.clone(), listener)); // Wait for the server to start let (url, client) = match start_config.tls_certificates { diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 9a4b0a2a6..11d192d52 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -3,6 +3,7 @@ use std::{ net::{Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, sync::{Arc, Once}, + time::Duration, }; use alloy::primitives::{B256, U256}; @@ -21,8 +22,11 @@ use cb_common::{ }; use eyre::Result; use rcgen::generate_simple_self_signed; +use reqwest::StatusCode; use url::Url; +use crate::mock_validator::MockValidator; + pub const HEADER_API_KEY: &str = "x-api-key"; pub const API_KEY: &str = "123e4567-e89b-12d3-a456-426614174000"; /// Distinct from [`API_KEY`], which `MockValidator` also sends to PBS: a relay @@ -236,3 +240,18 @@ pub fn create_module_config(id: ModuleId, signing_id: B256) -> StaticModuleConfi pub fn bls_pubkey_from_hex_unchecked(hex: &str) -> BlsPublicKey { bls_pubkey_from_hex(hex).unwrap() } + +/// Poll /status until PBS and its relays are up. relay_check makes a 200 mean +/// the whole chain is ready; the fixed 100ms sleep used elsewhere flakes under +/// parallel suite load. +pub async fn wait_for_ready(mock_validator: &MockValidator) -> Result<()> { + for _ in 0..100 { + if let Ok(res) = mock_validator.do_get_status().await && + res.status() == StatusCode::OK + { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + eyre::bail!("PBS/relays did not become ready within 2s") +} diff --git a/tests/tests/pbs_cfg_file_update.rs b/tests/tests/pbs_cfg_file_update.rs index c576a6a62..ec954f531 100644 --- a/tests/tests/pbs_cfg_file_update.rs +++ b/tests/tests/pbs_cfg_file_update.rs @@ -114,8 +114,7 @@ async fn test_cfg_file_update() -> Result<()> { // Run the PBS service let config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![relay1.clone()]); let state = PbsState::new(config, config_path.clone()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers - extra time for the file watcher tokio::time::sleep(Duration::from_millis(1000)).await; diff --git a/tests/tests/pbs_get_header.rs b/tests/tests/pbs_get_header.rs index 21ff3e6ef..e155041bf 100644 --- a/tests/tests/pbs_get_header.rs +++ b/tests/tests/pbs_get_header.rs @@ -255,8 +255,7 @@ async fn test_get_header_impl( pbs_config.rpc_url = rpc_url; let config = to_pbs_config(chain, pbs_config, vec![mock_relay.clone()]); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -336,8 +335,7 @@ async fn test_get_header_returns_204_if_no_relay_reachable() -> Result<()> { // Run the PBS service let config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![mock_relay.clone()]); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -371,8 +369,7 @@ async fn test_get_header_returns_400_if_request_is_invalid() -> Result<()> { // Run the PBS service let config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![mock_relay.clone()]); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -534,8 +531,7 @@ async fn test_get_header_tolerates_mime_params_in_content_type() -> Result<()> { let pbs_config = get_pbs_config(pbs_port); let config = to_pbs_config(chain, pbs_config, vec![mock_relay]); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); tokio::time::sleep(Duration::from_millis(100)).await; @@ -574,8 +570,7 @@ async fn test_get_header_tolerates_json_charset_param() -> Result<()> { let pbs_config = get_pbs_config(pbs_port); let config = to_pbs_config(chain, pbs_config, vec![mock_relay]); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/tests/tests/pbs_get_status.rs b/tests/tests/pbs_get_status.rs index 08afb231a..82bc05a06 100644 --- a/tests/tests/pbs_get_status.rs +++ b/tests/tests/pbs_get_status.rs @@ -3,9 +3,7 @@ use std::{io::Write, path::PathBuf, sync::Arc, time::Duration}; use cb_common::{signer::random_secret, types::Chain}; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState, STATUS_ENDPOINT_TAG}; use cb_tests::{ - mock_relay::{ - MockRelayState, start_mock_relay_service, start_mock_relay_service_with_listener, - }, + mock_relay::{MockRelayState, start_mock_relay_service_with_listener}, mock_validator::MockValidator, utils::{ generate_mock_relay, generate_mock_relay_with_api_key_file, get_free_listener, @@ -23,21 +21,24 @@ async fn test_get_status() -> Result<()> { let pubkey = signer.public_key(); let chain = Chain::Holesky; - let pbs_port = 3500; - let relay_0_port = pbs_port + 1; - let relay_1_port = pbs_port + 2; + let pbs_listener = get_free_listener().await; + let relay_0_listener = get_free_listener().await; + let relay_1_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let relay_0_port = relay_0_listener.local_addr()?.port(); + let relay_1_port = relay_1_listener.local_addr()?.port(); let relays = vec![ generate_mock_relay(relay_0_port, pubkey.clone())?, generate_mock_relay(relay_1_port, pubkey)?, ]; let mock_state = Arc::new(MockRelayState::new(chain, signer)); - tokio::spawn(start_mock_relay_service(mock_state.clone(), relay_0_port)); - tokio::spawn(start_mock_relay_service(mock_state.clone(), relay_1_port)); + tokio::spawn(start_mock_relay_service_with_listener(mock_state.clone(), relay_0_listener)); + tokio::spawn(start_mock_relay_service_with_listener(mock_state.clone(), relay_1_listener)); let config = to_pbs_config(chain, get_pbs_config(pbs_port), relays.clone()); let state = PbsState::new(config, PathBuf::new()); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -59,18 +60,21 @@ async fn test_get_status_returns_502_if_relay_down() -> Result<()> { let pubkey = signer.public_key(); let chain = Chain::Holesky; - let pbs_port = 3600; - let relay_port = pbs_port + 1; + let pbs_listener = get_free_listener().await; + let relay_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let relay_port = relay_listener.local_addr()?.port(); + // Release the relay's port so nothing listens on it: the relay is down + drop(relay_listener); let relays = vec![generate_mock_relay(relay_port, pubkey)?]; let mock_state = Arc::new(MockRelayState::new(chain, signer)); // Don't start the relay - // tokio::spawn(start_mock_relay_service(mock_state.clone(), relay_port)); let config = to_pbs_config(chain, get_pbs_config(pbs_port), relays.clone()); let state = PbsState::new(config, PathBuf::new()); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/tests/tests/pbs_mux.rs b/tests/tests/pbs_mux.rs index a34e5f861..7778fb456 100644 --- a/tests/tests/pbs_mux.rs +++ b/tests/tests/pbs_mux.rs @@ -39,8 +39,7 @@ async fn test_ssv_public_network_fetch() -> Result<()> { // Start the mock server let listener = get_free_listener().await; let port = listener.local_addr().unwrap().port(); - drop(listener); - let server_handle = create_mock_public_ssv_server(port, None).await?; + let server_handle = create_mock_public_ssv_server(listener, None).await?; let url = Url::parse(&format!("http://localhost:{port}/api/v4/test_chain/validators/in_operator/1")) .unwrap(); @@ -79,9 +78,8 @@ async fn test_ssv_network_fetch_big_data() -> Result<()> { // Start the mock server let listener = get_free_listener().await; let port = listener.local_addr().unwrap().port(); - drop(listener); let server_handle = - cb_tests::mock_ssv_public::create_mock_public_ssv_server(port, None).await?; + cb_tests::mock_ssv_public::create_mock_public_ssv_server(listener, None).await?; let url = Url::parse(&format!("http://localhost:{port}/big_data")).unwrap(); let response = request_ssv_pubkeys_from_public_api(url.clone(), Duration::from_secs(120)).await; @@ -113,12 +111,11 @@ async fn test_ssv_network_fetch_timeout() -> Result<()> { // Start the mock server let listener = get_free_listener().await; let port = listener.local_addr().unwrap().port(); - drop(listener); let state = PublicSsvMockState { validators: Arc::new(RwLock::new(vec![])), force_timeout: Arc::new(RwLock::new(true)), }; - let server_handle = create_mock_public_ssv_server(port, Some(state)).await?; + let server_handle = create_mock_public_ssv_server(listener, Some(state)).await?; let url = Url::parse(&format!("http://localhost:{port}/api/v4/test_chain/validators/in_operator/1")) .unwrap(); @@ -144,9 +141,8 @@ async fn test_ssv_network_fetch_big_data_without_content_length() -> Result<()> // Start the mock server let listener = get_free_listener().await; let port = listener.local_addr().unwrap().port(); - drop(listener); set_ignore_content_length(true); - let server_handle = create_mock_public_ssv_server(port, None).await?; + let server_handle = create_mock_public_ssv_server(listener, None).await?; let url = Url::parse(&format!("http://localhost:{port}/big_data")).unwrap(); let response = request_ssv_pubkeys_from_public_api(url.clone(), Duration::from_secs(120)).await; @@ -178,8 +174,7 @@ async fn test_ssv_node_network_fetch() -> Result<()> { // Start the mock server let listener = get_free_listener().await; let port = listener.local_addr().unwrap().port(); - drop(listener); - let _server_handle = create_mock_ssv_node_server(port, None).await?; + let _server_handle = create_mock_ssv_node_server(listener, None).await?; let url = Url::parse(&format!("http://localhost:{port}/v1/validators")).unwrap(); let response = request_ssv_pubkeys_from_ssv_node( url, @@ -248,8 +243,7 @@ async fn test_mux() -> Result<()> { // Run PBS service let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -337,10 +331,6 @@ async fn test_ssv_multi_with_node() -> Result<()> { let ssv_node_port = ssv_node_listener.local_addr().unwrap().port(); let ssv_public_port = ssv_public_listener.local_addr().unwrap().port(); let relay_port = relay_listener.local_addr().unwrap().port(); - // Drop SSV node + public listeners because their mock server helpers bind the - // port themselves. - drop(ssv_node_listener); - drop(ssv_public_listener); // Start the mock SSV node let ssv_node_url = Url::parse(&format!("http://localhost:{ssv_node_port}/v1/"))?; @@ -352,7 +342,7 @@ async fn test_ssv_multi_with_node() -> Result<()> { force_timeout: Arc::new(RwLock::new(false)), }; let ssv_node_handle = - create_mock_ssv_node_server(ssv_node_port, Some(mock_ssv_node_state.clone())).await?; + create_mock_ssv_node_server(ssv_node_listener, Some(mock_ssv_node_state.clone())).await?; // Start the mock SSV public API let ssv_public_url = Url::parse(&format!("http://localhost:{ssv_public_port}/api/v4/"))?; @@ -361,7 +351,8 @@ async fn test_ssv_multi_with_node() -> Result<()> { force_timeout: Arc::new(RwLock::new(false)), }; let ssv_public_handle = - create_mock_public_ssv_server(ssv_public_port, Some(mock_ssv_public_state.clone())).await?; + create_mock_public_ssv_server(ssv_public_listener, Some(mock_ssv_public_state.clone())) + .await?; // Start a mock relay to be used by the mux let relay = generate_mock_relay(relay_port, pubkey.clone())?; @@ -403,8 +394,8 @@ async fn test_ssv_multi_with_node() -> Result<()> { // Run PBS service let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - let pbs_server = tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + let pbs_server = + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); info!("Started PBS server with pubkey {pubkey}"); // Wait for the server to start @@ -449,8 +440,6 @@ async fn test_ssv_multi_with_public() -> Result<()> { let relay_port = relay_listener.local_addr().unwrap().port(); // SSV node is intentionally down โ€” release its reserved port. drop(ssv_node_listener); - // SSV public mock helper binds the port itself. - drop(ssv_public_listener); // Start the mock SSV node let ssv_node_url = Url::parse(&format!("http://localhost:{ssv_node_port}/v1/"))?; @@ -469,7 +458,8 @@ async fn test_ssv_multi_with_public() -> Result<()> { force_timeout: Arc::new(RwLock::new(false)), }; let ssv_public_handle = - create_mock_public_ssv_server(ssv_public_port, Some(mock_ssv_public_state.clone())).await?; + create_mock_public_ssv_server(ssv_public_listener, Some(mock_ssv_public_state.clone())) + .await?; // Start a mock relay to be used by the mux let relay = generate_mock_relay(relay_port, pubkey.clone())?; @@ -511,8 +501,8 @@ async fn test_ssv_multi_with_public() -> Result<()> { // Run PBS service let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - let pbs_server = tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + let pbs_server = + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); info!("Started PBS server with pubkey {pubkey}"); // Wait for the server to start diff --git a/tests/tests/pbs_mux_refresh.rs b/tests/tests/pbs_mux_refresh.rs index 5bfb4aabf..10a9f1c87 100644 --- a/tests/tests/pbs_mux_refresh.rs +++ b/tests/tests/pbs_mux_refresh.rs @@ -47,8 +47,6 @@ async fn test_auto_refresh() -> Result<()> { let ssv_api_port = ssv_api_listener.local_addr().unwrap().port(); let default_relay_port = default_relay_listener.local_addr().unwrap().port(); let mux_relay_port = mux_relay_listener.local_addr().unwrap().port(); - // create_mock_public_ssv_server binds the port itself. - drop(ssv_api_listener); // Start the mock SSV API server // Intentionally missing a trailing slash to ensure this is handled properly @@ -60,7 +58,7 @@ async fn test_auto_refresh() -> Result<()> { force_timeout: Arc::new(RwLock::new(false)), }; let ssv_server_handle = - create_mock_public_ssv_server(ssv_api_port, Some(mock_ssv_state.clone())).await?; + create_mock_public_ssv_server(ssv_api_listener, Some(mock_ssv_state.clone())).await?; // Start a default relay for non-mux keys let default_relay = generate_mock_relay(default_relay_port, default_pubkey.clone())?; @@ -111,8 +109,8 @@ async fn test_auto_refresh() -> Result<()> { // Run PBS service let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - let pbs_server = tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + let pbs_server = + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); info!("Started PBS server with pubkey {default_pubkey}"); // Wait for the server to start diff --git a/tests/tests/pbs_post_blinded_blocks.rs b/tests/tests/pbs_post_blinded_blocks.rs index 06e143c2b..3f9566116 100644 --- a/tests/tests/pbs_post_blinded_blocks.rs +++ b/tests/tests/pbs_post_blinded_blocks.rs @@ -259,8 +259,7 @@ async fn test_submit_block_too_large() -> Result<()> { let config = to_pbs_config(chain, get_pbs_config(pbs_port), relays); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -476,8 +475,7 @@ async fn submit_block_impl( let pbs_config = get_pbs_config(pbs_port); let config = to_pbs_config(chain, pbs_config, vec![mock_relay]); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -549,8 +547,7 @@ async fn submit_block_ssz_override( let pbs_config = get_pbs_config(pbs_port); let config = to_pbs_config(chain, pbs_config, vec![mock_relay]); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); tokio::time::sleep(Duration::from_millis(100)).await; @@ -689,8 +686,7 @@ async fn test_submit_block_tolerates_mime_params_in_content_type() -> Result<()> let pbs_config = get_pbs_config(pbs_port); let config = to_pbs_config(chain, pbs_config, vec![mock_relay]); let state = PbsState::new(config, PathBuf::new()); - drop(pbs_listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/tests/tests/pbs_post_validators.rs b/tests/tests/pbs_post_validators.rs index 12601cdad..22bd1a5d2 100644 --- a/tests/tests/pbs_post_validators.rs +++ b/tests/tests/pbs_post_validators.rs @@ -7,9 +7,11 @@ use cb_common::{ }; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use cb_tests::{ - mock_relay::{MockRelayState, start_mock_relay_service}, + mock_relay::{MockRelayState, start_mock_relay_service_with_listener}, mock_validator::MockValidator, - utils::{generate_mock_relay, get_pbs_config, setup_test_env, to_pbs_config}, + utils::{ + generate_mock_relay, get_free_listener, get_pbs_config, setup_test_env, to_pbs_config, + }, }; use eyre::Result; use reqwest::StatusCode; @@ -22,17 +24,20 @@ async fn test_register_validators() -> Result<()> { let pubkey: BlsPublicKey = signer.public_key(); let chain = Chain::Holesky; - let pbs_port = 4000; + let pbs_listener = get_free_listener().await; + let relay_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let relay_port = relay_listener.local_addr()?.port(); // Run a mock relay - let relays = vec![generate_mock_relay(pbs_port + 1, pubkey)?]; + let relays = vec![generate_mock_relay(relay_port, pubkey)?]; let mock_state = Arc::new(MockRelayState::new(chain, signer)); - tokio::spawn(start_mock_relay_service(mock_state.clone(), pbs_port + 1)); + tokio::spawn(start_mock_relay_service_with_listener(mock_state.clone(), relay_listener)); // Run the PBS service let config = to_pbs_config(chain, get_pbs_config(pbs_port), relays); let state = PbsState::new(config, PathBuf::new()); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -68,20 +73,26 @@ async fn test_register_validators_does_not_retry_on_429() -> Result<()> { let pubkey: BlsPublicKey = signer.public_key(); let chain = Chain::Holesky; - let pbs_port = 4200; + let pbs_listener = get_free_listener().await; + let relay_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let relay_port = relay_listener.local_addr()?.port(); // Set up mock relay state and override response to 429 let mock_state = Arc::new(MockRelayState::new(chain, signer)); mock_state.set_response_override(StatusCode::TOO_MANY_REQUESTS); // Run a mock relay - let relays = vec![generate_mock_relay(pbs_port + 1, pubkey)?]; - tokio::spawn(start_mock_relay_service(mock_state.clone(), pbs_port + 1)); + let relays = vec![generate_mock_relay(relay_port, pubkey)?]; + tokio::spawn(start_mock_relay_service_with_listener(mock_state.clone(), relay_listener)); // Run the PBS service let config = to_pbs_config(chain, get_pbs_config(pbs_port), relays); let state = PbsState::new(config, PathBuf::new()); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state.clone())); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>( + state.clone(), + pbs_listener, + )); // Leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -121,14 +132,17 @@ async fn test_register_validators_retries_on_500() -> Result<()> { let pubkey: BlsPublicKey = signer.public_key(); let chain = Chain::Holesky; - let pbs_port = 4300; + let pbs_listener = get_free_listener().await; + let relay_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let relay_port = relay_listener.local_addr()?.port(); // Set up internal mock relay with 500 response override let mock_state = Arc::new(MockRelayState::new(chain, signer)); mock_state.set_response_override(StatusCode::INTERNAL_SERVER_ERROR); // 500 - let relays = vec![generate_mock_relay(pbs_port + 1, pubkey)?]; - tokio::spawn(start_mock_relay_service(mock_state.clone(), pbs_port + 1)); + let relays = vec![generate_mock_relay(relay_port, pubkey)?]; + tokio::spawn(start_mock_relay_service_with_listener(mock_state.clone(), relay_listener)); // Set retry limit to 3 let mut pbs_config = get_pbs_config(pbs_port); @@ -136,7 +150,10 @@ async fn test_register_validators_retries_on_500() -> Result<()> { let config = to_pbs_config(chain, pbs_config, relays); let state = PbsState::new(config, PathBuf::new()); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state.clone())); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>( + state.clone(), + pbs_listener, + )); tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/tests/tests/signer_jwt_auth.rs b/tests/tests/signer_jwt_auth.rs index d1b65b3f7..61cd6defc 100644 --- a/tests/tests/signer_jwt_auth.rs +++ b/tests/tests/signer_jwt_auth.rs @@ -12,7 +12,7 @@ use cb_common::{ }; use cb_tests::{ signer_service::{start_server, verify_pubkeys}, - utils::{self, setup_test_env}, + utils::{self, get_free_listener, setup_test_env}, }; use eyre::Result; use reqwest::StatusCode; @@ -41,7 +41,8 @@ async fn test_signer_jwt_auth_success() -> Result<()> { setup_test_env(); let module_id = ModuleId(JWT_MODULE.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20100, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; let jwt_config = mod_cfgs.get(&module_id).expect("JWT config for test module not found"); // Run a pubkeys request @@ -61,7 +62,8 @@ async fn test_signer_jwt_auth_fail() -> Result<()> { setup_test_env(); let module_id = ModuleId(JWT_MODULE.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20101, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; // Run a pubkeys request - this should fail due to invalid JWT let jwt = create_jwt(&module_id, "incorrect secret", GET_PUBKEYS_PATH, None)?; @@ -82,7 +84,8 @@ async fn test_signer_jwt_rate_limit() -> Result<()> { setup_test_env(); let module_id = ModuleId(JWT_MODULE.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20102, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; let mod_cfg = mod_cfgs.get(&module_id).expect("JWT config for test module not found"); // Run as many pubkeys requests as the fail limit @@ -116,7 +119,8 @@ async fn test_signer_revoked_jwt_fail() -> Result<()> { let admin_secret = ADMIN_SECRET.to_string(); let module_id = ModuleId(JWT_MODULE.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20400, &mod_cfgs, admin_secret.clone(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, admin_secret.clone(), false).await?; // Run as many pubkeys requests as the fail limit let jwt = create_jwt(&module_id, JWT_SECRET, GET_PUBKEYS_PATH, None)?; @@ -149,7 +153,8 @@ async fn test_signer_only_admin_can_revoke() -> Result<()> { let admin_secret = ADMIN_SECRET.to_string(); let module_id = ModuleId(JWT_MODULE.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20500, &mod_cfgs, admin_secret.clone(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, admin_secret.clone(), false).await?; let revoke_body = RevokeModuleRequest { module_id: ModuleId(JWT_MODULE.to_string()) }; let body_bytes = serde_json::to_vec(&revoke_body)?; @@ -177,7 +182,8 @@ async fn test_signer_admin_jwt_rate_limit() -> Result<()> { let admin_secret = ADMIN_SECRET.to_string(); let module_id = ModuleId(JWT_MODULE.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20510, &mod_cfgs, admin_secret.clone(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, admin_secret.clone(), false).await?; let revoke_body = RevokeModuleRequest { module_id: ModuleId(JWT_MODULE.to_string()) }; let body_bytes = serde_json::to_vec(&revoke_body)?; diff --git a/tests/tests/signer_jwt_auth_cleanup.rs b/tests/tests/signer_jwt_auth_cleanup.rs index d6fde2a43..09f64721d 100644 --- a/tests/tests/signer_jwt_auth_cleanup.rs +++ b/tests/tests/signer_jwt_auth_cleanup.rs @@ -9,7 +9,7 @@ use cb_common::{ }; use cb_tests::{ signer_service::start_server, - utils::{self}, + utils::{self, get_free_listener}, }; use eyre::Result; use reqwest::StatusCode; @@ -38,7 +38,8 @@ async fn test_signer_jwt_fail_cleanup() -> Result<()> { // setup_test_env() isn't used because we want to capture logs with tracing_test let module_id = ModuleId(JWT_MODULE.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20102, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; let mod_cfg = mod_cfgs.get(&module_id).expect("JWT config for test module not found"); // Run as many pubkeys requests as the fail limit diff --git a/tests/tests/signer_request_sig.rs b/tests/tests/signer_request_sig.rs index 78efbf9e8..a60416c4c 100644 --- a/tests/tests/signer_request_sig.rs +++ b/tests/tests/signer_request_sig.rs @@ -12,7 +12,7 @@ use cb_common::{ }; use cb_tests::{ signer_service::start_server, - utils::{self, setup_test_env}, + utils::{self, get_free_listener, setup_test_env}, }; use eyre::Result; use reqwest::StatusCode; @@ -53,7 +53,8 @@ async fn test_signer_sign_request_good() -> Result<()> { setup_test_env(); let module_id = ModuleId(MODULE_ID_1.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20200, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; let jwt_config = mod_cfgs.get(&module_id).expect("JWT config for test module not found"); // Send a signing request @@ -96,7 +97,8 @@ async fn test_signer_sign_request_different_module() -> Result<()> { setup_test_env(); let module_id = ModuleId(MODULE_ID_2.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20201, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; let jwt_config = mod_cfgs.get(&module_id).expect("JWT config for 2nd test module not found"); // Send a signing request @@ -142,7 +144,8 @@ async fn test_signer_sign_request_incorrect_hash() -> Result<()> { setup_test_env(); let module_id = ModuleId(MODULE_ID_2.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20202, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; let jwt_config = mod_cfgs.get(&module_id).expect("JWT config for 2nd test module not found"); // Send a signing request @@ -178,7 +181,8 @@ async fn test_signer_sign_request_missing_hash() -> Result<()> { setup_test_env(); let module_id = ModuleId(MODULE_ID_2.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20203, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), false).await?; let jwt_config = mod_cfgs.get(&module_id).expect("JWT config for 2nd test module not found"); // Send a signing request diff --git a/tests/tests/signer_tls.rs b/tests/tests/signer_tls.rs index 2df98d73a..718100eba 100644 --- a/tests/tests/signer_tls.rs +++ b/tests/tests/signer_tls.rs @@ -9,7 +9,7 @@ use cb_common::{ }; use cb_tests::{ signer_service::{start_server, verify_pubkeys}, - utils::{self, setup_test_env}, + utils::{self, get_free_listener, setup_test_env}, }; use eyre::{Result, bail}; use reqwest::Certificate; @@ -37,7 +37,8 @@ async fn test_signer_tls() -> Result<()> { setup_test_env(); let module_id = ModuleId(JWT_MODULE.to_string()); let mod_cfgs = create_mod_signing_configs().await; - let start_config = start_server(20100, &mod_cfgs, ADMIN_SECRET.to_string(), true).await?; + let start_config = + start_server(get_free_listener().await, &mod_cfgs, ADMIN_SECRET.to_string(), true).await?; let jwt_config = mod_cfgs.get(&module_id).expect("JWT config for test module not found"); // Run a pubkeys request From 66e3b1fb5fbb79151e78ef06d91960fb25221c0f Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 5 Oct 2026 19:21:13 -0700 Subject: [PATCH 08/24] feat(pbs): serve the gloas ePBS builder-API endpoints From the Gloas fork the beacon node calls three builder-API endpoints on Commit-Boost instead of get_header and get_payload: - POST /eth/v1/builder/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey} sends the request to the one relay its auth data names and returns that relay's bid (200), its 400 or 401, or 204 when it has no bid or fails. Commit-Boost does not validate or rank the bid: the beacon node checks it, and each request names exactly one builder. - POST /eth/v1/builder/builder_preferences/{proposer_pubkey} forwards to the relay the auth data names and returns that builder's answer. - POST /eth/v1/builder/beacon_blocks forwards the SSZ block bytes unparsed to every configured relay and answers 202 when one accepts. Requests must carry Eth-Consensus-Version: gloas. A body without Content-Type is JSON and a request without Accept gets JSON, as builder-specs requires. Bid requests need Date-Milliseconds and X-Timeout-Ms; Commit-Boost clamps the deadline to one slot, keeps proposer_deadline_buffer_ms back for the return trip and sends the rest to the builder as its own X-Timeout-Ms, or returns 204 when nothing is left. Routing follows builder-specs #168: auth data matches a relay when it equals the lowercase hostname of the relay URL. The first match in config order wins and unmatched auth data is a 400. Commit-Boost never verifies auth data; the builder does. Relay-supplied amounts are saturated. The new operator page (docs/get_started/epbs.md) covers writing each validator key's builder config through the keymanager API, with a copy-paste call, plus routing, timing, metrics and troubleshooting. Unreleased, shipping from v0.12.0-rc1. --- Cargo.lock | 2 + config.example.toml | 5 +- crates/common/Cargo.toml | 1 + crates/common/src/config/pbs.rs | 18 +- crates/common/src/config/signer.rs | 1 + crates/common/src/pbs/constants.rs | 11 + crates/common/src/pbs/error.rs | 2 +- crates/common/src/pbs/relay.rs | 84 +++ crates/common/src/pbs/types/mod.rs | 154 +++++- crates/common/src/utils.rs | 21 +- crates/common/src/wire.rs | 192 ++++++- crates/pbs/src/constants.rs | 6 + crates/pbs/src/error.rs | 48 +- crates/pbs/src/routes/builder_preferences.rs | 127 +++++ .../pbs/src/routes/execution_payload_bid.rs | 352 +++++++++++++ crates/pbs/src/routes/mod.rs | 6 + crates/pbs/src/routes/router.rs | 23 +- .../src/routes/submit_signed_beacon_block.rs | 97 ++++ crates/pbs/src/utils.rs | 250 +++++++++ docs/docs/get_started/epbs.md | 103 ++++ docs/sidebars.js | 1 + tests/Cargo.toml | 1 + tests/src/mock_relay.rs | 308 ++++++++++- tests/src/mock_validator.rs | 129 ++++- tests/src/utils.rs | 111 +++- tests/tests/pbs_cfg_file_update.rs | 1 + tests/tests/pbs_get_execution_payload_bid.rs | 479 ++++++++++++++++++ tests/tests/pbs_submit_builder_preferences.rs | 272 ++++++++++ tests/tests/pbs_submit_signed_beacon_block.rs | 162 ++++++ 29 files changed, 2920 insertions(+), 47 deletions(-) create mode 100644 crates/pbs/src/routes/builder_preferences.rs create mode 100644 crates/pbs/src/routes/execution_payload_bid.rs create mode 100644 crates/pbs/src/routes/submit_signed_beacon_block.rs create mode 100644 docs/docs/get_started/epbs.md create mode 100644 tests/tests/pbs_get_execution_payload_bid.rs create mode 100644 tests/tests/pbs_submit_builder_preferences.rs create mode 100644 tests/tests/pbs_submit_signed_beacon_block.rs diff --git a/Cargo.lock b/Cargo.lock index 0214e688b..140c33039 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1964,6 +1964,7 @@ dependencies = [ "tracing-subscriber", "tree_hash", "tree_hash_derive", + "typenum", "types", "unicode-normalization", "url", @@ -2067,6 +2068,7 @@ dependencies = [ "reqwest 0.13.5", "serde", "serde_json", + "ssz_types", "tempfile", "tokio", "tokio-tungstenite", diff --git a/config.example.toml b/config.example.toml index a825f6534..4ce750cb9 100644 --- a/config.example.toml +++ b/config.example.toml @@ -31,6 +31,9 @@ wait_all_registrations = true # this should be lower than that, leaving some margin for overhead # OPTIONAL, DEFAULT: 950 timeout_get_header_ms = 950 +# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline, must be under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing) +# OPTIONAL, DEFAULT: 50 +# proposer_deadline_buffer_ms = 50 # Timeout in milliseconds for the `submit_blinded_block` call to relays. # OPTIONAL, DEFAULT: 4000 timeout_get_payload_ms = 4000 @@ -139,7 +142,7 @@ frequency_get_header_ms = 300 # Configuration for the PBS multiplexers, which enable different configs to be used for get header requests, depending on validator pubkey # Note that: # - multiple sets of keys can be defined by adding multiple [[mux]] sections. The validator pubkey sets need to be disjoint -# - the mux is only used for get header requests +# - the mux is only used for get header requests, and (unreleased, from v0.12.0-rc1) ePBS bid and builder preferences requests # - if any value is missing from the mux config, the default value from the main config will be used [[mux]] # Unique ID for the mux config diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 23aa9a0c4..483b64a45 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -54,6 +54,7 @@ tracing-appender.workspace = true tracing-subscriber.workspace = true tree_hash.workspace = true tree_hash_derive.workspace = true +typenum.workspace = true unicode-normalization.workspace = true url.workspace = true uuid.workspace = true diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index cdade80ed..22f7a1eda 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -32,7 +32,7 @@ use crate::{ }, pbs::{ DEFAULT_PBS_PORT, DEFAULT_REGISTRY_REFRESH_SECONDS, DefaultTimeout, LATE_IN_SLOT_TIME_MS, - REGISTER_VALIDATOR_RETRY_LIMIT, RelayClient, RelayEntry, + PROPOSER_DEADLINE_BUFFER_MS, REGISTER_VALIDATOR_RETRY_LIMIT, RelayClient, RelayEntry, }, types::{BlsPublicKey, Chain, Jwt, ModuleId}, utils::{ @@ -202,9 +202,14 @@ pub struct PbsConfig { /// Minimum bid that will be accepted from get_header #[serde(rename = "min_bid_eth", with = "as_eth_str", default = "default_u256")] pub min_bid_wei: U256, - /// How late in the slot we consider to be "late" + /// How late in the slot we consider to be "late" (legacy get_header path) #[serde(default = "default_u64::")] pub late_in_slot_time_ms: u64, + /// ePBS bid path only: ms reserved before the proposer's declared deadline + /// (Date-Milliseconds + X-Timeout-Ms) for the winning bid's return trip. CB + /// asks the builder for `deadline - this`. + #[serde(default = "default_u64::")] + pub proposer_deadline_buffer_ms: u64, /// Enable extra validation of get_header responses #[serde(default = "default_bool::")] pub extra_validation_enabled: bool, @@ -245,6 +250,15 @@ impl PbsConfig { ); ensure!(self.late_in_slot_time_ms > 0, "late_in_slot_time_ms must be greater than 0"); + // The buffer comes out of the proposer's deadline, which is clamped to + // one slot, so a buffer of a slot or more leaves no time for the bid + // request. 0 is allowed (no reserve). + let slot_time_ms = chain.slot_time_sec().saturating_mul(1000); + ensure!( + self.proposer_deadline_buffer_ms < slot_time_ms, + "proposer_deadline_buffer_ms must be less than one slot ({slot_time_ms} ms)" + ); + ensure!( self.timeout_get_header_ms < self.late_in_slot_time_ms, "timeout_get_header_ms must be less than late_in_slot_time_ms" diff --git a/crates/common/src/config/signer.rs b/crates/common/src/config/signer.rs index 0ac6ce1b9..6d948adc2 100644 --- a/crates/common/src/config/signer.rs +++ b/crates/common/src/config/signer.rs @@ -476,6 +476,7 @@ mod tests { skip_sigverify: false, min_bid_wei: Uint::<256, 4>::from(0), late_in_slot_time_ms: 0, + proposer_deadline_buffer_ms: 0, extra_validation_enabled: false, rpc_url: None, http_timeout_seconds: 30, diff --git a/crates/common/src/pbs/constants.rs b/crates/common/src/pbs/constants.rs index 66ab42f06..e76395301 100644 --- a/crates/common/src/pbs/constants.rs +++ b/crates/common/src/pbs/constants.rs @@ -11,6 +11,11 @@ pub const REGISTER_VALIDATOR_PATH: &str = "/validators"; pub const SUBMIT_BLOCK_PATH: &str = "/blinded_blocks"; pub const RELOAD_PATH: &str = "/reload"; +pub const GET_EXECUTION_PAYLOAD_BID_PATH: &str = + "/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}"; +pub const SUBMIT_BUILDER_PREFERENCES_PATH: &str = "/builder_preferences/{proposer_pubkey}"; +pub const SUBMIT_SIGNED_BEACON_BLOCK_PATH: &str = "/beacon_blocks"; + // https://ethereum.github.io/builder-specs/#/Builder // Currently unused to enable a stateless default PBS module @@ -36,6 +41,12 @@ impl DefaultTimeout { pub const LATE_IN_SLOT_TIME_MS: u64 = 2000; +/// ePBS bid path: ms reserved before the proposer's own deadline +/// (Date-Milliseconds + X-Timeout-Ms) for the winning bid's return trip to the +/// beacon node and the beacon node's own selection/assembly. CB asks the +/// builder for `proposer_deadline - this`. +pub const PROPOSER_DEADLINE_BUFFER_MS: u64 = 50; + // Maximum number of retries for validator registration request per relay pub const REGISTER_VALIDATOR_RETRY_LIMIT: u32 = 3; diff --git a/crates/common/src/pbs/error.rs b/crates/common/src/pbs/error.rs index 8fcd928f1..1f40fa287 100644 --- a/crates/common/src/pbs/error.rs +++ b/crates/common/src/pbs/error.rs @@ -59,7 +59,7 @@ impl PbsError { } /// Extract the HTTP status code from relay-originated errors. - fn relay_status_code(&self) -> Option { + pub fn relay_status_code(&self) -> Option { match self { PbsError::RelayResponse { code, .. } => Some(*code), PbsError::ReadResponse(ResponseReadError::NonSuccess { status_code, .. }) => { diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index bb043df08..ae7f4b9aa 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -12,6 +12,7 @@ use super::{ HEADER_VERSION_KEY, HEADER_VERSION_VALUE, constants::{ GET_HEADER_STREAM_PATH, GET_STATUS_PATH, REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH, + SUBMIT_SIGNED_BEACON_BLOCK_PATH, }, error::PbsError, }; @@ -219,6 +220,39 @@ impl RelayClient { pub fn submit_block_url(&self, api_version: BuilderApiVersion) -> Result { self.builder_api_url(SUBMIT_BLOCK_PATH, api_version) } + + /// builder-API: POST /eth/v1/builder/execution_payload_bid/{slot}/ + /// {parent_hash}/{parent_root}/{proposer_pubkey} + pub fn get_execution_payload_bid_url( + &self, + slot: u64, + parent_hash: &B256, + parent_root: &B256, + validator_pubkey: &BlsPublicKey, + ) -> Result { + self.builder_api_url( + &format!( + "/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/{validator_pubkey}" + ), + BuilderApiVersion::V1, + ) + } + + /// builder-API: POST /eth/v1/builder/builder_preferences/{proposer_pubkey} + pub fn submit_builder_preferences_url( + &self, + validator_pubkey: &BlsPublicKey, + ) -> Result { + self.builder_api_url( + &format!("/builder_preferences/{validator_pubkey}"), + BuilderApiVersion::V1, + ) + } + + /// builder-API: POST /eth/v1/builder/beacon_blocks + pub fn submit_signed_beacon_block_url(&self) -> Result { + self.builder_api_url(SUBMIT_SIGNED_BEACON_BLOCK_PATH, BuilderApiVersion::V1) + } } /// First 4 bytes of the value's SHA-256: enough to see a rotation @@ -468,4 +502,54 @@ mod tests { let config = serde_json::from_str::(relay_config).unwrap(); assert_eq!(config.get_header, GetHeaderTransport::Http); } + + #[test] + fn test_relay_url_get_execution_payload() { + let slot = 0; + let parent_hash = B256::ZERO; + let parent_root = B256::ZERO; + let validator_pubkey = bls_pubkey_from_hex_unchecked( + "0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae", + ); + let expected = format!( + "http://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@abc.xyz/eth/v1/builder/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/{validator_pubkey}" + ); + + let relay_config = r#" + { + "url": "http://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@abc.xyz" + }"#; + + let config = serde_json::from_str::(relay_config).unwrap(); + let relay = RelayClient::new(config).unwrap(); + + assert_eq!( + relay + .get_execution_payload_bid_url(slot, &parent_hash, &parent_root, &validator_pubkey) + .unwrap() + .to_string(), + expected + ); + + let relay_config = r#" + { + "url": "http://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@abc.xyz//" + }"#; + + let config = serde_json::from_str::(relay_config).unwrap(); + let relay = RelayClient::new(config).unwrap(); + + assert_eq!( + relay + .get_execution_payload_bid_url(slot, &parent_hash, &parent_root, &validator_pubkey) + .unwrap() + .to_string(), + expected + ); + + assert_eq!( + relay.submit_builder_preferences_url(&validator_pubkey).unwrap().to_string(), + "http://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@abc.xyz/eth/v1/builder/builder_preferences/0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae" + ); + } } diff --git a/crates/common/src/pbs/types/mod.rs b/crates/common/src/pbs/types/mod.rs index 738221a8d..57845d4e1 100644 --- a/crates/common/src/pbs/types/mod.rs +++ b/crates/common/src/pbs/types/mod.rs @@ -1,10 +1,12 @@ use alloy::primitives::{B256, U256, b256}; pub use lh_eth2::ForkVersionedResponse; pub use lh_types::ForkName; -use lh_types::{BlindedPayload, ExecPayload, MainnetEthSpec}; +use lh_types::{BlindedPayload, ExecPayload, MainnetEthSpec, Slot}; use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use ssz_types::VariableList; -use crate::types::BlsPublicKey; +use crate::types::{BlsPublicKey, BlsSignature}; pub const EMPTY_TX_ROOT_HASH: B256 = b256!("7ffe241ea60187fdb0187bfa22de35d1f9bed7ab061d9401fd47e34a54fbede1"); @@ -13,6 +15,8 @@ pub type ExecutionRequests = lh_types::ExecutionRequests; /// Request object of POST `/eth/v1/builder/blinded_blocks` pub type SignedBlindedBeaconBlock = lh_types::SignedBlindedBeaconBlock; +/// Request object of POST `/eth/v1/builder/beacon_blocks` (Gloas onwards) +pub type SignedBeaconBlock = lh_types::SignedBeaconBlock; pub type BlindedBeaconBlock<'a> = lh_types::BeaconBlockRef<'a, MainnetEthSpec, BlindedPayload>; pub type BlindedBeaconBlockElectra = @@ -62,6 +66,52 @@ pub struct GetHeaderParams { pub pubkey: BlsPublicKey, } +pub type ExecutionPayloadBid = lh_types::ExecutionPayloadBid; +pub type SignedExecutionPayloadBid = lh_types::SignedExecutionPayloadBid; + +/// Response object of POST +/// `/eth/v1/builder/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/ +/// {proposer_pubkey}` +pub type GetExecutionPayloadBidResponse = ForkVersionedResponse; + +pub trait GetExecutionPayloadBidInfo { + fn block_hash(&self) -> B256; + fn value(&self) -> u64; + fn execution_payment(&self) -> u64; + fn builder_index(&self) -> u64; +} + +impl GetExecutionPayloadBidInfo for GetExecutionPayloadBidResponse { + fn block_hash(&self) -> B256 { + self.data.message.block_hash.0 + } + + fn value(&self) -> u64 { + self.data.message.value + } + + fn execution_payment(&self) -> u64 { + self.data.message.execution_payment + } + + fn builder_index(&self) -> u64 { + self.data.message.builder_index + } +} + +/// Path params of POST +/// `/eth/v1/builder/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/ +/// {proposer_pubkey}` +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct GetExecutionPayloadBidParams { + pub slot: u64, + /// The hash of the execution layer block the proposer will build on. + pub parent_hash: B256, + /// The root of the beacon block the proposer will build on. + pub parent_root: B256, + pub proposer_pubkey: BlsPublicKey, +} + pub trait GetHeaderInfo { fn block_hash(&self) -> B256; fn value(&self) -> &U256; @@ -107,3 +157,103 @@ impl GetPayloadInfo for SignedBlindedBeaconBlock { self.message().body().execution_payload().map(|r| r.parent_hash().0).unwrap_or_default() } } + +/// Spec limit on `auth.message.data` (builder-specs +/// `types/gloas/request_auth.yaml`). +pub type MaxBuilderAuthData = typenum::U4096; + +// `BuilderRequestAuth` is used to authenticate requests to a builder. +#[derive(Debug, Serialize, Deserialize, Encode, Decode, Clone)] +pub struct BuilderRequestAuth { + /// Opaque authentication data agreed with the builder out of band; hex + /// string on the JSON wire + #[serde(with = "ssz_types::serde_utils::hex_var_list")] + pub data: VariableList, + pub slot: Slot, +} + +#[derive(Debug, Serialize, Deserialize, Encode, Decode, Clone)] +pub struct SignedBuilderRequestAuth { + pub message: BuilderRequestAuth, + pub signature: BlsSignature, +} + +/// Per-builder preferences a proposer submits ahead of the bid request. +#[derive(Debug, Serialize, Deserialize, Encode, Decode, Clone)] +pub struct BuilderPreferences { + /// Maximum execution-layer payment, in Gwei, this proposer will accept from + /// this builder; quoted string on the JSON wire + #[serde(with = "serde_utils::quoted_u64")] + pub max_execution_payment: u64, +} + +/// The `submitBuilderPreferences` request body. +#[derive(Debug, Serialize, Deserialize, Encode, Decode, Clone)] +pub struct BuilderPreferencesRequest { + pub preferences: BuilderPreferences, + pub auth: SignedBuilderRequestAuth, +} + +/// Path params for `POST /eth/v1/builder/builder_preferences/{proposer_pubkey}` +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct SubmitBuilderPreferencesParams { + /// The public key of the proposer expressing these preferences + pub proposer_pubkey: BlsPublicKey, +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Spec vector for the SSZ layout of `BuilderPreferencesRequest`. + /// The order-determining fixed part is cross-checked byte-for-byte + /// against the canonical example + /// `examples/gloas/builder_preferences_request.ssz`. + #[test] + fn test_builder_preferences_request_ssz_spec_vector() { + use ssz::{Decode, Encode}; + + // The infinity signature: 0xc0 followed by 95 zero bytes + let mut infinity_sig = vec![0u8; 96]; + infinity_sig[0] = 0xc0; + + let auth = SignedBuilderRequestAuth { + message: BuilderRequestAuth { + data: VariableList::new(vec![0x12, 0x34, 0x56, 0x78, 0x90, 0xab, 0xcd, 0xef]) + .unwrap(), + slot: Slot::new(1234), + }, + signature: BlsSignature::deserialize(&infinity_sig).unwrap(), + }; + let request = BuilderPreferencesRequest { + preferences: BuilderPreferences { max_execution_payment: 1_000_000_000 }, + auth: auth.clone(), + }; + + // Outer container is `(preferences, auth)`: the 8-byte fixed + // `max_execution_payment` LE, then a 4-byte offset to the variable-size + // `auth` (12 = 8-byte fixed `preferences` + 4-byte offset), then `auth`. + let auth_bytes = auth.as_ssz_bytes(); + let mut expected = Vec::new(); + expected.extend_from_slice(&1_000_000_000u64.to_le_bytes()); + expected.extend_from_slice(&12u32.to_le_bytes()); + expected.extend_from_slice(&auth_bytes); + + assert_eq!(request.as_ssz_bytes(), expected); + + // The fixed part matches the canonical spec example byte-for-byte: + // `max_execution_payment` (1_000_000_000 Gwei LE) precedes the `auth` + // offset (12). A `(auth, preferences)` layout would put the offset first. + assert_eq!(&request.as_ssz_bytes()[..12], &[ + 0x00, 0xca, 0x9a, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + ]); + + let decoded = BuilderPreferencesRequest::from_ssz_bytes(&expected).unwrap(); + assert_eq!(decoded.preferences.max_execution_payment, 1_000_000_000); + assert_eq!(decoded.auth.message.slot, Slot::new(1234)); + assert_eq!(decoded.auth.message.data.to_vec(), vec![ + 0x12, 0x34, 0x56, 0x78, 0x90, 0xab, 0xcd, 0xef + ]); + assert_eq!(decoded.auth.signature.serialize().to_vec(), infinity_sig); + } +} diff --git a/crates/common/src/utils.rs b/crates/common/src/utils.rs index c026849db..c167c33db 100644 --- a/crates/common/src/utils.rs +++ b/crates/common/src/utils.rs @@ -26,11 +26,13 @@ use crate::{ const MILLIS_PER_SECOND: u64 = 1_000; +// Saturating: an attacker-supplied huge slot must clamp to the far future, not +// overflow-panic in debug builds and drop the connection with no response pub fn timestamp_of_slot_start_sec(slot: u64, chain: Chain) -> u64 { - chain.genesis_time_sec() + slot * chain.slot_time_sec() + chain.genesis_time_sec().saturating_add(slot.saturating_mul(chain.slot_time_sec())) } pub fn timestamp_of_slot_start_millis(slot: u64, chain: Chain) -> u64 { - timestamp_of_slot_start_sec(slot, chain) * MILLIS_PER_SECOND + timestamp_of_slot_start_sec(slot, chain).saturating_mul(MILLIS_PER_SECOND) } pub fn ms_into_slot(slot: u64, chain: Chain) -> u64 { let slot_start_ms = timestamp_of_slot_start_millis(slot, chain); @@ -466,14 +468,25 @@ mod test { use alloy::primitives::keccak256; use super::{ - create_admin_jwt, create_jwt, decode_admin_jwt, decode_jwt, random_jwt_secret, + create_admin_jwt, create_jwt, decode_admin_jwt, decode_jwt, ms_into_slot, + random_jwt_secret, timestamp_of_slot_start_millis, timestamp_of_slot_start_sec, validate_admin_jwt, validate_jwt, }; use crate::{ constants::SIGNER_JWT_EXPIRATION, - types::{Jwt, JwtAdminClaims, ModuleId}, + types::{Chain, Jwt, JwtAdminClaims, ModuleId}, }; + // An attacker-supplied huge slot must saturate to the far future, not + // overflow-panic in debug builds and drop the connection with no response + #[test] + fn test_slot_timestamp_saturates_on_huge_slot() { + assert_eq!(timestamp_of_slot_start_sec(u64::MAX, Chain::Mainnet), u64::MAX); + assert_eq!(timestamp_of_slot_start_millis(u64::MAX, Chain::Mainnet), u64::MAX); + // The far-future slot has not started, so no time has elapsed into it + assert_eq!(ms_into_slot(u64::MAX, Chain::Mainnet), 0); + } + #[test] fn test_jwt_validation_no_payload_hash() { // Check valid JWT diff --git a/crates/common/src/wire.rs b/crates/common/src/wire.rs index 4f4263980..16037c5e0 100644 --- a/crates/common/src/wire.rs +++ b/crates/common/src/wire.rs @@ -6,12 +6,13 @@ use axum::http::HeaderValue; use bytes::Bytes; use futures::StreamExt; use headers_accept::Accept; -use lh_types::{BeaconBlock, ForkName, SignedBeaconBlock, map_fork_name}; +use lh_types::{BeaconBlock, ForkName, SignedBeaconBlock as LhSignedBeaconBlock, map_fork_name}; use mediatype::{MediaType, ReadParams, names}; use reqwest::{ Response, header::{ACCEPT, CONTENT_TYPE, HeaderMap, ToStrError}, }; +use ssz::Decode; use thiserror::Error; use crate::pbs::{HEADER_VERSION_VALUE, SignedBlindedBeaconBlock}; @@ -72,7 +73,6 @@ pub async fn read_chunked_body_with_max( max_size: usize, request_url: &str, ) -> Result, ResponseReadError> { - // Get the content length from the response headers #[cfg(not(feature = "testing-flags"))] let content_length = res.content_length(); @@ -290,6 +290,38 @@ pub fn get_content_type(req_headers: &HeaderMap) -> EncodingType { .unwrap_or(EncodingType::Json) } +/// Reads `Eth-Consensus-Version`, which builder-specs requires on every request +/// with a body: absent, or naming a fork this build does not recognize, is a +/// 400. +pub fn require_consensus_version_header( + req_headers: &HeaderMap, +) -> Result { + let value = req_headers + .get(CONSENSUS_VERSION_HEADER) + .ok_or(BodyDeserializeError::MissingVersionHeader)?; + let value = value + .to_str() + .map_err(|_| BodyDeserializeError::InvalidVersionHeader("".to_string()))?; + if value.is_empty() { + return Err(BodyDeserializeError::InvalidVersionHeader("".to_string())); + } + // Echoed into the 400 body, so bound attacker-controlled length + let unsupported = + || BodyDeserializeError::InvalidVersionHeader(value.chars().take(64).collect()); + // Gloas-only until later forks are defined + match ForkName::from_str(value).map_err(|_| unsupported())? { + ForkName::Gloas => Ok(ForkName::Gloas), + ForkName::Base | + ForkName::Altair | + ForkName::Bellatrix | + ForkName::Capella | + ForkName::Deneb | + ForkName::Electra | + ForkName::Fulu | + ForkName::Heze => Err(unsupported()), + } +} + pub fn get_consensus_version_header(req_headers: &HeaderMap) -> Option { ForkName::from_str( req_headers @@ -384,23 +416,28 @@ pub enum BodyDeserializeError { UnsupportedMediaType, #[error("missing consensus version header")] MissingVersionHeader, + #[error("unsupported consensus version header: {0}")] + InvalidVersionHeader(String), + #[error("missing request body")] + MissingBody, +} + +/// The request body encoding to decode with, from the Content-Type. +pub fn content_type_encoding(headers: &HeaderMap) -> Result { + match headers.get(CONTENT_TYPE) { + None => Ok(NO_PREFERENCE_DEFAULT), + Some(hv) => { + let value = hv.to_str().map_err(|_| BodyDeserializeError::UnsupportedMediaType)?; + EncodingType::from_str(value).map_err(|_| BodyDeserializeError::UnsupportedMediaType) + } + } } pub fn deserialize_body( headers: &HeaderMap, body: Bytes, ) -> Result { - // Determine the encoding to decode with. Precedence: - // - Content-Type absent โ†’ NO_PREFERENCE_DEFAULT - // - Content-Type recognized โ†’ use it. - // - Content-Type present but unrecognized โ†’ UnsupportedMediaType. - let encoding = match headers.get(CONTENT_TYPE) { - None => NO_PREFERENCE_DEFAULT, - Some(hv) => { - let value = hv.to_str().map_err(|_| BodyDeserializeError::UnsupportedMediaType)?; - EncodingType::from_str(value).map_err(|_| BodyDeserializeError::UnsupportedMediaType)? - } - }; + let encoding = content_type_encoding(headers)?; match encoding { EncodingType::Json => match get_consensus_version_header(headers) { @@ -409,13 +446,13 @@ pub fn deserialize_body( // reports the wrong fork for every Fulu block. Some(version) => Ok(map_fork_name!( version, - SignedBeaconBlock, + LhSignedBeaconBlock, serde_json::from_slice(&body).map_err(BodyDeserializeError::SerdeJsonError)? )), // builder-specs doesn't require the header for JSON bodies. // A request without it still has to decode and an untagged decode would silently pick // Electra. Assume Fulu to be conservative until ePBS warrants the refactor - None => Ok(SignedBeaconBlock::Fulu( + None => Ok(LhSignedBeaconBlock::Fulu( serde_json::from_slice(&body).map_err(BodyDeserializeError::SerdeJsonError)?, )), }, @@ -429,6 +466,37 @@ pub fn deserialize_body( } } +/// Decode a fork-versioned ePBS request body (builder-specs fork-versions +/// `SignedBuilderRequestAuth` and `BuilderPreferencesRequest`) as JSON or SSZ, +/// defaulting to JSON when no `Content-Type` is set. An empty body is rejected +/// first so a missing body reads as `MissingBody`. `Eth-Consensus-Version` is +/// required for BOTH encodings and its value must name a fork this build +/// recognizes (absent -> `MissingVersionHeader`, unrecognized -> +/// `InvalidVersionHeader`, both -> 400), per builder-specs +/// specs/gloas/builder.md. +pub fn decode_versioned_request_body( + headers: &HeaderMap, + body: &Bytes, +) -> Result +where + T: serde::de::DeserializeOwned + Decode, +{ + if body.is_empty() { + return Err(BodyDeserializeError::MissingBody); + } + // Content-Type first so an unsupported media type stays a 415 + let encoding = content_type_encoding(headers)?; + require_consensus_version_header(headers)?; + match encoding { + EncodingType::Json => { + serde_json::from_slice(body.as_ref()).map_err(BodyDeserializeError::SerdeJsonError) + } + EncodingType::Ssz => { + T::from_ssz_bytes(body.as_ref()).map_err(BodyDeserializeError::SszDecodeError) + } + } +} + #[cfg(test)] mod test { use axum::http::{HeaderMap, HeaderName, HeaderValue}; @@ -1002,4 +1070,98 @@ mod test { "a headerless JSON body must not fall back to the untagged Electra match" ); } + + // โ”€โ”€ decode_versioned_request_body โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + fn sample_preferences_request() -> crate::pbs::BuilderPreferencesRequest { + use crate::pbs::{BuilderPreferences, BuilderRequestAuth, SignedBuilderRequestAuth}; + crate::pbs::BuilderPreferencesRequest { + preferences: BuilderPreferences { max_execution_payment: 7 }, + auth: SignedBuilderRequestAuth { + message: BuilderRequestAuth { + data: vec![0xde, 0xad].try_into().unwrap(), + slot: lh_types::Slot::new(3), + }, + signature: crate::types::BlsSignature::empty(), + }, + } + } + + fn versioned_headers( + content_type: Option<&'static str>, + version: Option<&'static str>, + ) -> HeaderMap { + let mut headers = HeaderMap::new(); + if let Some(ct) = content_type { + headers.insert(CONTENT_TYPE, HeaderValue::from_static(ct)); + } + if let Some(v) = version { + headers.insert( + HeaderName::try_from(CONSENSUS_VERSION_HEADER).unwrap(), + HeaderValue::from_static(v), + ); + } + headers + } + + /// The three ePBS request bodies share this decoder, so the encoding and + /// version-header rules are pinned here once rather than per endpoint. + #[test] + fn test_decode_versioned_request_body() { + use ssz::Encode; + + use super::decode_versioned_request_body; + use crate::pbs::BuilderPreferencesRequest; + + let request = sample_preferences_request(); + let ssz_body = Bytes::from(request.as_ssz_bytes()); + let json_body = Bytes::from(serde_json::to_vec(&request).unwrap()); + let decode = |headers: &HeaderMap, body: &Bytes| { + decode_versioned_request_body::(headers, body) + }; + + // An empty body is a missing body, whatever the headers say: checked + // before the version header, so a bare POST names the body + assert!(matches!( + decode(&versioned_headers(None, None), &Bytes::new()), + Err(BodyDeserializeError::MissingBody) + )); + + // No Content-Type means JSON (builder-specs), so SSZ must be labeled + let decoded = decode(&versioned_headers(None, Some("gloas")), &json_body).unwrap(); + assert_eq!(decoded.auth.message.slot.as_u64(), 3); + assert!(matches!( + decode(&versioned_headers(None, Some("gloas")), &ssz_body), + Err(BodyDeserializeError::SerdeJsonError(_)) + )); + let decoded = + decode(&versioned_headers(Some(APPLICATION_OCTET_STREAM), Some("gloas")), &ssz_body) + .unwrap(); + assert_eq!(decoded.preferences.max_execution_payment, 7); + + // lighthouse's FromStr lowercases, so an uppercase gloas passes + decode(&versioned_headers(None, Some("GLOAS")), &json_body).unwrap(); + + // The version header is required for both encodings + for (ct, body) in [(Some(APPLICATION_OCTET_STREAM), &ssz_body), (None, &json_body)] { + assert!(matches!( + decode(&versioned_headers(ct, None), body), + Err(BodyDeserializeError::MissingVersionHeader) + )); + // Only gloas is accepted: an earlier fork or an unknown name names + // the bad value + for bad in ["fulu", "electra", "futurefork"] { + assert!(matches!( + decode(&versioned_headers(ct, Some(bad)), body), + Err(BodyDeserializeError::InvalidVersionHeader(ref v)) if v == bad + )); + } + } + + // An unsupported media type is a 415 before the version header is read + assert!(matches!( + decode(&versioned_headers(Some("text/plain"), None), &ssz_body), + Err(BodyDeserializeError::UnsupportedMediaType) + )); + } } diff --git a/crates/pbs/src/constants.rs b/crates/pbs/src/constants.rs index 3d5ee7258..5790af416 100644 --- a/crates/pbs/src/constants.rs +++ b/crates/pbs/src/constants.rs @@ -4,6 +4,9 @@ pub const STATUS_ENDPOINT_TAG: &str = "status"; pub const REGISTER_VALIDATOR_ENDPOINT_TAG: &str = "register_validator"; pub const SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG: &str = "submit_blinded_block"; pub const GET_HEADER_ENDPOINT_TAG: &str = "get_header"; +pub const GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG: &str = "get_execution_payload_bid"; +pub const SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG: &str = "submit_builder_preferences"; +pub const SUBMIT_SIGNED_BEACON_BLOCK_ENDPOINT_TAG: &str = "submit_signed_beacon_block"; pub const RELOAD_ENDPOINT_TAG: &str = "reload"; pub const GET_HEADER_STREAM_ENDPOINT_TAG: &str = "get_header_stream"; @@ -32,6 +35,9 @@ pub const MAX_SIZE_SUBMIT_BLOCK_RESPONSE: usize = 20 * 1024 * 1024; /// 20 MiB, enough to process ~45000 registrations in one request pub const MAX_SIZE_REGISTER_VALIDATOR_REQUEST: usize = 20 * 1024 * 1024; +/// A Gloas block carries the bid, not the payload: blinded-block-sized +pub const MAX_SIZE_SUBMIT_SIGNED_BEACON_BLOCK: usize = MAX_SIZE_SUBMIT_BLOCK_RESPONSE; + /// 5 MiB, to account for max execution requests / commitments pub const MAX_SIZE_GET_HEADER_RESPONSE: usize = 5 * 1024 * 1024; diff --git a/crates/pbs/src/error.rs b/crates/pbs/src/error.rs index 98f8a2f16..9b9aaf078 100644 --- a/crates/pbs/src/error.rs +++ b/crates/pbs/src/error.rs @@ -20,6 +20,20 @@ struct ErrorResponse { pub enum PbsClientError { #[error("no response from relays")] NoResponse, + /// 500, not 502: 502 is in neither endpoint's builder-specs response set. + /// Legacy routes keep `NoResponse` -> 502. + #[error("no builder accepted the submission")] + NoBuilderResponse, + #[error("auth data does not match a configured builder")] + AuthDataMismatch, + #[error("missing or invalid timing headers")] + MissingTimingHeader, + #[error("auth slot does not match the request path")] + AuthSlotMismatch, + /// Propagated so the proposer learns which builder was rejected; a blanket + /// 500 would hide that. + #[error("the addressed builder rejected the request with {}", .0.as_u16())] + BuilderRejected(StatusCode), #[error("no payload from relays")] NoPayload, #[error("internal server error")] @@ -34,6 +48,11 @@ impl PbsClientError { pub fn status_code(&self) -> StatusCode { match self { PbsClientError::NoResponse => StatusCode::BAD_GATEWAY, + PbsClientError::NoBuilderResponse => StatusCode::INTERNAL_SERVER_ERROR, + PbsClientError::AuthDataMismatch => StatusCode::BAD_REQUEST, + PbsClientError::MissingTimingHeader => StatusCode::BAD_REQUEST, + PbsClientError::AuthSlotMismatch => StatusCode::BAD_REQUEST, + PbsClientError::BuilderRejected(code) => *code, PbsClientError::NoPayload => StatusCode::BAD_GATEWAY, PbsClientError::Internal => StatusCode::INTERNAL_SERVER_ERROR, PbsClientError::DecodeError(BodyDeserializeError::UnsupportedMediaType) => { @@ -50,10 +69,25 @@ impl IntoResponse for PbsClientError { let status = self.status_code(); let message = match &self { PbsClientError::NoResponse => "no response from relays".to_string(), + PbsClientError::NoBuilderResponse => "no builder accepted the submission".to_string(), + PbsClientError::AuthDataMismatch => { + "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder".to_string() + } + PbsClientError::MissingTimingHeader => { + "Invalid request: Date-Milliseconds and X-Timeout-Ms headers are required".to_string() + } + PbsClientError::AuthSlotMismatch => { + "Invalid SignedBuilderRequestAuth: auth.message.slot does not match the proposal slot in the request path".to_string() + } + // The builder's own body is never forwarded: it is untrusted and may be + // arbitrarily large + PbsClientError::BuilderRejected(code) => { + format!("The addressed builder rejected the request with status {}", code.as_u16()) + } PbsClientError::NoPayload => "no payload from relays".to_string(), PbsClientError::Internal => "internal server error".to_string(), - PbsClientError::DecodeError(e) => format!("error decoding request: {e}"), - PbsClientError::HeaderError(e) => format!("header error: {e}"), + PbsClientError::DecodeError(err) => format!("error decoding request: {err}"), + PbsClientError::HeaderError(err) => format!("header error: {err}"), }; // Return the spec's JSON `ErrorMessage` rather than plain text so clients @@ -80,6 +114,16 @@ mod test { PbsClientError::DecodeError(BodyDeserializeError::MissingVersionHeader).status_code(), StatusCode::BAD_REQUEST, ); + // The unrecognized-fork variant must stay 400: it sits under the + // variant-specific 415 arm, and only the DecodeError(_) catch-all + // routes it today + assert_eq!( + PbsClientError::DecodeError(BodyDeserializeError::InvalidVersionHeader( + "futurefork".to_string() + )) + .status_code(), + StatusCode::BAD_REQUEST, + ); } #[tokio::test] diff --git a/crates/pbs/src/routes/builder_preferences.rs b/crates/pbs/src/routes/builder_preferences.rs new file mode 100644 index 000000000..fab0902aa --- /dev/null +++ b/crates/pbs/src/routes/builder_preferences.rs @@ -0,0 +1,127 @@ +use axum::{ + body::Bytes, + extract::{Path, State}, + http::HeaderMap, + response::IntoResponse, +}; +use cb_common::{ + pbs::{ + BuilderPreferencesRequest, RelayClient, SubmitBuilderPreferencesParams, error::PbsError, + }, + wire::{decode_versioned_request_body, get_user_agent}, +}; +use reqwest::StatusCode; +use ssz::Encode; +use tracing::{debug, error, info}; + +use crate::{ + PbsStateGuard, + constants::SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG, + error::PbsClientError, + state::{BuilderApiState, PbsState}, + utils::{ + builder_rejection, epbs_base_send_headers, log_mux_selection, post_ssz_expect_accepted, + record_beacon_status, record_client_error, record_request_failure, resolve_addressed_relay, + }, +}; + +pub async fn handle_submit_builder_preferences( + State(state): State>, + req_headers: HeaderMap, + Path(params): Path, + body: Bytes, +) -> Result { + let request = + decode_versioned_request_body::(&req_headers, &body) + .map_err(|err| record_client_error(err, SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG))?; + tracing::Span::current().record("validator", tracing::field::debug(¶ms.proposer_pubkey)); + tracing::Span::current().record("slot", request.auth.message.slot.as_u64()); + + let state = state.read().clone(); + let ua = get_user_agent(&req_headers); + info!( + ua, + slot = %request.auth.message.slot, + max_execution_payment = request.preferences.max_execution_payment, + "new request" + ); + + match submit_builder_preferences(params, request, req_headers, state).await { + Ok(()) => { + record_beacon_status("202", SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG); + Ok(StatusCode::ACCEPTED.into_response()) + } + Err(err) => Err(record_request_failure(err, SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG)), + } +} + +/// Implements https://ethereum.github.io/builder-specs/?urls.primaryName=dev#/Builder/submitBuilderPreferences +/// Returns 202 if the addressed builder accepts +pub async fn submit_builder_preferences( + params: SubmitBuilderPreferencesParams, + request: BuilderPreferencesRequest, + req_headers: HeaderMap, + state: PbsState, +) -> Result<(), PbsClientError> { + let (pbs_config, relays, maybe_mux_id) = state.mux_config_and_relays(¶ms.proposer_pubkey); + + log_mux_selection(maybe_mux_id, relays.len(), ¶ms.proposer_pubkey); + + let relay = resolve_addressed_relay(relays, request.auth.message.data.as_ref())?; + + let send_headers = epbs_base_send_headers(&req_headers)?; + + // SSZ on the relay hop + let body = Bytes::from(request.as_ssz_bytes()); + + // Preferences are submitted an epoch ahead, so they share the registration + // timeout rather than the block-production one + let timeout_ms = pbs_config.timeout_register_validator_ms; + + let relay_id = relay.id.clone(); + match send_submit_builder_preferences( + params.proposer_pubkey, + body, + relay, + send_headers, + timeout_ms, + ) + .await + { + Ok(()) => { + info!(%relay_id, "builder preferences submitted"); + Ok(()) + } + Err(err) => { + if err.is_timeout() { + error!(err = "Timed Out", %relay_id); + } else { + error!(%err, %relay_id); + } + Err(builder_rejection(&err).unwrap_or(PbsClientError::NoBuilderResponse)) + } + } +} + +async fn send_submit_builder_preferences( + proposer_pubkey: cb_common::types::BlsPublicKey, + body: Bytes, + relay: RelayClient, + headers: HeaderMap, + timeout_ms: u64, +) -> Result<(), PbsError> { + let url = relay.submit_builder_preferences_url(&proposer_pubkey)?; + + let request_latency = post_ssz_expect_accepted( + &relay, + url, + body, + headers, + timeout_ms, + SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG, + ) + .await?; + + debug!(relay_id = relay.id.as_ref(), latency = ?request_latency, "preferences accepted"); + Ok(()) +} diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs new file mode 100644 index 000000000..68c96774f --- /dev/null +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -0,0 +1,352 @@ +use std::time::Duration; + +use axum::{ + body::Bytes, + extract::{Path, State}, + http::{HeaderMap, HeaderValue}, + response::{IntoResponse, Response}, +}; +use cb_common::{ + pbs::{ + GetExecutionPayloadBidInfo, GetExecutionPayloadBidParams, GetExecutionPayloadBidResponse, + HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, + SignedExecutionPayloadBid, error::PbsError, + }, + utils::{ms_into_slot, utcnow_ms}, + wire::{ + CONSENSUS_VERSION_HEADER, EncodingType, OUTBOUND_ACCEPT_SSZ_FIRST, + decode_versioned_request_body, get_accept_types, get_user_agent, + parse_response_encoding_and_fork, safe_read_http_response, + }, +}; +use reqwest::{ + StatusCode, + header::{ACCEPT, CONTENT_TYPE}, +}; +use ssz::{Decode, Encode}; +use tracing::{debug, error, info, warn}; + +use crate::{ + PbsStateGuard, + constants::{GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE}, + error::PbsClientError, + metrics::{RELAY_HEADER_VALUE, RELAY_LAST_SLOT}, + state::{BuilderApiState, PbsState}, + utils::{ + builder_rejection, epbs_base_send_headers, format_gwei_as_eth, log_mux_selection, + record_beacon_status, record_client_error, record_request_failure, resolve_addressed_relay, + send_to_relay, + }, +}; + +pub async fn handle_get_execution_payload_bid( + State(state): State>, + req_headers: HeaderMap, + Path(params): Path, + body: Bytes, +) -> Result { + let auth = decode_versioned_request_body::(&req_headers, &body) + .map_err(|err| record_client_error(err, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG))?; + tracing::Span::current().record("slot", params.slot); + tracing::Span::current().record("parent_hash", tracing::field::debug(params.parent_hash)); + tracing::Span::current().record("parent_root", tracing::field::debug(params.parent_root)); + tracing::Span::current().record("validator", tracing::field::debug(¶ms.proposer_pubkey)); + + let state = state.read().clone(); + + let ua = get_user_agent(&req_headers); + let ms_into_slot = ms_into_slot(params.slot, state.config.chain); + + let response_encoding = get_accept_types(&req_headers) + .inspect_err(|err| error!(%err, "error parsing accept header")) + .map_err(|err| record_client_error(err, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG))? + .primary; + + info!(ua, ms_into_slot, "new request"); + + match get_execution_payload_bid(params, auth, req_headers, state).await { + Ok(Some(bid)) => { + encode_bid_response(bid, response_encoding, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG) + } + Ok(None) => { + info!("no bid for slot"); + record_beacon_status("204", GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG); + Ok(StatusCode::NO_CONTENT.into_response()) + } + Err(err) => Err(record_request_failure(err, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG)), + } +} + +fn encode_bid_response( + bid: GetExecutionPayloadBidResponse, + response_encoding: EncodingType, + endpoint: &str, +) -> Result { + info!( + trustless_bid_eth = format_gwei_as_eth(bid.value()), + execution_payment_eth = format_gwei_as_eth(bid.execution_payment()), + block_hash = %bid.block_hash(), + builder_index = bid.builder_index(), + "received header" + ); + + // Eth-Consensus-Version is required on the 200 for both encodings + let consensus_version_header = HeaderValue::from_str(&bid.version.to_string()) + .expect("fork name is always a valid header value"); + + record_beacon_status("200", endpoint); + let mut res = match response_encoding { + EncodingType::Ssz => { + let mut res = bid.data.as_ssz_bytes().into_response(); + res.headers_mut().insert(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()); + res + } + EncodingType::Json => axum::Json(bid).into_response(), + }; + res.headers_mut().insert(CONSENSUS_VERSION_HEADER, consensus_version_header); + Ok(res) +} + +/// Implements https://ethereum.github.io/builder-specs/?urls.primaryName=dev#/Builder/getExecutionPayloadBid +/// Returns 200 with the addressed builder's bid, else 204 +pub async fn get_execution_payload_bid( + params: GetExecutionPayloadBidParams, + auth: SignedBuilderRequestAuth, + req_headers: HeaderMap, + state: PbsState, +) -> Result, PbsClientError> { + let (pbs_config, relays, maybe_mux_id) = state.mux_config_and_relays(¶ms.proposer_pubkey); + + log_mux_selection(maybe_mux_id, relays.len(), ¶ms.proposer_pubkey); + + if auth.message.slot.as_u64() != params.slot { + warn!(auth_slot = %auth.message.slot, path_slot = params.slot, "auth slot mismatch"); + return Err(PbsClientError::AuthSlotMismatch); + } + + let relay = resolve_addressed_relay(relays, auth.message.data.as_ref())?; + + // The beacon node's deadline, not timeout_get_header_ms, bounds the request + let slot_ms = state.config.chain.slot_time_sec().saturating_mul(1000); + let budget_ms = request_budget_ms(&req_headers, utcnow_ms(), slot_ms)?; + let max_timeout_ms = budget_ms.saturating_sub(pbs_config.proposer_deadline_buffer_ms); + debug!( + budget_ms, + buffer_ms = pbs_config.proposer_deadline_buffer_ms, + max_timeout_ms, + "ePBS bid request budget" + ); + + // No bid could reach the beacon node before its deadline + if max_timeout_ms == 0 { + warn!(budget_ms, "proposer deadline reached, no time to solicit a bid"); + return Ok(None); + } + + let mut send_headers = epbs_base_send_headers(&req_headers)?; + + // The bid is re-encoded for the BN anyway, so ask for the cheaper SSZ + send_headers.insert(ACCEPT, OUTBOUND_ACCEPT_SSZ_FIRST.clone()); + let body = Bytes::from(auth.as_ssz_bytes()); + + let slot = params.slot; + let relay_id = relay.id.clone(); + // A relay that errors or times out contributes no bid: 204, never a 502. + // The builder's own 400 and 401 still reach the proposer. + match send_get_execution_payload_bid(params, body, relay, send_headers, max_timeout_ms).await { + Ok(Some(bid)) => { + RELAY_LAST_SLOT.with_label_values(&[relay_id.as_str()]).set(slot as i64); + // value() is already gwei (the gauge is labelled gwei), so it is set unscaled + RELAY_HEADER_VALUE + .with_label_values(&[relay_id.as_str()]) + .set(i64::try_from(bid.value()).unwrap_or_default()); + Ok(Some(bid)) + } + Ok(None) => Ok(None), + Err(err) if err.is_timeout() => { + error!(err = "Timed Out", %relay_id, timeout_ms = max_timeout_ms); + Ok(None) + } + Err(err) => { + error!(%err, %relay_id); + builder_rejection(&err).map_or(Ok(None), Err) + } + } +} + +/// Milliseconds left until `Date-Milliseconds + X-Timeout-Ms`, but no more than +/// `X-Timeout-Ms` from now (a proposer clock running ahead) or one slot (the +/// header has no upper bound). 0 once the deadline has passed. +fn request_budget_ms( + req_headers: &HeaderMap, + now_ms: u64, + slot_ms: u64, +) -> Result { + fn header_u64(req_headers: &HeaderMap, name: &str) -> Result { + req_headers + .get(name) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .ok_or(PbsClientError::MissingTimingHeader) + } + + let sent_at_ms = header_u64(req_headers, HEADER_START_TIME_UNIX_MS)?; + let timeout_ms = header_u64(req_headers, HEADER_TIMEOUT_MS)?; + + let until_deadline = sent_at_ms.saturating_add(timeout_ms).saturating_sub(now_ms); + Ok(until_deadline.min(timeout_ms).min(slot_ms)) +} + +/// Sends the bid request to the relay with whatever remains of the proposer's +/// deadline. +async fn send_get_execution_payload_bid( + params: GetExecutionPayloadBidParams, + body: Bytes, + relay: RelayClient, + mut headers: HeaderMap, + timeout_ms: u64, +) -> Result, PbsError> { + let url = relay.get_execution_payload_bid_url( + params.slot, + ¶ms.parent_hash, + ¶ms.parent_root, + ¶ms.proposer_pubkey, + )?; + + headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); + + // The timeout header indicating how long a relay has to respond + headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); + + let request = relay + .client + .post(url) + .timeout(Duration::from_millis(timeout_ms)) + .headers(headers) + .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .body(body); + let (res, request_latency) = + send_to_relay(request, &relay, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG).await?; + let code = res.status(); + + // Parse the negotiated Content-Type (and optional fork) before the body is + // consumed. Only successful responses carry a meaningful encoding; on + // non-success we fall through to safe_read_http_response's NonSuccess error, + // so these values are never consumed. + let (content_type, fork) = if code.is_success() { + parse_response_encoding_and_fork(res.headers(), code.as_u16())? + } else { + (EncodingType::Json, None) + }; + + let response_bytes = safe_read_http_response(res, MAX_SIZE_GET_HEADER_RESPONSE).await?; + let header_size_bytes = response_bytes.len(); + if code == StatusCode::NO_CONTENT { + debug!( + relay_id = relay.id.as_ref(), + ?code, + latency = ?request_latency, + response = ?response_bytes, + "no header from relay" + ); + return Ok(None); + } + + let get_header_response = match content_type { + EncodingType::Json => serde_json::from_slice::( + &response_bytes, + ) + .map_err(|err| PbsError::JsonDecode { + err, + raw: String::from_utf8_lossy(&response_bytes).into_owned(), + })?, + EncodingType::Ssz => { + // SSZ requires the fork from Eth-Consensus-Version; its absence is a + // relay protocol violation. + let fork = fork.ok_or_else(|| PbsError::RelayResponse { + error_msg: "relay did not provide consensus version header for ssz payload" + .to_string(), + code: code.as_u16(), + })?; + let data = + SignedExecutionPayloadBid::from_ssz_bytes(&response_bytes).map_err(|err| { + PbsError::SSZDecode { + err: format!("error decoding relay payload: {err:?}"), + fork, + } + })?; + GetExecutionPayloadBidResponse { version: fork, data, metadata: Default::default() } + } + }; + + debug!( + relay_id = relay.id.as_ref(), + header_size_bytes, + latency = ?request_latency, + version =? get_header_response.version, + trustless_bid_eth = format_gwei_as_eth(get_header_response.data.message.value), + execution_payment_eth = format_gwei_as_eth(get_header_response.data.message.execution_payment), + block_hash = %get_header_response.data.message.block_hash, + "received new header" + ); + + Ok(Some(get_header_response)) +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_request_budget_ms() { + let headers = |sent: u64, timeout: u64| { + let mut h = HeaderMap::new(); + h.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(sent)); + h.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout)); + h + }; + let now = 1_000_000; + const SLOT_MS: u64 = 12_000; + + // Transit delay eats the budget: the deadline is absolute + assert_eq!(request_budget_ms(&headers(now, 1000), now, SLOT_MS).unwrap(), 1000); + assert_eq!(request_budget_ms(&headers(now - 400, 1000), now, SLOT_MS).unwrap(), 600); + + // A deadline already in the past, or a zero timeout, leaves nothing + assert_eq!(request_budget_ms(&headers(now - 5000, 1000), now, SLOT_MS).unwrap(), 0); + assert_eq!(request_budget_ms(&headers(now, 0), now, SLOT_MS).unwrap(), 0); + + // A proposer clock running ahead cannot grant more than it advertised + assert_eq!(request_budget_ms(&headers(now + 10_000, 1000), now, SLOT_MS).unwrap(), 1000); + + // Nor more than one slot + assert_eq!(request_budget_ms(&headers(now, 60_000), now, SLOT_MS).unwrap(), SLOT_MS); + + // Both headers are required and must parse + for h in [ + HeaderMap::new(), + { + let mut h = HeaderMap::new(); + h.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(now)); + h + }, + { + let mut h = HeaderMap::new(); + h.insert(HEADER_TIMEOUT_MS, HeaderValue::from(1000u64)); + h + }, + { + let mut h = HeaderMap::new(); + h.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from_static("soon")); + h.insert(HEADER_TIMEOUT_MS, HeaderValue::from(1000u64)); + h + }, + ] { + assert!(matches!( + request_budget_ms(&h, now, SLOT_MS), + Err(PbsClientError::MissingTimingHeader) + )); + } + } +} diff --git a/crates/pbs/src/routes/mod.rs b/crates/pbs/src/routes/mod.rs index 84853d9ea..595552f58 100644 --- a/crates/pbs/src/routes/mod.rs +++ b/crates/pbs/src/routes/mod.rs @@ -1,12 +1,18 @@ +mod builder_preferences; +mod execution_payload_bid; mod get_header; mod register_validator; mod reload; mod router; mod status; mod submit_block; +mod submit_signed_beacon_block; +use builder_preferences::handle_submit_builder_preferences; +use execution_payload_bid::handle_get_execution_payload_bid; use get_header::handle_get_header; use register_validator::handle_register_validator; pub use router::create_app_router; use status::handle_get_status; use submit_block::handle_submit_block_v1; +use submit_signed_beacon_block::handle_submit_signed_beacon_block; diff --git a/crates/pbs/src/routes/router.rs b/crates/pbs/src/routes/router.rs index e98c89c14..54118d8cf 100644 --- a/crates/pbs/src/routes/router.rs +++ b/crates/pbs/src/routes/router.rs @@ -8,19 +8,22 @@ use axum::{ }; use axum_extra::headers::{ContentType, HeaderMapExt, UserAgent}; use cb_common::pbs::{ - BUILDER_V1_API_PATH, BUILDER_V2_API_PATH, GET_HEADER_PATH, GET_STATUS_PATH, - REGISTER_VALIDATOR_PATH, RELOAD_PATH, SUBMIT_BLOCK_PATH, + BUILDER_V1_API_PATH, BUILDER_V2_API_PATH, GET_EXECUTION_PAYLOAD_BID_PATH, GET_HEADER_PATH, + GET_STATUS_PATH, REGISTER_VALIDATOR_PATH, RELOAD_PATH, SUBMIT_BLOCK_PATH, + SUBMIT_BUILDER_PREFERENCES_PATH, SUBMIT_SIGNED_BEACON_BLOCK_PATH, }; use tower_http::trace::TraceLayer; use tracing::{info, trace, warn}; use uuid::Uuid; use super::{ - handle_get_header, handle_get_status, handle_register_validator, handle_submit_block_v1, - reload::handle_reload, + handle_get_execution_payload_bid, handle_get_header, handle_get_status, + handle_register_validator, handle_submit_block_v1, handle_submit_builder_preferences, + handle_submit_signed_beacon_block, reload::handle_reload, }; use crate::{ MAX_SIZE_REGISTER_VALIDATOR_REQUEST, MAX_SIZE_SUBMIT_BLOCK_RESPONSE, + MAX_SIZE_SUBMIT_SIGNED_BEACON_BLOCK, api::BuilderApi, routes::submit_block::handle_submit_block_v2, state::{BuilderApiState, PbsStateGuard}, @@ -42,7 +45,14 @@ pub fn create_app_router>(state: PbsStateGu SUBMIT_BLOCK_PATH, post(handle_submit_block_v1::) .route_layer(DefaultBodyLimit::max(MAX_SIZE_SUBMIT_BLOCK_RESPONSE)), - ); // header is smaller than the response but err on the safe side + ) // header is smaller than the response but err on the safe side + .route(GET_EXECUTION_PAYLOAD_BID_PATH, post(handle_get_execution_payload_bid::)) + .route(SUBMIT_BUILDER_PREFERENCES_PATH, post(handle_submit_builder_preferences::)) + .route( + SUBMIT_SIGNED_BEACON_BLOCK_PATH, + post(handle_submit_signed_beacon_block::) + .route_layer(DefaultBodyLimit::max(MAX_SIZE_SUBMIT_SIGNED_BEACON_BLOCK)), + ); let v2_builder_routes = Router::new().route( SUBMIT_BLOCK_PATH, post(handle_submit_block_v2::) @@ -70,7 +80,7 @@ pub fn create_app_router>(state: PbsStateGu } #[tracing::instrument( - name = "", + name = "", skip_all, fields( method = %req.extensions().get::().map(|m| m.as_str()).unwrap_or("unknown"), @@ -79,6 +89,7 @@ pub fn create_app_router>(state: PbsStateGu block_hash = tracing::field::Empty, block_number = tracing::field::Empty, parent_hash = tracing::field::Empty, + parent_root = tracing::field::Empty, validator = tracing::field::Empty, ), )] diff --git a/crates/pbs/src/routes/submit_signed_beacon_block.rs b/crates/pbs/src/routes/submit_signed_beacon_block.rs new file mode 100644 index 000000000..8f60f1db2 --- /dev/null +++ b/crates/pbs/src/routes/submit_signed_beacon_block.rs @@ -0,0 +1,97 @@ +use axum::{body::Bytes, extract::State, http::HeaderMap, response::IntoResponse}; +use cb_common::wire::{ + BodyDeserializeError, EncodingType, content_type_encoding, get_user_agent, + require_consensus_version_header, +}; +use reqwest::StatusCode; +use tracing::{debug, info}; + +use crate::{ + PbsStateGuard, + constants::SUBMIT_SIGNED_BEACON_BLOCK_ENDPOINT_TAG, + error::PbsClientError, + state::{BuilderApiState, PbsState}, + utils::{ + epbs_base_send_headers, join_detached_sends, post_ssz_expect_accepted, + record_beacon_status, record_request_failure, + }, +}; + +pub async fn handle_submit_signed_beacon_block( + State(state): State>, + req_headers: HeaderMap, + body: Bytes, +) -> Result { + let state = state.read().clone(); + + match submit_signed_beacon_block(body, req_headers, state).await { + Ok(()) => { + record_beacon_status("202", SUBMIT_SIGNED_BEACON_BLOCK_ENDPOINT_TAG); + Ok(StatusCode::ACCEPTED.into_response()) + } + Err(err) => Err(record_request_failure(err, SUBMIT_SIGNED_BEACON_BLOCK_ENDPOINT_TAG)), + } +} + +/// Implements https://ethereum.github.io/builder-specs/?urls.primaryName=dev#/Builder/submitSignedBeaconBlock +/// Forwards the block bytes, undecoded, to every configured builder, since CB +/// keeps no auction state to know which bid won. Returns 202 if one accepts +pub async fn submit_signed_beacon_block( + body: Bytes, + req_headers: HeaderMap, + state: PbsState, +) -> Result<(), PbsClientError> { + let ua = get_user_agent(&req_headers); + + require_consensus_version_header(&req_headers)?; + + if body.is_empty() { + return Err(BodyDeserializeError::MissingBody.into()); + } + // Forwarded undecoded as SSZ, so anything else, an unlabelled (JSON) body + // included, is a 415 + if content_type_encoding(&req_headers)? != EncodingType::Ssz { + return Err(BodyDeserializeError::UnsupportedMediaType.into()); + } + + info!(ua, "new request"); + + let send_headers = epbs_base_send_headers(&req_headers)?; + let timeout_ms = state.pbs_config().timeout_get_payload_ms; + let relays = state.all_relays(); + let results = join_detached_sends(relays.iter().map(|relay| { + let (relay, body, headers) = (relay.clone(), body.clone(), send_headers.clone()); + async move { + let url = relay.submit_signed_beacon_block_url()?; + post_ssz_expect_accepted( + &relay, + url, + body, + headers, + timeout_ms, + SUBMIT_SIGNED_BEACON_BLOCK_ENDPOINT_TAG, + ) + .await?; + Ok(()) + } + })) + .await; + let accepted = results + .into_iter() + .zip(relays.iter()) + .filter(|(res, relay)| match res { + Ok(()) => true, + Err(err) => { + debug!(relay_id = relay.id.as_ref(), %err, "builder did not accept the block; only the winner accepts"); + false + } + }) + .count(); + + // Only the winner accepts, so one 202 across the broadcast is success + if accepted == 0 { + return Err(PbsClientError::NoBuilderResponse); + } + info!(accepted, addressed = relays.len(), "signed beacon block submitted"); + Ok(()) +} diff --git a/crates/pbs/src/utils.rs b/crates/pbs/src/utils.rs index 782ae79b6..c07da9d40 100644 --- a/crates/pbs/src/utils.rs +++ b/crates/pbs/src/utils.rs @@ -1,3 +1,180 @@ +use std::{ + future::Future, + time::{Duration, Instant}, +}; + +use alloy::primitives::utils::{ParseUnits, Unit}; +use cb_common::{ + pbs::{ForkName, RelayClient, error::PbsError}, + types::BlsPublicKey, + wire::{ + CONSENSUS_VERSION_HEADER, EncodingType, get_user_agent_with_version, + read_chunked_body_with_max, + }, +}; +use futures::future::join_all; +use reqwest::{ + StatusCode, + header::{CONTENT_TYPE, HeaderMap, HeaderValue, USER_AGENT}, +}; +use tracing::{Instrument, debug, error, warn}; +use url::Url; + +use crate::{ + constants::{MAX_SIZE_DEFAULT, TIMEOUT_ERROR_CODE_STR}, + error::PbsClientError, + metrics::{BEACON_NODE_STATUS, RELAY_LATENCY, RELAY_STATUS_CODE}, +}; + +/// Sends one already-built relay request, recording the per-relay metrics +/// shared by all three ePBS endpoints, and returns the response and its latency +/// so the caller can read/decode the body itself. `tag` is the per-endpoint +/// metric label. Callers build their own `RequestBuilder` because the requests +/// legitimately differ (bid sets a per-call timeout and timing headers). +pub(crate) async fn send_to_relay( + req: reqwest::RequestBuilder, + relay: &RelayClient, + tag: &str, +) -> Result<(reqwest::Response, Duration), PbsError> { + let start_request = Instant::now(); + let res = match req.send().await { + Ok(res) => res, + Err(err) => { + RELAY_STATUS_CODE.with_label_values(&[TIMEOUT_ERROR_CODE_STR, tag, &relay.id]).inc(); + return Err(err.into()); + } + }; + + let request_latency = start_request.elapsed(); + RELAY_LATENCY.with_label_values(&[tag, &relay.id]).observe(request_latency.as_secs_f64()); + + let code = res.status(); + RELAY_STATUS_CODE.with_label_values(&[code.as_str(), tag, &relay.id]).inc(); + + Ok((res, request_latency)) +} + +/// Count a request-rejection in `BEACON_NODE_STATUS` before it short-circuits +/// the handler, so rejected requests show up as 4xx in the endpoint counter. +pub(crate) fn record_client_error( + err: impl Into, + endpoint: &str, +) -> PbsClientError { + let err = err.into(); + BEACON_NODE_STATUS.with_label_values(&[err.status_code().as_str(), endpoint]).inc(); + err +} + +pub(crate) fn record_beacon_status(code: &str, endpoint: &str) { + BEACON_NODE_STATUS.with_label_values(&[code, endpoint]).inc(); +} + +/// Logs and counts a failed ePBS request before it is returned to the beacon +/// node. A 4xx is the caller's fault, not CB's: only a 5xx is an error. +pub(crate) fn record_request_failure(err: PbsClientError, endpoint: &str) -> PbsClientError { + if err.status_code().is_server_error() { + error!(%err, "{endpoint} failed"); + } else { + warn!(%err, "{endpoint} failed"); + } + record_beacon_status(err.status_code().as_str(), endpoint); + err +} + +/// Fans `sends` out on detached tasks and waits for all of them +pub(crate) async fn join_detached_sends( + sends: impl IntoIterator, +) -> Vec> +where + F: Future> + Send + 'static, +{ + let handles: Vec<_> = + sends.into_iter().map(|send| tokio::spawn(send.in_current_span())).collect(); + join_all(handles) + .await + .into_iter() + .map(|joined| joined.unwrap_or_else(|err| Err(PbsError::TokioJoinError(err)))) + .collect() +} + +pub(crate) fn log_mux_selection( + maybe_mux_id: Option<&str>, + relay_count: usize, + pubkey: &BlsPublicKey, +) { + match maybe_mux_id { + Some(mux_id) => { + debug!(mux_id, relays = relay_count, pubkey = %pubkey, "using mux config") + } + None => debug!(relays = relay_count, pubkey = %pubkey, "using default config"), + } +} + +/// POSTs an SSZ body to a builder; 202 Accepted is the only success. A failed +/// response's body, capped at `MAX_SIZE_DEFAULT`, becomes the error message. +/// Returns the request latency. +pub(crate) async fn post_ssz_expect_accepted( + relay: &RelayClient, + url: Url, + body: impl Into, + headers: HeaderMap, + timeout_ms: u64, + tag: &str, +) -> Result { + let req = relay + .client + .post(url) + .timeout(Duration::from_millis(timeout_ms)) + .headers(headers) + .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .body(body); + let (res, latency) = send_to_relay(req, relay, tag).await?; + let code = res.status(); + if code == StatusCode::ACCEPTED { + return Ok(latency); + } + // Read after the status check, so a body over the cap still reports the + // builder's status + let url = res.url().to_string(); + let error_msg = match read_chunked_body_with_max(res, MAX_SIZE_DEFAULT, &url).await { + Ok(body) if !body.is_empty() => String::from_utf8_lossy(&body).into_owned(), + Ok(_) => "expected 202".to_string(), + Err(err) => err.to_string(), + }; + Err(PbsError::RelayResponse { error_msg, code: code.as_u16() }) +} + +/// A gwei amount in ETH, for logs +pub(crate) fn format_gwei_as_eth(gwei: u64) -> String { + ParseUnits::from(gwei).format_units(Unit::GWEI) +} + +/// The builder's own 400 or 401, handed back to the proposer; any other relay +/// failure is the caller's to map. +pub(crate) fn builder_rejection(err: &PbsError) -> Option { + match err.relay_status_code() { + Some(400) => Some(PbsClientError::BuilderRejected(StatusCode::BAD_REQUEST)), + Some(401) => Some(PbsClientError::BuilderRejected(StatusCode::UNAUTHORIZED)), + _ => None, + } +} + +/// Headers every ePBS relay request carries: the versioned `User-Agent`, and +/// `Eth-Consensus-Version` since each body is a fork-versioned type. +pub(crate) fn epbs_base_send_headers(req_headers: &HeaderMap) -> Result { + let mut headers = HeaderMap::new(); + headers.insert( + USER_AGENT, + get_user_agent_with_version(req_headers).map_err(|_| PbsClientError::Internal)?, + ); + headers.insert( + CONSENSUS_VERSION_HEADER, + HeaderValue::from_str(&ForkName::Gloas.to_string()) + .expect("fork name is always a valid header value"), + ); + Ok(headers) +} + const GAS_LIMIT_ADJUSTMENT_FACTOR: u64 = 1024; const GAS_LIMIT_MINIMUM: u64 = 5_000; @@ -19,3 +196,76 @@ pub fn check_gas_limit(gas_limit: u64, parent_gas_limit: u64) -> bool { true } + +/// The relay an ePBS request addresses: the first whose URL hostname equals +/// `auth_data`. No match is a 400. +pub(crate) fn resolve_addressed_relay( + relays: &[RelayClient], + auth_data: &[u8], +) -> Result { + let addressed = relays.iter().find(|relay| { + relay.config.entry.url.host_str().is_some_and(|host| host.as_bytes() == auth_data) + }); + match addressed { + Some(relay) => Ok(relay.clone()), + None => { + warn!( + auth_data = %String::from_utf8_lossy(auth_data), + "auth data matches no configured relay" + ); + Err(PbsClientError::AuthDataMismatch) + } + } +} + +#[cfg(test)] +mod tests { + use cb_common::{ + config::{GetHeaderTransport, RelayConfig}, + pbs::RelayEntry, + types::BlsSecretKey, + }; + + use super::*; + + fn test_relay(url: &str) -> RelayClient { + let entry = RelayEntry { + id: url.to_string(), + pubkey: BlsSecretKey::random().public_key(), + url: Url::parse(url).unwrap(), + }; + let config = RelayConfig { + entry, + id: None, + headers: None, + get_params: None, + get_header: GetHeaderTransport::Http, + enable_timing_games: false, + target_first_request_ms: None, + frequency_get_header_ms: None, + validator_registration_batch_size: None, + }; + RelayClient::new(config).unwrap() + } + + #[test] + fn resolve_relay_by_hostname() { + let relays = vec![ + test_relay("https://0xdeadbeef@builder-a.example.com"), + test_relay("https://builder-b.example.com:8443/eth"), + test_relay("http://[::1]:18550"), + ]; + let host = |data: &[u8]| -> Option { + resolve_addressed_relay(&relays, data) + .ok() + .map(|relay| relay.config.entry.url.host_str().unwrap().to_string()) + }; + assert_eq!(host(b"builder-a.example.com").as_deref(), Some("builder-a.example.com")); + assert_eq!(host(b"builder-b.example.com").as_deref(), Some("builder-b.example.com")); + assert_eq!(host(b"[::1]").as_deref(), Some("[::1]")); + assert!(host(b"Builder-A.example.com").is_none()); + assert!(host(b"builder-a.example.com:443").is_none()); + // a URL is not a hostname + assert!(host(b"https://builder-a.example.com").is_none()); + } +} diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md new file mode 100644 index 000000000..748db1b82 --- /dev/null +++ b/docs/docs/get_started/epbs.md @@ -0,0 +1,103 @@ +--- +description: Run Commit-Boost for ePBS proposals after the Gloas fork +--- + +# ePBS + +:::info Unreleased +ePBS support is not in a Commit-Boost release yet; it ships from v0.12.0-rc1. +::: + +From the Gloas fork, a proposer commits to a builder's signed execution payload bid instead of a blinded header, and the winning builder publishes the payload itself. Commit-Boost serves the builder-API endpoints the beacon node calls for this and forwards each call to builders, relays, or MPBC operators. + +## What changes at the fork + +Normal PBS paths are unaffected. From the Gloas fork, apart from the `status` endpoint, the beacon node calls three new endpoints on Commit-Boost: + +| Endpoint | What the beacon node sends | What Commit-Boost does | +|---|---|---| +| `POST /eth/v1/builder/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}` | A bid request for the proposal slot | Asks the [addressed](#routing-by-auth-data) builder for its bid | +| `POST /eth/v1/builder/builder_preferences/{proposer_pubkey}` | The proposer's `max_execution_payment` for one builder, ahead of the slot | Forwards it to the addressed builder | +| `POST /eth/v1/builder/beacon_blocks` | The signed beacon block that commits to the winning bid | Forwards it to every builder in `[[relays]]` and `[[mux.relays]]`; only the winning builder accepts it | + +With ePBS the beacon node validates each bid and weighs it against your builder config (`min_bid`, `builder_boost_factor`) and its local block, so Commit-Boost adds no redundant verification on the hot path. + +The ePBS bid endpoint does not use these PBS options, which will be deprecated after the hard fork: + +- `skip_sigverify`, `min_bid_eth` and `extra_validation_enabled`: the beacon node checks the bid against the on-chain builder registry and applies the `min_bid` from its builder config. +- `timeout_get_header_ms`, `late_in_slot_time_ms`, their `[[mux]]` overrides and the timing-games options: the beacon node's deadline bounds the request (see [Timing](#timing)). + +A `get_header = "stream"` relay is asked for ePBS bids over plain HTTP. + +## How a bid request reaches a builder + +From the fork, each validator key has a builder config: a list of entries, each a `url` and an `auth_data`. For every entry the beacon node sends a bid request to the entry's `url`, carrying its `auth_data` in a `SignedBuilderRequestAuth` that the validator signs. `auth_data` tells a builder that a request was meant for it, so a request signed for one builder is rejected by another. Unless the proposer and builder agree on another value, it is the builder's hostname. + +To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and its `auth_data` is the hostname of the builder it stands for. Commit-Boost reads the `auth_data` of each request, finds the relay entry with that hostname and forwards the request there. Builder preferences are routed the same way. The signed block goes to every builder, since only the one whose bid won accepts it. + +## Setup + +### 1. Add the builders to Commit-Boost + +List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be under one slot, `12000` on mainnet. + +### 2. Point each validator key at Commit-Boost + +Write each key's builder config through its validator client's keymanager API, which must implement the builder config endpoint ([keymanager-APIs #88](https://github.com/ethereum/keymanager-APIs/pull/88)). Add one entry per builder: `url` is Commit-Boost's URL, and `auth_data` is the hex of the builder's hostname. A key without builder config sends the default auth data of Commit-Boost's own URL, which matches no builder, so every bid request gets `400`. + +With the relay entry `url = "https://0xa1ce...@builder-a.example.com"`, Commit-Boost listening at `http://cb.example.com:18550`, the keymanager API at `$KEYMANAGER_URL`, its token in `$TOKEN` and the validator key in `$PUBKEY`: + +```bash +curl -X POST "$KEYMANAGER_URL/eth/v1/validator/$PUBKEY/builder_config" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "builders": [ + { + "url": "http://cb.example.com:18550", + "auth_data": "0x6275696c6465722d612e6578616d706c652e636f6d", + "builder_pubkeys": [] + } + ] + }' +``` + +`echo 0x$(printf builder-a.example.com | xxd -p | tr -d '\n')` prints the `auth_data` hex. + +The call is `POST /eth/v1/validator/{pubkey}/builder_config`, authenticated with the keymanager API's bearer token. The body replaces the key's config in full, and the validator client answers `202` once it is stored. Besides `url` and `auth_data`, an entry or the top level can set: + +- `min_bid` and `builder_boost_factor` are optional. At the top level they apply to every entry, and to p2p bids, unless an entry sets its own. A top-level `min_bid` also floors the bids that come through Commit-Boost, so a value above your builders' bids sends every proposal to p2p bids or a local block. +- `max_execution_payment`, when an entry omits it, gets the validator client's default, which differs between clients (Lodestar stores `0`, Nimbus the maximum). Lodestar accepts a nonzero value only when started with `--allowDangerousTrustedPayments`. +- `builder_pubkeys` limits which builder keys' bids the beacon node accepts; leave it empty to accept any. Don't copy the pubkey from the relay URL: that is the relay's key, not the builder's bid-signing key. + +## Routing by auth data + +Commit-Boost sends a bid or preferences request to the first builder serving the proposer's key (the relays of its `[[mux]]`, otherwise `[[relays]]`) whose URL hostname equals the request's `auth_data`, byte for byte. That is the [builder specs](https://github.com/ethereum/builder-specs/blob/main/specs/gloas/validator.md#default-auth-data) default auth data: the lowercase hostname, without scheme, port or path, with an IPv6 address in brackets, such as `[::1]`. Builders on one host share it, so only the first is asked. Commit-Boost routes by hostname only, so a value agreed with a builder works through it only if it is that hostname. When no builder matches, the request gets `400`. + +## Timing + +The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts your `proposer_deadline_buffer_ms` from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. + +Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon node's remaining time: enough for the bid to get back from Commit-Boost to the beacon node and for the beacon node to process it. If no time is left, Commit-Boost does not ask the builder. + +Builder preferences use `timeout_register_validator_ms`, and the signed block `timeout_get_payload_ms`. + +## Metrics + +With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoint` labels `get_execution_payload_bid`, `submit_builder_preferences` and `submit_signed_beacon_block`. + +| Question | Series | +|---|---| +| What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences and the signed block, `4xx` for a rejected request, `500` when no builder accepted preferences or the signed block | +| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure) | +| How fast are builders? | `cb_pbs_relay_latency`, by `relay_id` | +| What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from each bid a builder serves | + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data matches no builder serving the key. Usually the key has no builder config, so its auth data is Commit-Boost's own hostname | Write the key's [builder config](#validator-builder-config), or add the builder it names as a relay entry for the key | +| A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than its hostname | Have the builder accept its hostname, the only auth data Commit-Boost [routes by](#routing-by-auth-data) | +| The signed block gets `415` | The beacon node sends it as JSON | Configure the beacon node to send SSZ | +| Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/sidebars.js b/docs/sidebars.js index 3fcef6cfc..e0c4f9741 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -46,6 +46,7 @@ const sidebars = { 'get_started/running/metrics-catalog', ], }, + 'get_started/epbs', 'get_started/troubleshooting', ], }, diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 646f4994b..c8503378c 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -20,6 +20,7 @@ rcgen.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true +ssz_types.workspace = true tempfile.workspace = true tokio.workspace = true tokio-tungstenite.workspace = true diff --git a/tests/src/mock_relay.rs b/tests/src/mock_relay.rs index 336c91f1a..a05dcbeab 100644 --- a/tests/src/mock_relay.rs +++ b/tests/src/mock_relay.rs @@ -23,12 +23,17 @@ use axum::{ use cb_common::{ pbs::{ BUILDER_V1_API_PATH, BUILDER_V2_API_PATH, BlobsBundle, BuilderBid, BuilderBidFulu, - ExecutionPayloadElectra, ExecutionPayloadHeaderFulu, ForkName, GET_HEADER_PATH, - GET_STATUS_PATH, GetHeaderParams, GetHeaderResponse, GetPayloadInfo, PayloadAndBlobs, - REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH, SignedBuilderBid, SubmitBlindedBlockResponse, + BuilderPreferencesRequest, ExecutionPayloadBid, ExecutionPayloadElectra, + ExecutionPayloadHeaderFulu, ForkName, ForkVersionDecode, GET_EXECUTION_PAYLOAD_BID_PATH, + GET_HEADER_PATH, GET_STATUS_PATH, GetExecutionPayloadBidResponse, GetHeaderParams, + GetHeaderResponse, GetPayloadInfo, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, + PayloadAndBlobs, REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH, + SUBMIT_BUILDER_PREFERENCES_PATH, SUBMIT_SIGNED_BEACON_BLOCK_PATH, SignedBeaconBlock, + SignedBuilderBid, SignedBuilderRequestAuth, SignedExecutionPayloadBid, + SubmitBlindedBlockResponse, }, signature::sign_builder_root, - types::{BlsSecretKey, Chain}, + types::{BlsPublicKey, BlsSecretKey, BlsSignature, Chain}, utils::{TestRandomSeed, timestamp_of_slot_start_sec}, wire::{ CONSENSUS_VERSION_HEADER, EncodingType, deserialize_body, get_accept_types, @@ -39,9 +44,9 @@ use cb_pbs::{ GET_HEADER_ENDPOINT_TAG, MAX_SIZE_SUBMIT_BLOCK_RESPONSE, REGISTER_VALIDATOR_ENDPOINT_TAG, STATUS_ENDPOINT_TAG, SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, }; -use lh_types::KzgProof; +use lh_types::{KzgProof, Slot}; use reqwest::header::{ACCEPT, CONTENT_TYPE}; -use ssz::Encode; +use ssz::{Decode, Encode}; use tokio::net::TcpListener; use tracing::{debug, error}; use tree_hash::TreeHash; @@ -92,6 +97,24 @@ pub struct MockRelayState { received_get_status: Arc, received_register_validator: Arc, received_submit_block: Arc, + received_execution_payload_bid: Arc, + received_builder_preferences: Arc, + received_signed_beacon_block: Arc, + /// `slot` of the last signed beacon block forwarded, decoded from the SSZ + /// body PBS sent, so a test can assert the block survived the hop + received_block_slot: RwLock>, + /// The last `BuilderPreferencesRequest` submitted, so a test can assert + /// both the preferences and the auth were forwarded unchanged + received_preferences: RwLock>, + /// The `{proposer_pubkey}` path segment of the last preferences submission + received_preferences_pubkey: RwLock>, + /// Hold every bid request this long before answering, simulating a builder + /// that sits on a request instead of answering promptly + bid_delay_ms: Option, + /// The last `SignedBuilderRequestAuth` forwarded on a bid request + received_auth: RwLock>, + /// The `X-Timeout-Ms` of the last bid request + received_bid_timeout_ms: RwLock>, response_override: RwLock>, bid_value: RwLock, /// The raw `Accept` header PBS sent on the most recent get_header request, @@ -100,6 +123,10 @@ pub struct MockRelayState { /// Api key header seen per endpoint tag, so a test can assert the relay's /// configured key rides on every request PBS sends it. api_keys_seen: RwLock>, + /// Served as the bid's `value` + trustless_bid_gwei: u64, // default 10 + /// When true, an SSZ bid is served without `Eth-Consensus-Version` + epbs_omit_consensus_version: bool, } impl MockRelayState { @@ -118,6 +145,45 @@ impl MockRelayState { pub fn received_submit_block(&self) -> u64 { self.received_submit_block.load(Ordering::Relaxed) } + pub fn received_execution_payload_bid(&self) -> u64 { + self.received_execution_payload_bid.load(Ordering::Relaxed) + } + pub fn received_builder_preferences(&self) -> u64 { + self.received_builder_preferences.load(Ordering::Relaxed) + } + pub fn received_signed_beacon_block(&self) -> u64 { + self.received_signed_beacon_block.load(Ordering::Relaxed) + } + /// `slot` of the last signed beacon block PBS forwarded + pub fn received_block_slot(&self) -> Option { + *self.received_block_slot.read().unwrap() + } + + /// `max_execution_payment` of the last submitted preferences + pub fn received_max_execution_payment(&self) -> Option { + self.received_preferences + .read() + .unwrap() + .as_ref() + .map(|r| r.preferences.max_execution_payment) + } + + /// The `SignedBuilderRequestAuth` carried by the last submitted preferences + pub fn received_preferences_auth(&self) -> Option { + self.received_preferences.read().unwrap().as_ref().map(|r| r.auth.clone()) + } + + /// The proposer the last preferences submission was filed under + pub fn received_preferences_pubkey(&self) -> Option { + self.received_preferences_pubkey.read().unwrap().clone() + } + + pub fn received_bid_timeout_ms(&self) -> Option { + *self.received_bid_timeout_ms.read().unwrap() + } + pub fn received_auth_data(&self) -> Option> { + self.received_auth.read().unwrap().as_ref().map(|a| a.message.data.to_vec()) + } pub fn large_body(&self) -> bool { self.large_body } @@ -165,10 +231,21 @@ impl MockRelayState { received_get_status: Default::default(), received_register_validator: Default::default(), received_submit_block: Default::default(), + received_execution_payload_bid: Default::default(), + received_builder_preferences: Default::default(), + received_signed_beacon_block: Default::default(), + received_block_slot: RwLock::new(None), + received_preferences: RwLock::new(None), + received_preferences_pubkey: RwLock::new(None), + bid_delay_ms: None, + received_auth: RwLock::new(None), + received_bid_timeout_ms: RwLock::new(None), response_override: RwLock::new(None), bid_value: RwLock::new(U256::from(10)), received_get_header_accept: RwLock::new(None), api_keys_seen: RwLock::new(HashMap::new()), + trustless_bid_gwei: 10, + epbs_omit_consensus_version: false, supported_content_types: Arc::new( [EncodingType::Json, EncodingType::Ssz].iter().cloned().collect(), ), @@ -217,6 +294,35 @@ impl MockRelayState { pub fn with_submit_block_version(self, fork: ForkName) -> Self { Self { submit_block_version_override: Some(fork), ..self } } + + /// Restrict this relay to SSZ responses on the bid endpoint, so the bid + /// 200 is served as SSZ regardless of the caller's fallback preference. + pub fn with_ssz_only_response(self) -> Self { + Self { + supported_content_types: Arc::new([EncodingType::Ssz].into_iter().collect()), + ..self + } + } + + /// Restrict this relay to JSON responses on the bid endpoint. + pub fn with_json_only_response(self) -> Self { + Self { + supported_content_types: Arc::new([EncodingType::Json].into_iter().collect()), + ..self + } + } + + /// Hold every bid request `delay_ms` before answering, so a caller with a + /// shorter `X-Timeout-Ms` times out. + pub fn with_bid_delay_ms(self, delay_ms: u64) -> Self { + Self { bid_delay_ms: Some(delay_ms), ..self } + } + + /// Serve an SSZ bid 200 WITHOUT the `Eth-Consensus-Version` header, to + /// exercise the PBS missing-fork error path on the outbound SSZ decode. + pub fn with_epbs_omit_consensus_version(self) -> Self { + Self { epbs_omit_consensus_version: true, ..self } + } } pub fn mock_relay_app_router(state: Arc) -> Router { @@ -224,7 +330,11 @@ pub fn mock_relay_app_router(state: Arc) -> Router { .route(GET_HEADER_PATH, get(handle_get_header)) .route(GET_STATUS_PATH, get(handle_get_status)) .route(REGISTER_VALIDATOR_PATH, post(handle_register_validator)) - .route(SUBMIT_BLOCK_PATH, post(handle_submit_block_v1)); + .route(SUBMIT_BLOCK_PATH, post(handle_submit_block_v1)) + // ePBS endpoints are v1 of new resources per builder-specs + .route(GET_EXECUTION_PAYLOAD_BID_PATH, post(handle_get_execution_payload_bid)) + .route(SUBMIT_BUILDER_PREFERENCES_PATH, post(handle_submit_builder_preferences)) + .route(SUBMIT_SIGNED_BEACON_BLOCK_PATH, post(handle_submit_signed_beacon_block)); let v2_builder_routes = if state.supports_submit_block_v2 { Router::new().route(SUBMIT_BLOCK_PATH, post(handle_submit_block_v2)) @@ -264,6 +374,129 @@ pub fn mock_signed_builder_bid( SignedBuilderBid { message, signature } } +async fn handle_get_execution_payload_bid( + State(state): State>, + Path((slot, parent_hash, parent_root, _pubkey)): Path<(u64, B256, B256, BlsPublicKey)>, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Response { + state.received_execution_payload_bid.fetch_add(1, Ordering::Relaxed); + // Builders MUST 400 a bid request without both timing headers + let header_u64 = + |name| headers.get(name).and_then(|v| v.to_str().ok()).and_then(|v| v.parse::().ok()); + let (Some(_), Some(timeout_ms)) = + (header_u64(HEADER_START_TIME_UNIX_MS), header_u64(HEADER_TIMEOUT_MS)) + else { + return (StatusCode::BAD_REQUEST, "missing Date-Milliseconds or X-Timeout-Ms header") + .into_response(); + }; + *state.received_bid_timeout_ms.write().unwrap() = Some(timeout_ms); + + // Decode the request auth the way a real builder does: Content-Type + // selects JSON vs SSZ. The wire type is fork-versioned per builder-specs, + // so the SSZ form additionally requires Eth-Consensus-Version; PBS always + // forwards SSZ, making this the assertion that the header arrives as gloas. + if !body.is_empty() { + let auth = match get_content_type(&headers) { + EncodingType::Ssz => { + if get_consensus_version_header(&headers) != Some(ForkName::Gloas) { + return ( + StatusCode::BAD_REQUEST, + "missing Eth-Consensus-Version header".to_string(), + ) + .into_response(); + } + SignedBuilderRequestAuth::from_ssz_bytes(&body).ok() + } + EncodingType::Json => serde_json::from_slice::(&body).ok(), + }; + if let Some(auth) = auth { + *state.received_auth.write().unwrap() = Some(auth); + } + } + + // Honor a forced status like the other handlers, so a test can make a relay + // fail on the bid endpoint (its bid is then dropped by PBS). The request was + // already counted above. + if let Some(status) = *state.response_override.read().unwrap() { + return status.into_response(); + } + + // Sleep, never block: concurrent polls must overlap, not serialize + if let Some(delay_ms) = state.bid_delay_ms { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + + let mut block_hash = B256::ZERO; + block_hash.0[0] = 1; + + let message = ExecutionPayloadBid { + parent_block_hash: parent_hash.into(), + parent_block_root: parent_root, + block_hash: block_hash.into(), + gas_limit: 30_000_000, + builder_index: 42, + slot: Slot::new(slot), + value: state.trustless_bid_gwei, + ..Default::default() + }; + + // CB does not verify bid signatures (the beacon node does), so the mock + // serves an unsigned bid + let data = SignedExecutionPayloadBid { message, signature: BlsSignature::empty() }; + + // Negotiate the RESPONSE encoding from the forwarded Accept, mirroring + // handle_get_header: honor supported_content_types + the caller's Accept. + let accept_types = match get_accept_types(&headers) { + Ok(a) => a, + Err(e) => { + return (StatusCode::BAD_REQUEST, format!("error parsing accept header: {e}")) + .into_response(); + } + }; + let content_type = if state.supported_content_types.contains(&EncodingType::Ssz) && + accept_types.contains(EncodingType::Ssz) + { + EncodingType::Ssz + } else if state.supported_content_types.contains(&EncodingType::Json) && + accept_types.contains(EncodingType::Json) + { + EncodingType::Json + } else { + return (StatusCode::NOT_ACCEPTABLE, "No acceptable content type found".to_string()) + .into_response(); + }; + + let response_body = match content_type { + // SSZ carries the inner bid; the fork travels in Eth-Consensus-Version. + EncodingType::Ssz => data.as_ssz_bytes(), + // JSON carries the fork-versioned wrapper (fork is in the body). + EncodingType::Json => { + let versioned = GetExecutionPayloadBidResponse { + version: ForkName::Gloas, + data, + metadata: Default::default(), + }; + serde_json::to_vec(&versioned).unwrap() + } + }; + + let mut response = (StatusCode::OK, response_body).into_response(); + // A real builder tags the 200 with the fork so a client can decode the + // (non-self-describing) SSZ bytes. The omit knob drives the PBS + // "SSZ response missing Eth-Consensus-Version" error path. + if !state.epbs_omit_consensus_version { + response.headers_mut().insert( + CONSENSUS_VERSION_HEADER, + HeaderValue::from_str(&ForkName::Gloas.to_string()).unwrap(), + ); + } + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_str(&content_type.to_string()).unwrap()); + response +} + async fn handle_get_header( State(state): State>, Path(GetHeaderParams { parent_hash, slot, .. }): Path, @@ -350,6 +583,67 @@ async fn handle_get_status( StatusCode::OK } +/// Decodes the submission the way a real builder does (Content-Type selects +/// JSON vs SSZ), records it, and 202s unless the test overrode the response. +async fn handle_submit_builder_preferences( + Path(proposer_pubkey): Path, + headers: HeaderMap, + State(state): State>, + body: axum::body::Bytes, +) -> Response { + state.received_builder_preferences.fetch_add(1, Ordering::Relaxed); + // A real builder keys preferences by proposer, so the path segment PBS sent + // is part of what a test must be able to assert + *state.received_preferences_pubkey.write().unwrap() = Some(proposer_pubkey); + + let decoded = match get_content_type(&headers) { + EncodingType::Json => serde_json::from_slice::(&body).ok(), + // The wire type is fork-versioned per builder-specs, so a real builder + // requires Eth-Consensus-Version on the SSZ form; PBS always forwards + // SSZ, making this the assertion that the header arrives as gloas. + EncodingType::Ssz => get_consensus_version_header(&headers) + .filter(|fork| *fork == ForkName::Gloas) + .and_then(|_| BuilderPreferencesRequest::from_ssz_bytes(&body).ok()), + }; + let Some(request) = decoded else { + return StatusCode::BAD_REQUEST.into_response(); + }; + *state.received_preferences.write().unwrap() = Some(request); + + if let Some(status) = state.response_override.read().unwrap().as_ref() { + // An error body over PBS's 1 KiB read cap + if state.large_body() { + return (*status, "x".repeat(2048)).into_response(); + } + return (*status).into_response(); + } + + StatusCode::ACCEPTED.into_response() +} + +/// Decodes the forwarded block (PBS always sends SSZ with +/// `Eth-Consensus-Version`), records its slot, and 202s +/// unless the test overrode the response. +async fn handle_submit_signed_beacon_block( + headers: HeaderMap, + State(state): State>, + body: axum::body::Bytes, +) -> Response { + state.received_signed_beacon_block.fetch_add(1, Ordering::Relaxed); + + if let Some(fork) = get_consensus_version_header(&headers) && + let Ok(block) = SignedBeaconBlock::from_ssz_bytes_by_fork(&body, fork) + { + *state.received_block_slot.write().unwrap() = Some(block.slot().as_u64()); + } + + if let Some(status) = state.response_override.read().unwrap().as_ref() { + return (*status).into_response(); + } + + StatusCode::ACCEPTED.into_response() +} + async fn handle_register_validator( State(state): State>, headers: HeaderMap, diff --git a/tests/src/mock_validator.rs b/tests/src/mock_validator.rs index b2da4a9c1..2059a2456 100644 --- a/tests/src/mock_validator.rs +++ b/tests/src/mock_validator.rs @@ -1,8 +1,11 @@ use alloy::{primitives::B256, rpc::types::beacon::relay::ValidatorRegistration}; use cb_common::{ - pbs::{BuilderApiVersion, RelayClient, SignedBlindedBeaconBlock}, + pbs::{ + BuilderApiVersion, BuilderPreferencesRequest, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, + RelayClient, SignedBeaconBlock, SignedBlindedBeaconBlock, SignedBuilderRequestAuth, + }, types::{BlsPublicKey, KnownChain}, - utils::bls_pubkey_from_hex, + utils::{bls_pubkey_from_hex, utcnow_ms}, wire::{CONSENSUS_VERSION_HEADER, EncodingType}, }; use lh_types::ForkName; @@ -12,7 +15,11 @@ use reqwest::{ }; use ssz::Encode; -use crate::utils::generate_mock_relay; +use crate::utils::{TEST_PROPOSER_PUBKEY, generate_mock_relay}; + +/// Timeout a test beacon node advertises on bid requests; long enough that the +/// deadline never bites in tests. +const DEFAULT_TEST_TIMEOUT_MS: u64 = 60_000; pub struct MockValidator { pub comm_boost: RelayClient, @@ -63,6 +70,122 @@ impl MockValidator { Ok(res) } + /// Submits builder preferences for `pubkey`, encoding the body as + /// `content_type` so a test can exercise both wire formats. The spec makes + /// `Eth-Consensus-Version` required, so a compliant BN always sets it to + /// Gloas; header-less negative tests build their request by hand. + pub async fn do_submit_builder_preferences( + &self, + pubkey: Option, + request: &BuilderPreferencesRequest, + content_type: EncodingType, + ) -> eyre::Result { + let default_pubkey = bls_pubkey_from_hex(TEST_PROPOSER_PUBKEY)?; + let url = + self.comm_boost.submit_builder_preferences_url(&pubkey.unwrap_or(default_pubkey))?; + + let body = match content_type { + EncodingType::Json => serde_json::to_vec(request)?, + EncodingType::Ssz => request.as_ssz_bytes(), + }; + let res = self + .comm_boost + .client + .post(url) + .header(CONTENT_TYPE, content_type.content_type_header().clone()) + .header(CONSENSUS_VERSION_HEADER, ForkName::Gloas.to_string()) + .body(body) + .send() + .await?; + Ok(res) + } + + /// Submits a `SignedBeaconBlock`, encoding the body as `content_type`. The + /// spec requires `Eth-Consensus-Version` on every submission, so it is + /// always set to Gloas (the only fork this endpoint serves). + pub async fn do_submit_signed_beacon_block( + &self, + block: &SignedBeaconBlock, + content_type: EncodingType, + ) -> eyre::Result { + let url = self.comm_boost.submit_signed_beacon_block_url()?; + let body = match content_type { + EncodingType::Json => serde_json::to_vec(block)?, + EncodingType::Ssz => block.as_ssz_bytes(), + }; + let res = self + .comm_boost + .client + .post(url) + .header(CONTENT_TYPE, content_type.content_type_header().clone()) + .header(CONSENSUS_VERSION_HEADER, ForkName::Gloas.to_string()) + .body(body) + .send() + .await?; + Ok(res) + } + + #[allow(clippy::too_many_arguments)] + pub async fn do_get_execution_payload_bid( + &self, + slot: u64, + parent_hash: B256, + parent_root: B256, + pubkey: Option, + auth: Option<&SignedBuilderRequestAuth>, + accept: Vec, + ) -> eyre::Result { + self.do_get_execution_payload_bid_with_timeout( + slot, + parent_hash, + parent_root, + pubkey, + auth, + accept, + DEFAULT_TEST_TIMEOUT_MS, + ) + .await + } + + /// Same, but with the proposer's `X-Timeout-Ms` under the test's control + #[allow(clippy::too_many_arguments)] + pub async fn do_get_execution_payload_bid_with_timeout( + &self, + slot: u64, + parent_hash: B256, + parent_root: B256, + pubkey: Option, + auth: Option<&SignedBuilderRequestAuth>, + accept: Vec, + timeout_ms: u64, + ) -> eyre::Result { + let default_pubkey = bls_pubkey_from_hex(TEST_PROPOSER_PUBKEY)?; + let url = self.comm_boost.get_execution_payload_bid_url( + slot, + &parent_hash, + &parent_root, + &pubkey.unwrap_or(default_pubkey), + )?; + // The spec requires both timing headers and `Eth-Consensus-Version` on + // every bid request; header-less negative tests build theirs by hand + let mut req = self + .comm_boost + .client + .post(url) + .header(HEADER_START_TIME_UNIX_MS, utcnow_ms()) + .header(HEADER_TIMEOUT_MS, timeout_ms) + .header(CONSENSUS_VERSION_HEADER, ForkName::Gloas.to_string()); + if !accept.is_empty() { + let accept_header = accept.iter().map(|e| e.to_string()).collect::>().join(", "); + req = req.header(ACCEPT, accept_header); + } + let req = match auth { + Some(auth) => req.json(auth), + None => req, + }; + Ok(req.send().await?) + } + pub async fn do_get_status(&self) -> eyre::Result { let url = self.comm_boost.get_status_url()?; Ok(self.comm_boost.client.get(url).send().await?) diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 11d192d52..d3c924166 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -15,23 +15,32 @@ use cb_common::{ SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_PORT_DEFAULT, SignerConfig, SignerType, StartSignerConfig, StaticModuleConfig, StaticPbsConfig, TlsMode, }, - pbs::{RelayClient, RelayEntry}, - signer::SignerLoader, - types::{BlsPublicKey, Chain, ModuleId}, + pbs::{BuilderRequestAuth, RelayClient, RelayEntry, SignedBuilderRequestAuth}, + signer::{SignerLoader, random_secret}, + types::{BlsPublicKey, BlsSignature, Chain, ModuleId}, utils::{bls_pubkey_from_hex, default_host}, }; +use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use eyre::Result; +use lh_types::Slot; use rcgen::generate_simple_self_signed; use reqwest::StatusCode; use url::Url; -use crate::mock_validator::MockValidator; +use crate::{ + mock_relay::{MockRelayState, start_mock_relay_service_with_listener}, + mock_validator::MockValidator, +}; pub const HEADER_API_KEY: &str = "x-api-key"; pub const API_KEY: &str = "123e4567-e89b-12d3-a456-426614174000"; /// Distinct from [`API_KEY`], which `MockValidator` also sends to PBS: a relay /// that sees this one can only have got it from its own config. pub const RELAY_API_KEY: &str = "f81d4fae-7dec-11d0-a765-00a0c91e6bf6"; +/// The mock relays' hostname, so their default auth data +pub const TEST_AUTH_DATA: &[u8] = b"0.0.0.0"; +/// The proposer pubkey the mock validator's ePBS requests are filed under. +pub const TEST_PROPOSER_PUBKEY: &str = "0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae"; pub fn get_local_address(port: u16) -> String { format!("http://0.0.0.0:{port}") @@ -129,6 +138,7 @@ pub fn get_pbs_config(port: u16) -> PbsConfig { skip_sigverify: false, min_bid_wei: U256::ZERO, late_in_slot_time_ms: u64::MAX, + proposer_deadline_buffer_ms: 0, extra_validation_enabled: false, ssv_node_api_url: Url::parse("http://localhost:0").unwrap(), @@ -241,6 +251,99 @@ pub fn bls_pubkey_from_hex_unchecked(hex: &str) -> BlsPublicKey { bls_pubkey_from_hex(hex).unwrap() } +/// Build a `SignedBuilderRequestAuth` carrying opaque `data`. CB forwards it +/// unmodified and leaves the signature to the builder, so an empty one +/// suffices. +pub fn opaque_auth(data: &[u8], slot: u64) -> SignedBuilderRequestAuth { + SignedBuilderRequestAuth { + message: BuilderRequestAuth { + data: ssz_types::VariableList::new(data.to_vec()) + .expect("data fits in MaxBuilderAuthData"), + slot: Slot::new(slot), + }, + signature: BlsSignature::empty(), + } +} + +/// Starts a mock relay on a free port, returning its state and port: the +/// building block of every PBS boot below. +pub async fn spawn_mock_relay(state: MockRelayState) -> Result<(Arc, u16)> { + let listener = get_free_listener().await; + let port = listener.local_addr()?.port(); + let state = Arc::new(state); + tokio::spawn(start_mock_relay_service_with_listener(state.clone(), listener)); + Ok((state, port)) +} + +/// Boot PBS in front of already-spawned mock relays, letting the test shape +/// the PBS config first. Readiness is awaited: relay_check makes a 200 on +/// /status mean the whole chain is up. +pub async fn setup_pbs( + chain: Chain, + relays: Vec, + tweak: impl FnOnce(&mut PbsConfig), +) -> Result { + setup_test_env(); + let pbs_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + + let mut pbs_config = get_pbs_config(pbs_port); + tweak(&mut pbs_config); + let state = PbsState::new(to_pbs_config(chain, pbs_config, relays), PathBuf::new()); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); + + let mock_validator = MockValidator::new(pbs_port)?; + wait_for_ready(&mock_validator).await?; + Ok(mock_validator) +} + +/// Boot PBS in front of one default-state mock relay, letting the test shape +/// the PBS config and the relay entry. +pub async fn setup_relay( + chain: Chain, + tweak: impl FnOnce(&mut PbsConfig), + make_relay: impl FnOnce(u16, BlsPublicKey) -> Result, +) -> Result<(MockValidator, Arc)> { + let (state, port) = spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let relay = make_relay(port, state.signer.public_key())?; + Ok((setup_pbs(chain, vec![relay], tweak).await?, state)) +} + +/// Boot PBS in front of several default-entry mock relays, one per state, so +/// per-relay knobs and counters stay independent. Returns the relay states in +/// configuration order. +pub async fn setup_relays( + chain: Chain, + states: Vec, +) -> Result<(MockValidator, Vec>)> { + let mut relays = Vec::with_capacity(states.len()); + let mut arc_states = Vec::with_capacity(states.len()); + for state in states { + let (state, port) = spawn_mock_relay(state).await?; + relays.push(generate_mock_relay(port, state.signer.public_key())?); + arc_states.push(state); + } + Ok((setup_pbs(chain, relays, |_| {}).await?, arc_states)) +} + +/// Starts one mock relay per state, each with a URL naming its host, so tests +/// can address one builder among several by its hostname. +pub async fn setup_relays_on_hosts( + chain: Chain, + states: Vec<(MockRelayState, &str)>, +) -> Result<(MockValidator, Vec>)> { + let mut relays = Vec::with_capacity(states.len()); + let mut arc_states = Vec::with_capacity(states.len()); + for (state, host) in states { + let (state, port) = spawn_mock_relay(state).await?; + let mut config = mock_relay_config(port, state.signer.public_key())?; + config.entry.url = format!("http://{host}:{port}").parse()?; + relays.push(RelayClient::new(config)?); + arc_states.push(state); + } + Ok((setup_pbs(chain, relays, |_| {}).await?, arc_states)) +} + /// Poll /status until PBS and its relays are up. relay_check makes a 200 mean /// the whole chain is ready; the fixed 100ms sleep used elsewhere flakes under /// parallel suite load. diff --git a/tests/tests/pbs_cfg_file_update.rs b/tests/tests/pbs_cfg_file_update.rs index ec954f531..23d59e94e 100644 --- a/tests/tests/pbs_cfg_file_update.rs +++ b/tests/tests/pbs_cfg_file_update.rs @@ -66,6 +66,7 @@ async fn test_cfg_file_update() -> Result<()> { min_bid_wei: U256::ZERO, late_in_slot_time_ms: u64::MAX / 2, /* serde gets very upset about serializing u64::MAX * or anything close to it */ + proposer_deadline_buffer_ms: 0, extra_validation_enabled: false, rpc_url: None, ssv_node_api_url: Url::parse("http://example.com").unwrap(), diff --git a/tests/tests/pbs_get_execution_payload_bid.rs b/tests/tests/pbs_get_execution_payload_bid.rs new file mode 100644 index 000000000..b31aabbe1 --- /dev/null +++ b/tests/tests/pbs_get_execution_payload_bid.rs @@ -0,0 +1,479 @@ +use std::{collections::HashMap, path::PathBuf}; + +use alloy::primitives::{B256, U256}; +use cb_common::{ + config::RuntimeMuxConfig, + pbs::{ + GetExecutionPayloadBidInfo, GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, + HEADER_TIMEOUT_MS, SignedBuilderRequestAuth, SignedExecutionPayloadBid, + }, + signer::random_secret, + types::Chain, + utils::utcnow_ms, + wire::{CONSENSUS_VERSION_HEADER, EncodingType}, +}; +use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; +use cb_tests::{ + mock_relay::MockRelayState, + mock_validator::MockValidator, + utils::{ + TEST_AUTH_DATA, TEST_PROPOSER_PUBKEY, generate_mock_relay, get_free_listener, + get_pbs_config, opaque_auth, setup_relay, setup_relays, setup_relays_on_hosts, + spawn_mock_relay, to_pbs_config, wait_for_ready, + }, +}; +use eyre::Result; +use reqwest::{StatusCode, header::CONTENT_TYPE}; +use ssz::{Decode, Encode}; +use tracing::info; + +const TEST_SLOT: u64 = 100; + +/// The request most tests send: a JSON-accept bid request for `TEST_SLOT` on +/// a zero parent hash/root, carrying `auth`. +async fn get_json_bid( + mock_validator: &MockValidator, + auth: &SignedBuilderRequestAuth, +) -> Result { + mock_validator + .do_get_execution_payload_bid(TEST_SLOT, B256::ZERO, B256::ZERO, None, Some(auth), vec![ + EncodingType::Json, + ]) + .await +} + +/// The literal spec URL of the bid endpoint for `TEST_SLOT`, for the tests that +/// build their request by hand (bare-URL shape, missing headers, raw bodies). +fn bid_url(mock_validator: &MockValidator) -> String { + format!( + "{}eth/v1/builder/execution_payload_bid/{}/{}/{}/{}", + mock_validator.comm_boost.config.entry.url, + TEST_SLOT, + B256::ZERO, + B256::ZERO, + TEST_PROPOSER_PUBKEY, + ) +} + +/// The relay is asked and its bid is returned as JSON with the 200's required +/// Eth-Consensus-Version +#[tokio::test] +async fn test_get_execution_payload_bid() -> Result<()> { + let (mock_validator, mock_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + + info!("Sending get execution payload bid"); + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(mock_state.received_execution_payload_bid(), 1); + + let version_header = + res.headers().get("eth-consensus-version").and_then(|v| v.to_str().ok()).map(str::to_owned); + assert_eq!( + version_header.as_deref(), + Some("gloas"), + "200 response must set Eth-Consensus-Version: gloas" + ); + + let res = serde_json::from_slice::(&res.bytes().await?)?; + assert_eq!(res.version.to_string(), "gloas"); + assert_ne!(res.block_hash(), B256::ZERO); + assert_eq!(res.value(), 10); + Ok(()) +} + +/// `min_bid_eth` does NOT floor ePBS bids: the BN enforces the per-key +/// min_bid on this path (beacon-APIs #630), so a bid below the global CB +/// minimum still passes through +#[tokio::test] +async fn test_get_execution_payload_bid_below_min_bid_passes() -> Result<()> { + // Default mock bid: trustless 10 gwei, no execution payment; CB floor 20 gwei + let (mock_validator, mock_state) = setup_relay( + Chain::Hoodi, + |cfg| cfg.min_bid_wei = U256::from(20_000_000_000u64), + generate_mock_relay, + ) + .await?; + + let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT); + let res = get_json_bid(&mock_validator, &auth).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(mock_state.received_execution_payload_bid(), 1); + Ok(()) +} + +/// Relays on one host share its default auth data: only the first configured +/// one is asked +#[tokio::test] +async fn test_get_execution_payload_bid_shared_auth_data_asks_the_first() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, states) = setup_relays(chain, vec![ + MockRelayState::new(chain, random_secret()), + MockRelayState::new(chain, random_secret()), + ]) + .await?; + + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(states[0].received_execution_payload_bid(), 1); + assert_eq!(states[1].received_execution_payload_bid(), 0); + Ok(()) +} + +/// The spec default auth data, the builder's hostname, routes to the relay on +/// that host, and the auth reaches the relay unchanged. A hostname no relay +/// has is a 400 with the builder's data-mismatch message, and no relay is +/// contacted. +#[tokio::test] +async fn test_get_execution_payload_bid_demux_by_hostname() -> Result<()> { + let chain = Chain::Hoodi; + // 0.0.0.0 and 127.0.0.1 both reach the local mocks under distinct hostnames + let (mock_validator, states) = setup_relays_on_hosts(chain, vec![ + (MockRelayState::new(chain, random_secret()), "0.0.0.0"), + (MockRelayState::new(chain, random_secret()), "127.0.0.1"), + ]) + .await?; + + let res = get_json_bid(&mock_validator, &opaque_auth(b"127.0.0.1", TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(states[0].received_execution_payload_bid(), 0); + assert_eq!(states[1].received_execution_payload_bid(), 1); + assert_eq!(states[1].received_auth_data(), Some(b"127.0.0.1".to_vec())); + + let res = get_json_bid(&mock_validator, &opaque_auth(b"localhost", TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + assert_eq!(states[0].received_execution_payload_bid(), 0); + assert_eq!(states[1].received_execution_payload_bid(), 1); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(body["code"], 400); + assert_eq!( + body["message"], + "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder" + ); + Ok(()) +} + +/// A key listed in a `[[mux]]` is served by that mux's relays: its bid request +/// reaches the mux relay and not the default `[[relays]]` one. +#[tokio::test] +async fn test_get_execution_payload_bid_mux_routes_to_mux_relays() -> Result<()> { + let chain = Chain::Hoodi; + let (default_state, default_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let (mux_state, mux_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let default_relay = generate_mock_relay(default_port, default_state.signer.public_key())?; + let mux_relay = generate_mock_relay(mux_port, mux_state.signer.public_key())?; + + let pbs_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]); + config.all_relays = vec![mux_relay.clone(), default_relay]; + let muxed_key = random_secret().public_key(); + config.mux_lookup = Some(HashMap::from([(muxed_key.clone(), RuntimeMuxConfig { + id: "test".to_string(), + config: config.pbs_config.clone(), + relays: vec![mux_relay], + })])); + let state = PbsState::new(config, PathBuf::new()); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); + let mock_validator = MockValidator::new(pbs_port)?; + wait_for_ready(&mock_validator).await?; + + let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT); + let res = mock_validator + .do_get_execution_payload_bid( + TEST_SLOT, + B256::ZERO, + B256::ZERO, + Some(muxed_key), + Some(&auth), + vec![EncodingType::Json], + ) + .await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(mux_state.received_execution_payload_bid(), 1); + assert_eq!(default_state.received_execution_payload_bid(), 0); + Ok(()) +} + +/// `auth.message.slot` must match the proposal slot in the request path. +#[tokio::test] +async fn test_get_execution_payload_bid_auth_slot_mismatch_400() -> Result<()> { + let (mock_validator, mock_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + + let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT + 1); + let res = get_json_bid(&mock_validator, &auth).await?; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + assert_eq!( + mock_state.received_execution_payload_bid(), + 0, + "slot mismatch precedes relay calls" + ); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(body["code"], 400); + assert_eq!( + body["message"], + "Invalid SignedBuilderRequestAuth: auth.message.slot does not match the proposal slot in the request path" + ); + Ok(()) +} + +/// Requests refused before any relay is contacted, one row per status: a +/// missing required header, an unsupported media type and an unacceptable +/// Accept. +#[tokio::test] +async fn test_get_execution_payload_bid_rejected_before_relays() -> Result<()> { + let (mock_validator, mock_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + // Unlabeled, so JSON (builder-specs) + let body = serde_json::to_vec(&opaque_auth(TEST_AUTH_DATA, TEST_SLOT))?; + + // (headers dropped, header added, expected status, message excerpt) + let cases = [ + ( + vec![CONSENSUS_VERSION_HEADER], + None, + StatusCode::BAD_REQUEST, + "missing consensus version", + ), + ( + vec![HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS], + None, + StatusCode::BAD_REQUEST, + "Invalid request: Date-Milliseconds and X-Timeout-Ms headers are required", + ), + (vec![], Some(("content-type", "text/plain")), StatusCode::UNSUPPORTED_MEDIA_TYPE, ""), + (vec![], Some(("accept", "application/xml")), StatusCode::NOT_ACCEPTABLE, ""), + ]; + for (dropped, added, status, message) in cases { + let mut headers = vec![ + (CONSENSUS_VERSION_HEADER, "gloas".to_string()), + (HEADER_START_TIME_UNIX_MS, utcnow_ms().to_string()), + (HEADER_TIMEOUT_MS, "60000".to_string()), + ]; + headers.retain(|(name, _)| !dropped.contains(name)); + if let Some((name, value)) = added { + headers.push((name, value.to_string())); + } + + let mut req = + mock_validator.comm_boost.client.post(bid_url(&mock_validator)).body(body.clone()); + for (name, value) in &headers { + req = req.header(*name, value); + } + let res = req.send().await?; + assert_eq!(res.status(), status, "headers: {headers:?}"); + let json: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(json["code"], status.as_u16(), "{json}"); + assert!(json["message"].as_str().unwrap_or_default().contains(message), "{json}"); + } + assert_eq!(mock_state.received_execution_payload_bid(), 0, "rejected before any relay"); + Ok(()) +} + +/// A deadline that has already passed means there is no time to serve the +/// request: CB returns 204 rather than calling a relay it cannot beat. +#[tokio::test] +async fn test_get_execution_payload_bid_expired_deadline_204() -> Result<()> { + let (mock_validator, mock_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + let url = mock_validator.comm_boost.get_execution_payload_bid_url( + TEST_SLOT, + &B256::ZERO, + &B256::ZERO, + &random_secret().public_key(), + )?; + let res = mock_validator + .comm_boost + .client + .post(url) + .header(HEADER_START_TIME_UNIX_MS, utcnow_ms() - 5_000) + .header(HEADER_TIMEOUT_MS, 1_000u64) + .header("Eth-Consensus-Version", "gloas") + .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .body(opaque_auth(TEST_AUTH_DATA, TEST_SLOT).as_ssz_bytes()) + .send() + .await?; + assert_eq!(res.status(), StatusCode::NO_CONTENT); + assert_eq!(mock_state.received_execution_payload_bid(), 0, "no relay call past the deadline"); + Ok(()) +} + +#[tokio::test] +async fn test_get_execution_payload_bid_spec_url() -> Result<()> { + let (mock_validator, mock_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + + let url = bid_url(&mock_validator); + // The auth body, timing headers and version header are required, so even + // the bare-URL shape test must carry them + let res = mock_validator + .comm_boost + .client + .post(url) + .header(HEADER_START_TIME_UNIX_MS, utcnow_ms()) + .header(HEADER_TIMEOUT_MS, 60_000u64) + .header("Eth-Consensus-Version", "gloas") + .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .body(opaque_auth(TEST_AUTH_DATA, TEST_SLOT).as_ssz_bytes()) + .send() + .await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(mock_state.received_execution_payload_bid(), 1); + Ok(()) +} + +/// The response encoding follows the caller's Accept and defaults to JSON when +/// none is sent (builder-specs), and the 200 carries Eth-Consensus-Version +/// either way. Default relays answer in SSZ, so the last row's relay serves +/// only JSON to cover the JSON decode on the relay leg too. +#[tokio::test] +async fn test_get_execution_payload_bid_response_encoding() -> Result<()> { + let chain = Chain::Hoodi; + let cases = [ + (vec![], MockRelayState::new(chain, random_secret()), EncodingType::Json), + (vec![EncodingType::Ssz], MockRelayState::new(chain, random_secret()), EncodingType::Ssz), + ( + vec![EncodingType::Json], + MockRelayState::new(chain, random_secret()).with_json_only_response(), + EncodingType::Json, + ), + ]; + for (accept, relay, expected) in cases { + let (mock_validator, _) = setup_relays(chain, vec![relay]).await?; + let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT); + let res = mock_validator + .do_get_execution_payload_bid( + TEST_SLOT, + B256::ZERO, + B256::ZERO, + None, + Some(&auth), + accept, + ) + .await?; + assert_eq!(res.status(), StatusCode::OK, "{expected}"); + + let header = + |name| res.headers().get(name).and_then(|v| v.to_str().ok()).map(str::to_owned); + assert_eq!(header(CONTENT_TYPE.as_str()), Some(expected.to_string())); + assert_eq!(header(CONSENSUS_VERSION_HEADER).as_deref(), Some("gloas"), "{expected}"); + + let body = res.bytes().await?; + let (slot, block_hash) = match expected { + EncodingType::Ssz => { + let bid = SignedExecutionPayloadBid::from_ssz_bytes(&body) + .expect("body must SSZ-decode to a SignedExecutionPayloadBid"); + (bid.message.slot.as_u64(), bid.message.block_hash.0) + } + EncodingType::Json => { + let bid = serde_json::from_slice::(&body)?; + (bid.data.message.slot.as_u64(), bid.block_hash()) + } + }; + assert_eq!(slot, TEST_SLOT, "{expected}"); + assert_ne!(block_hash, B256::ZERO, "{expected}"); + } + Ok(()) +} + +/// An SSZ bid without `Eth-Consensus-Version` cannot be forwarded, since CB's +/// 200 must name the fork: that relay contributes no bid, so the request is a +/// 204. The relay is still contacted, so the 204 proves the drop. +#[tokio::test] +async fn test_get_execution_payload_bid_unversioned_ssz_bid_dropped() -> Result<()> { + let relay = MockRelayState::new(Chain::Hoodi, random_secret()) + .with_ssz_only_response() + .with_epbs_omit_consensus_version(); + let (mock_validator, states) = setup_relays(Chain::Hoodi, vec![relay]).await?; + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::NO_CONTENT); + assert_eq!(states[0].received_execution_payload_bid(), 1); + Ok(()) +} + +/// A relay error is a no-bid 204, not a 502, except the builder's own 400 and +/// 401, which reach the proposer. +#[tokio::test] +async fn test_get_execution_payload_bid_relay_status() -> Result<()> { + let (mock_validator, mock_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + + // (builder's answer, CB's answer) + let cases = [ + (StatusCode::INTERNAL_SERVER_ERROR, StatusCode::NO_CONTENT), + (StatusCode::BAD_REQUEST, StatusCode::BAD_REQUEST), + (StatusCode::UNAUTHORIZED, StatusCode::UNAUTHORIZED), + ]; + for (builder, expected) in cases { + mock_state.set_response_override(builder); + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), expected, "builder answered {builder}"); + } + assert_eq!(mock_state.received_execution_payload_bid(), 3); + Ok(()) +} + +/// A relay slower than the proposer's `X-Timeout-Ms` is dropped, so the request +/// degrades to 204 rather than waiting the relay out. +#[tokio::test] +async fn test_get_execution_payload_bid_slow_relay_times_out_204() -> Result<()> { + const DELAY_MS: u64 = 600; + const BUDGET_MS: u64 = 200; + + let chain = Chain::Hoodi; + let (mock_validator, states) = setup_relays(chain, vec![ + MockRelayState::new(chain, random_secret()).with_bid_delay_ms(DELAY_MS), + ]) + .await?; + + let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT); + let res = mock_validator + .do_get_execution_payload_bid_with_timeout( + TEST_SLOT, + B256::ZERO, + B256::ZERO, + None, + Some(&auth), + vec![EncodingType::Json], + BUDGET_MS, + ) + .await?; + assert_eq!( + res.status(), + StatusCode::NO_CONTENT, + "a relay slower than the deadline is dropped, not awaited" + ); + assert_eq!(states[0].received_execution_payload_bid(), 1, "the relay was still contacted"); + Ok(()) +} + +/// The relay gets CB's own timing headers, not the beacon node's: an +/// `X-Timeout-Ms` of five slots is clamped to one 12 s slot, less the 100 ms +/// buffer. The mock relay 400s a request missing either header, so the 200 +/// also proves `Date-Milliseconds` arrived. +#[tokio::test] +async fn test_get_execution_payload_bid_relay_timing_headers() -> Result<()> { + let (mock_validator, mock_state) = + setup_relay(Chain::Hoodi, |cfg| cfg.proposer_deadline_buffer_ms = 100, generate_mock_relay) + .await?; + + let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT); + let res = mock_validator + .do_get_execution_payload_bid_with_timeout( + TEST_SLOT, + B256::ZERO, + B256::ZERO, + None, + Some(&auth), + vec![EncodingType::Json], + 60_000, + ) + .await?; + assert_eq!(res.status(), StatusCode::OK); + let timeout_ms = mock_state.received_bid_timeout_ms().expect("relay saw X-Timeout-Ms"); + assert!(0 < timeout_ms && timeout_ms <= 11_900, "relay saw X-Timeout-Ms {timeout_ms}"); + Ok(()) +} diff --git a/tests/tests/pbs_submit_builder_preferences.rs b/tests/tests/pbs_submit_builder_preferences.rs new file mode 100644 index 000000000..b1079e7e4 --- /dev/null +++ b/tests/tests/pbs_submit_builder_preferences.rs @@ -0,0 +1,272 @@ +use cb_common::{ + pbs::{BuilderPreferences, BuilderPreferencesRequest, SignedBuilderRequestAuth}, + signer::random_secret, + types::Chain, + utils::utcnow_ms, + wire::{CONSENSUS_VERSION_HEADER, EncodingType}, +}; +use cb_tests::{ + mock_relay::MockRelayState, + utils::{ + TEST_AUTH_DATA, generate_mock_relay, opaque_auth, setup_relay, setup_relays, + setup_relays_on_hosts, + }, +}; +use eyre::Result; +use reqwest::{StatusCode, header::CONTENT_TYPE}; + +const TEST_MAX_EXECUTION_PAYMENT: u64 = 1_000_000_000; + +/// CB does not gate preferences on slot age, so a fixed slot serves. +const TEST_SLOT: u64 = 100; + +/// A slot that has already ended. Saturating: a chain whose genesis is under 10 +/// slots old would otherwise underflow rather than yield slot 0. +fn past_slot(chain: Chain) -> u64 { + let now_sec = utcnow_ms() / 1_000; + ((now_sec.saturating_sub(chain.genesis_time_sec())) / chain.slot_time_sec()).saturating_sub(10) +} + +fn preferences( + auth: SignedBuilderRequestAuth, + max_execution_payment: u64, +) -> BuilderPreferencesRequest { + BuilderPreferencesRequest { auth, preferences: BuilderPreferences { max_execution_payment } } +} + +/// The happy path: an SSZ submission reaches the builder as a 202, with the +/// preferences and the auth forwarded unchanged. +#[tokio::test] +async fn test_submit_builder_preferences() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, mock_state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + + let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT); + let request = preferences(auth.clone(), TEST_MAX_EXECUTION_PAYMENT); + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + + assert_eq!(res.status(), StatusCode::ACCEPTED); + assert_eq!(mock_state.received_builder_preferences(), 1); + assert_eq!(mock_state.received_max_execution_payment(), Some(TEST_MAX_EXECUTION_PAYMENT)); + + // The builder verifies what the proposer signed, so the auth must survive + // the hop byte for byte + let forwarded = mock_state.received_preferences_auth().expect("auth forwarded"); + assert_eq!(forwarded.message.data.to_vec(), TEST_AUTH_DATA.to_vec()); + assert_eq!(forwarded.message.slot, auth.message.slot); + assert_eq!(forwarded.signature, auth.signature); + + // A builder files preferences per proposer, so the wrong path segment would + // store them against the wrong validator + assert_eq!( + mock_state.received_preferences_pubkey(), + Some(mock_validator.comm_boost.pubkey().clone()), + "preferences must be filed under the proposer from the request path" + ); + Ok(()) +} + +/// The JSON wire form is the spec's, not merely whatever our own Serialize +/// produces: Gwei and slot are quoted strings and `data` is hex. Built from a +/// literal so that dropping the serde attributes fails here instead of +/// round-tripping through the same impl the assertion uses. Quotes on the Gwei +/// are optional on decode by ecosystem convention, so the unquoted number is +/// accepted too. +#[tokio::test] +async fn test_submit_builder_preferences_json_wire_form() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, mock_state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let slot = TEST_SLOT; + let url = mock_validator + .comm_boost + .submit_builder_preferences_url(&mock_validator.comm_boost.pubkey().clone())?; + + for max_execution_payment in [ + serde_json::json!(TEST_MAX_EXECUTION_PAYMENT.to_string()), + serde_json::json!(TEST_MAX_EXECUTION_PAYMENT), + ] { + let body = serde_json::json!({ + "preferences": { "max_execution_payment": max_execution_payment }, + "auth": { + // "0.0.0.0", the mock relay's hostname + "message": { "data": "0x302e302e302e30", "slot": slot.to_string() }, + "signature": format!("0x{}", "0".repeat(192)), + } + }); + let res = mock_validator + .comm_boost + .client + .post(url.clone()) + .header(CONTENT_TYPE, EncodingType::Json.content_type_header().clone()) + .header(CONSENSUS_VERSION_HEADER, "gloas") + .body(serde_json::to_vec(&body)?) + .send() + .await?; + + assert_eq!(res.status(), StatusCode::ACCEPTED, "{body}"); + assert_eq!(mock_state.received_max_execution_payment(), Some(TEST_MAX_EXECUTION_PAYMENT)); + let forwarded = mock_state.received_preferences_auth().expect("auth forwarded"); + assert_eq!(forwarded.message.data.to_vec(), TEST_AUTH_DATA.to_vec()); + assert_eq!(forwarded.message.slot.as_u64(), slot); + } + Ok(()) +} + +/// Requests refused before any builder is asked, with an ErrorMessage body: +/// `Eth-Consensus-Version` is required for JSON and SSZ alike (builder-specs +/// #165), and an unsupported media type is a 415, distinct from the 400 of a +/// malformed body. The per-rule detail (both encodings, the JSON default, an +/// empty body) is pinned once in the wire.rs decoder test. +#[tokio::test] +async fn test_submit_builder_preferences_rejected_before_builders() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, mock_state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let request = preferences(opaque_auth(TEST_AUTH_DATA, TEST_SLOT), TEST_MAX_EXECUTION_PAYMENT); + let url = mock_validator + .comm_boost + .submit_builder_preferences_url(&mock_validator.comm_boost.pubkey().clone())?; + + // (Content-Type, Eth-Consensus-Version, body, expected status, message excerpt) + let cases = [ + ( + "application/json", + None, + serde_json::to_vec(&request)?, + StatusCode::BAD_REQUEST, + "missing consensus version", + ), + ("text/plain", Some("gloas"), b"nonsense".to_vec(), StatusCode::UNSUPPORTED_MEDIA_TYPE, ""), + ]; + for (content_type, version, body, status, message) in cases { + let mut req = + mock_validator.comm_boost.client.post(url.clone()).header(CONTENT_TYPE, content_type); + if let Some(version) = version { + req = req.header(CONSENSUS_VERSION_HEADER, version); + } + let res = req.body(body).send().await?; + assert_eq!(res.status(), status, "{content_type}"); + let json: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(json["code"], status.as_u16(), "{json}"); + assert!(json["message"].as_str().unwrap_or_default().contains(message), "{json}"); + } + assert_eq!(mock_state.received_builder_preferences(), 0, "rejected before any builder"); + Ok(()) +} + +/// CB does not gate preferences on slot age: freshness (rejecting a stale or +/// replayed submission) is the builder's call, not the relay's, so CB forwards +/// regardless. A preference naming a slot that has already ended still reaches +/// the builder and is accepted. +#[tokio::test] +async fn test_submit_builder_preferences_past_slot_forwarded() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, mock_state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + + let request = + preferences(opaque_auth(TEST_AUTH_DATA, past_slot(chain)), TEST_MAX_EXECUTION_PAYMENT); + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + + assert_eq!(res.status(), StatusCode::ACCEPTED); + assert_eq!(mock_state.received_builder_preferences(), 1, "past-slot preference is forwarded"); + Ok(()) +} + +/// Auth data matching no relay's hostname is a 400 with the builder's +/// data-mismatch message: unmatched means CB has no builder to proxy to, and +/// proposer-private preferences must never broadcast to builders the proposer +/// did not address. +#[tokio::test] +async fn test_submit_builder_preferences_unmatched_auth_data_400() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, mock_state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + + let request = preferences(opaque_auth(&[0xbe, 0xef], TEST_SLOT), TEST_MAX_EXECUTION_PAYMENT); + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + assert_eq!(mock_state.received_builder_preferences(), 0, "no relay receives anything"); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(body["code"], 400); + assert_eq!( + body["message"], + "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder" + ); + Ok(()) +} + +/// The addressed builder's answer: only 202 is an acceptance. Its 400 and +/// 401 propagate, so the proposer can see the spec's rejection and +/// authentication failure, with a CB-written message rather than the +/// builder's untrusted body. Any other status, another 2xx included, is the +/// blanket 500; the 503 row is not itself a 500, so passthrough would show. +#[tokio::test] +async fn test_submit_builder_preferences_builder_status() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, mock_state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let request = preferences(opaque_auth(TEST_AUTH_DATA, TEST_SLOT), TEST_MAX_EXECUTION_PAYMENT); + + // (builder's answer, CB's answer) + let cases = [ + (StatusCode::OK, StatusCode::INTERNAL_SERVER_ERROR), + (StatusCode::BAD_REQUEST, StatusCode::BAD_REQUEST), + (StatusCode::UNAUTHORIZED, StatusCode::UNAUTHORIZED), + (StatusCode::SERVICE_UNAVAILABLE, StatusCode::INTERNAL_SERVER_ERROR), + ]; + for (builder, expected) in cases { + mock_state.set_response_override(builder); + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + assert_eq!(res.status(), expected, "builder answered {builder}"); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(body["code"], expected.as_u16(), "{body}"); + if builder == StatusCode::BAD_REQUEST { + assert_eq!( + body["message"], + "The addressed builder rejected the request with status 400" + ); + } + } + assert_eq!(mock_state.received_builder_preferences(), 4, "the builder is asked every time"); + Ok(()) +} + +/// An error body over PBS's 1 KiB read cap still reports the builder's status. +#[tokio::test] +async fn test_submit_builder_preferences_large_error_body() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, states) = + setup_relays(chain, vec![MockRelayState::new(chain, random_secret()).with_large_body()]) + .await?; + states[0].set_response_override(StatusCode::UNAUTHORIZED); + let request = preferences(opaque_auth(TEST_AUTH_DATA, TEST_SLOT), TEST_MAX_EXECUTION_PAYMENT); + + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); + Ok(()) +} + +/// Two relays on distinct hostnames: preferences addressing one must reach +/// only that builder, the other's received counter stays 0. +#[tokio::test] +async fn test_submit_builder_preferences_two_relays_addressed_one_only() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, states) = setup_relays_on_hosts(chain, vec![ + (MockRelayState::new(chain, random_secret()), "0.0.0.0"), + (MockRelayState::new(chain, random_secret()), "127.0.0.1"), + ]) + .await?; + + // The second relay, so that asking the first configured one would fail + let request = preferences(opaque_auth(b"127.0.0.1", TEST_SLOT), TEST_MAX_EXECUTION_PAYMENT); + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + + assert_eq!(res.status(), StatusCode::ACCEPTED); + assert_eq!(states[0].received_builder_preferences(), 0, "the unaddressed builder is not"); + assert_eq!(states[1].received_builder_preferences(), 1, "the addressed builder is asked"); + Ok(()) +} diff --git a/tests/tests/pbs_submit_signed_beacon_block.rs b/tests/tests/pbs_submit_signed_beacon_block.rs new file mode 100644 index 000000000..a5e474a16 --- /dev/null +++ b/tests/tests/pbs_submit_signed_beacon_block.rs @@ -0,0 +1,162 @@ +use cb_common::{ + pbs::SignedBeaconBlock, + signer::random_secret, + types::Chain, + utils::TestRandomSeed, + wire::{CONSENSUS_VERSION_HEADER, EncodingType}, +}; +use cb_tests::{ + mock_relay::MockRelayState, + utils::{generate_mock_relay, setup_relay, setup_relays}, +}; +use eyre::Result; +use lh_types::{MainnetEthSpec, SignedBeaconBlockGloas, Slot}; +use reqwest::{StatusCode, header::CONTENT_TYPE}; +use ssz::Encode; + +const TEST_SLOT: u64 = 100; + +/// A Gloas `SignedBeaconBlock` at `slot`, otherwise random. +fn gloas_block(slot: u64) -> SignedBeaconBlock { + let mut block = SignedBeaconBlockGloas::::test_random(); + block.message.slot = Slot::new(slot); + SignedBeaconBlock::Gloas(block) +} + +/// The endpoint is stateless: a block sent to the literal spec route +/// `POST /eth/v1/builder/beacon_blocks` is broadcast to every configured +/// builder, and each decodes it, by the forwarded fork header, back to the +/// same slot. +#[tokio::test] +async fn test_submit_signed_beacon_block_broadcasts_to_all_relays() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, states) = setup_relays(chain, vec![ + MockRelayState::new(chain, random_secret()), + MockRelayState::new(chain, random_secret()), + ]) + .await?; + + let block = gloas_block(TEST_SLOT); + let url = format!("{}eth/v1/builder/beacon_blocks", mock_validator.comm_boost.config.entry.url); + let res = mock_validator + .comm_boost + .client + .post(url) + .header(CONTENT_TYPE, "application/octet-stream") + .header(CONSENSUS_VERSION_HEADER, "gloas") + .body(block.as_ssz_bytes()) + .send() + .await?; + + assert_eq!(res.status(), StatusCode::ACCEPTED); + for state in &states { + assert_eq!(state.received_signed_beacon_block(), 1, "every builder receives the block"); + assert_eq!(state.received_block_slot(), Some(TEST_SLOT)); + } + Ok(()) +} + +/// The block is broadcast; if every builder rejects it, PBS maps the all-reject +/// outcome to a 500 (no builder accepted). +#[tokio::test] +async fn test_submit_signed_beacon_block_broadcast_all_reject_500() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, states) = setup_relays(chain, vec![ + MockRelayState::new(chain, random_secret()), + MockRelayState::new(chain, random_secret()), + ]) + .await?; + + // Make every builder reject the broadcast + for state in &states { + state.set_response_override(StatusCode::INTERNAL_SERVER_ERROR); + } + + let block = gloas_block(TEST_SLOT); + let res = mock_validator.do_submit_signed_beacon_block(&block, EncodingType::Ssz).await?; + + assert_eq!(res.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(states[0].received_signed_beacon_block(), 1, "every builder is asked"); + assert_eq!(states[1].received_signed_beacon_block(), 1, "every builder is asked"); + Ok(()) +} + +/// A broadcast where one builder accepts and the other rejects is still a 202: +/// one acceptance across the broadcast is success. This is the core of the +/// stateless broadcast model, distinct from the all-accept and all-reject +/// extremes the other tests cover. +#[tokio::test] +async fn test_submit_signed_beacon_block_broadcast_one_accepts_202() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, states) = setup_relays(chain, vec![ + MockRelayState::new(chain, random_secret()), + MockRelayState::new(chain, random_secret()), + ]) + .await?; + + // Only the second builder accepts; the first rejects. PBS must still 202. + states[0].set_response_override(StatusCode::INTERNAL_SERVER_ERROR); + + let block = gloas_block(TEST_SLOT); + let res = mock_validator.do_submit_signed_beacon_block(&block, EncodingType::Ssz).await?; + + assert_eq!( + res.status(), + StatusCode::ACCEPTED, + "one accepting builder makes the broadcast a success" + ); + assert_eq!(states[0].received_signed_beacon_block(), 1, "every builder is asked"); + assert_eq!(states[1].received_signed_beacon_block(), 1, "every builder is asked"); + Ok(()) +} + +/// Requests refused before anything is broadcast, each with an ErrorMessage +/// body. `Eth-Consensus-Version` is required (builder-specs #165) and checked +/// first, and a missing body is named as such rather than as an undecodable +/// one. The reveal must be SSZ: a JSON reveal, and an unlabeled one +/// (builder-specs defaults it to JSON), is a 415 rather than forwarded +/// mislabeled to fail opaquely at the builder. +#[tokio::test] +async fn test_submit_signed_beacon_block_rejected_before_broadcast() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let block = gloas_block(TEST_SLOT); + let (json_body, ssz_body) = (serde_json::to_vec(&block)?, block.as_ssz_bytes()); + let url = mock_validator.comm_boost.submit_signed_beacon_block_url()?; + + // (Content-Type, Eth-Consensus-Version, body, expected status, message excerpt) + let cases = [ + ( + Some("application/json"), + None, + json_body.clone(), + StatusCode::BAD_REQUEST, + "missing consensus version", + ), + (None, Some("gloas"), vec![], StatusCode::BAD_REQUEST, "missing request body"), + ( + Some("application/json"), + Some("gloas"), + json_body, + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "", + ), + (None, Some("gloas"), ssz_body, StatusCode::UNSUPPORTED_MEDIA_TYPE, ""), + ]; + for (content_type, version, body, status, message) in cases { + let mut req = mock_validator.comm_boost.client.post(url.clone()); + if let Some(content_type) = content_type { + req = req.header(CONTENT_TYPE, content_type); + } + if let Some(version) = version { + req = req.header(CONSENSUS_VERSION_HEADER, version); + } + let res = req.body(body).send().await?; + assert_eq!(res.status(), status, "{content_type:?}, {version:?}"); + let json: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(json["code"], status.as_u16(), "{json}"); + assert!(json["message"].as_str().unwrap_or_default().contains(message), "{json}"); + } + assert_eq!(state.received_signed_beacon_block(), 0, "nothing is broadcast"); + Ok(()) +} From 852a585f8d5547ea72ec97b124fc263b1fd4fedb Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 5 Oct 2026 22:29:03 -0700 Subject: [PATCH 09/24] docs(epbs): keep the builder config anchor the troubleshooting table links to --- docs/docs/get_started/epbs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 748db1b82..27cc0f75d 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -41,7 +41,7 @@ To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and i List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be under one slot, `12000` on mainnet. -### 2. Point each validator key at Commit-Boost +### 2. Point each validator key at Commit-Boost {#validator-builder-config} Write each key's builder config through its validator client's keymanager API, which must implement the builder config endpoint ([keymanager-APIs #88](https://github.com/ethereum/keymanager-APIs/pull/88)). Add one entry per builder: `url` is Commit-Boost's URL, and `auth_data` is the hex of the builder's hostname. A key without builder config sends the default auth data of Commit-Boost's own URL, which matches no builder, so every bid request gets `400`. From bb66bea55e2628a0cd33922db5a08e2da2bc5cd8 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 6 Oct 2026 11:02:38 -0700 Subject: [PATCH 10/24] address review comments --- crates/common/src/pbs/types/mod.rs | 25 ----- crates/common/src/wire.rs | 9 +- crates/pbs/src/routes/builder_preferences.rs | 67 ++++------- .../pbs/src/routes/execution_payload_bid.rs | 104 +++++++++--------- .../src/routes/submit_signed_beacon_block.rs | 1 - crates/pbs/src/utils.rs | 37 +++---- tests/tests/pbs_get_execution_payload_bid.rs | 19 ++-- 7 files changed, 100 insertions(+), 162 deletions(-) diff --git a/crates/common/src/pbs/types/mod.rs b/crates/common/src/pbs/types/mod.rs index 57845d4e1..ccf79e381 100644 --- a/crates/common/src/pbs/types/mod.rs +++ b/crates/common/src/pbs/types/mod.rs @@ -74,31 +74,6 @@ pub type SignedExecutionPayloadBid = lh_types::SignedExecutionPayloadBid; -pub trait GetExecutionPayloadBidInfo { - fn block_hash(&self) -> B256; - fn value(&self) -> u64; - fn execution_payment(&self) -> u64; - fn builder_index(&self) -> u64; -} - -impl GetExecutionPayloadBidInfo for GetExecutionPayloadBidResponse { - fn block_hash(&self) -> B256 { - self.data.message.block_hash.0 - } - - fn value(&self) -> u64 { - self.data.message.value - } - - fn execution_payment(&self) -> u64 { - self.data.message.execution_payment - } - - fn builder_index(&self) -> u64 { - self.data.message.builder_index - } -} - /// Path params of POST /// `/eth/v1/builder/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/ /// {proposer_pubkey}` diff --git a/crates/common/src/wire.rs b/crates/common/src/wire.rs index 16037c5e0..48f3be15a 100644 --- a/crates/common/src/wire.rs +++ b/crates/common/src/wire.rs @@ -311,14 +311,7 @@ pub fn require_consensus_version_header( // Gloas-only until later forks are defined match ForkName::from_str(value).map_err(|_| unsupported())? { ForkName::Gloas => Ok(ForkName::Gloas), - ForkName::Base | - ForkName::Altair | - ForkName::Bellatrix | - ForkName::Capella | - ForkName::Deneb | - ForkName::Electra | - ForkName::Fulu | - ForkName::Heze => Err(unsupported()), + _ => Err(unsupported()), } } diff --git a/crates/pbs/src/routes/builder_preferences.rs b/crates/pbs/src/routes/builder_preferences.rs index fab0902aa..dc8596e72 100644 --- a/crates/pbs/src/routes/builder_preferences.rs +++ b/crates/pbs/src/routes/builder_preferences.rs @@ -5,14 +5,12 @@ use axum::{ response::IntoResponse, }; use cb_common::{ - pbs::{ - BuilderPreferencesRequest, RelayClient, SubmitBuilderPreferencesParams, error::PbsError, - }, + pbs::{BuilderPreferencesRequest, SubmitBuilderPreferencesParams}, wire::{decode_versioned_request_body, get_user_agent}, }; use reqwest::StatusCode; use ssz::Encode; -use tracing::{debug, error, info}; +use tracing::{error, info}; use crate::{ PbsStateGuard, @@ -21,7 +19,7 @@ use crate::{ state::{BuilderApiState, PbsState}, utils::{ builder_rejection, epbs_base_send_headers, log_mux_selection, post_ssz_expect_accepted, - record_beacon_status, record_client_error, record_request_failure, resolve_addressed_relay, + record_beacon_status, record_request_failure, resolve_addressed_relay, }, }; @@ -33,7 +31,7 @@ pub async fn handle_submit_builder_preferences( ) -> Result { let request = decode_versioned_request_body::(&req_headers, &body) - .map_err(|err| record_client_error(err, SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG))?; + .map_err(|err| record_request_failure(err, SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG))?; tracing::Span::current().record("validator", tracing::field::debug(¶ms.proposer_pubkey)); tracing::Span::current().record("slot", request.auth.message.slot.as_u64()); @@ -78,50 +76,33 @@ pub async fn submit_builder_preferences( // timeout rather than the block-production one let timeout_ms = pbs_config.timeout_register_validator_ms; - let relay_id = relay.id.clone(); - match send_submit_builder_preferences( - params.proposer_pubkey, - body, - relay, - send_headers, - timeout_ms, - ) - .await - { - Ok(()) => { - info!(%relay_id, "builder preferences submitted"); + let relay_id = relay.id.as_ref(); + let sent = match relay.submit_builder_preferences_url(¶ms.proposer_pubkey) { + Ok(url) => { + post_ssz_expect_accepted( + &relay, + url, + body, + send_headers, + timeout_ms, + SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG, + ) + .await + } + Err(err) => Err(err), + }; + match sent { + Ok(latency) => { + info!(relay_id, ?latency, "builder preferences submitted"); Ok(()) } Err(err) => { if err.is_timeout() { - error!(err = "Timed Out", %relay_id); + error!(err = "Timed Out", relay_id); } else { - error!(%err, %relay_id); + error!(%err, relay_id); } Err(builder_rejection(&err).unwrap_or(PbsClientError::NoBuilderResponse)) } } } - -async fn send_submit_builder_preferences( - proposer_pubkey: cb_common::types::BlsPublicKey, - body: Bytes, - relay: RelayClient, - headers: HeaderMap, - timeout_ms: u64, -) -> Result<(), PbsError> { - let url = relay.submit_builder_preferences_url(&proposer_pubkey)?; - - let request_latency = post_ssz_expect_accepted( - &relay, - url, - body, - headers, - timeout_ms, - SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG, - ) - .await?; - - debug!(relay_id = relay.id.as_ref(), latency = ?request_latency, "preferences accepted"); - Ok(()) -} diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index 68c96774f..1385ea3c5 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -8,9 +8,9 @@ use axum::{ }; use cb_common::{ pbs::{ - GetExecutionPayloadBidInfo, GetExecutionPayloadBidParams, GetExecutionPayloadBidResponse, - HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, - SignedExecutionPayloadBid, error::PbsError, + GetExecutionPayloadBidParams, GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, + HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, SignedExecutionPayloadBid, + error::PbsError, }, utils::{ms_into_slot, utcnow_ms}, wire::{ @@ -34,8 +34,7 @@ use crate::{ state::{BuilderApiState, PbsState}, utils::{ builder_rejection, epbs_base_send_headers, format_gwei_as_eth, log_mux_selection, - record_beacon_status, record_client_error, record_request_failure, resolve_addressed_relay, - send_to_relay, + record_beacon_status, record_request_failure, resolve_addressed_relay, send_to_relay, }, }; @@ -46,7 +45,7 @@ pub async fn handle_get_execution_payload_bid( body: Bytes, ) -> Result { let auth = decode_versioned_request_body::(&req_headers, &body) - .map_err(|err| record_client_error(err, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG))?; + .map_err(|err| record_request_failure(err, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG))?; tracing::Span::current().record("slot", params.slot); tracing::Span::current().record("parent_hash", tracing::field::debug(params.parent_hash)); tracing::Span::current().record("parent_root", tracing::field::debug(params.parent_root)); @@ -58,16 +57,19 @@ pub async fn handle_get_execution_payload_bid( let ms_into_slot = ms_into_slot(params.slot, state.config.chain); let response_encoding = get_accept_types(&req_headers) - .inspect_err(|err| error!(%err, "error parsing accept header")) - .map_err(|err| record_client_error(err, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG))? + .map_err(|err| record_request_failure(err, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG))? .primary; info!(ua, ms_into_slot, "new request"); + let version = req_headers.get(CONSENSUS_VERSION_HEADER).cloned(); match get_execution_payload_bid(params, auth, req_headers, state).await { - Ok(Some(bid)) => { - encode_bid_response(bid, response_encoding, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG) - } + Ok(Some(bid)) => Ok(encode_bid_response( + bid, + response_encoding, + version, + GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, + )), Ok(None) => { info!("no bid for slot"); record_beacon_status("204", GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG); @@ -77,34 +79,42 @@ pub async fn handle_get_execution_payload_bid( } } -fn encode_bid_response( +/// A relay's bid with the body and encoding it arrived in, so a beacon node +/// that asks for the same encoding gets the relay's bytes unchanged +pub(crate) struct RelayBid { bid: GetExecutionPayloadBidResponse, + body: Bytes, + encoding: EncodingType, +} + +/// `version` is the beacon node's own `Eth-Consensus-Version`: the 200 names +/// the fork it asked for +fn encode_bid_response( + RelayBid { bid, body, encoding }: RelayBid, response_encoding: EncodingType, + version: Option, endpoint: &str, -) -> Result { +) -> Response { + let message = &bid.data.message; info!( - trustless_bid_eth = format_gwei_as_eth(bid.value()), - execution_payment_eth = format_gwei_as_eth(bid.execution_payment()), - block_hash = %bid.block_hash(), - builder_index = bid.builder_index(), + trustless_bid_eth = format_gwei_as_eth(message.value), + execution_payment_eth = format_gwei_as_eth(message.execution_payment), + block_hash = %message.block_hash, + builder_index = message.builder_index, "received header" ); - // Eth-Consensus-Version is required on the 200 for both encodings - let consensus_version_header = HeaderValue::from_str(&bid.version.to_string()) - .expect("fork name is always a valid header value"); - record_beacon_status("200", endpoint); + let content_type = [(CONTENT_TYPE, response_encoding.content_type_header().clone())]; let mut res = match response_encoding { - EncodingType::Ssz => { - let mut res = bid.data.as_ssz_bytes().into_response(); - res.headers_mut().insert(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()); - res - } + _ if encoding == response_encoding => (content_type, body).into_response(), + EncodingType::Ssz => (content_type, bid.data.as_ssz_bytes()).into_response(), EncodingType::Json => axum::Json(bid).into_response(), }; - res.headers_mut().insert(CONSENSUS_VERSION_HEADER, consensus_version_header); - Ok(res) + if let Some(version) = version { + res.headers_mut().insert(CONSENSUS_VERSION_HEADER, version); + } + res } /// Implements https://ethereum.github.io/builder-specs/?urls.primaryName=dev#/Builder/getExecutionPayloadBid @@ -114,7 +124,7 @@ pub async fn get_execution_payload_bid( auth: SignedBuilderRequestAuth, req_headers: HeaderMap, state: PbsState, -) -> Result, PbsClientError> { +) -> Result, PbsClientError> { let (pbs_config, relays, maybe_mux_id) = state.mux_config_and_relays(¶ms.proposer_pubkey); log_mux_selection(maybe_mux_id, relays.len(), ¶ms.proposer_pubkey); @@ -145,7 +155,7 @@ pub async fn get_execution_payload_bid( let mut send_headers = epbs_base_send_headers(&req_headers)?; - // The bid is re-encoded for the BN anyway, so ask for the cheaper SSZ + // SSZ is smaller and reaches a beacon node that asks for SSZ unchanged send_headers.insert(ACCEPT, OUTBOUND_ACCEPT_SSZ_FIRST.clone()); let body = Bytes::from(auth.as_ssz_bytes()); @@ -154,13 +164,13 @@ pub async fn get_execution_payload_bid( // A relay that errors or times out contributes no bid: 204, never a 502. // The builder's own 400 and 401 still reach the proposer. match send_get_execution_payload_bid(params, body, relay, send_headers, max_timeout_ms).await { - Ok(Some(bid)) => { + Ok(Some(relay_bid)) => { RELAY_LAST_SLOT.with_label_values(&[relay_id.as_str()]).set(slot as i64); - // value() is already gwei (the gauge is labelled gwei), so it is set unscaled + // The bid's value is already gwei, the gauge's unit, so it is set unscaled RELAY_HEADER_VALUE .with_label_values(&[relay_id.as_str()]) - .set(i64::try_from(bid.value()).unwrap_or_default()); - Ok(Some(bid)) + .set(i64::try_from(relay_bid.bid.data.message.value).unwrap_or_default()); + Ok(Some(relay_bid)) } Ok(None) => Ok(None), Err(err) if err.is_timeout() => { @@ -205,7 +215,7 @@ async fn send_get_execution_payload_bid( relay: RelayClient, mut headers: HeaderMap, timeout_ms: u64, -) -> Result, PbsError> { +) -> Result, PbsError> { let url = relay.get_execution_payload_bid_url( params.slot, ¶ms.parent_hash, @@ -240,7 +250,6 @@ async fn send_get_execution_payload_bid( }; let response_bytes = safe_read_http_response(res, MAX_SIZE_GET_HEADER_RESPONSE).await?; - let header_size_bytes = response_bytes.len(); if code == StatusCode::NO_CONTENT { debug!( relay_id = relay.id.as_ref(), @@ -252,13 +261,9 @@ async fn send_get_execution_payload_bid( return Ok(None); } - let get_header_response = match content_type { - EncodingType::Json => serde_json::from_slice::( - &response_bytes, - ) - .map_err(|err| PbsError::JsonDecode { - err, - raw: String::from_utf8_lossy(&response_bytes).into_owned(), + let bid = match content_type { + EncodingType::Json => serde_json::from_slice(&response_bytes).map_err(|err| { + PbsError::JsonDecode { err, raw: String::from_utf8_lossy(&response_bytes).into_owned() } })?, EncodingType::Ssz => { // SSZ requires the fork from Eth-Consensus-Version; its absence is a @@ -279,18 +284,7 @@ async fn send_get_execution_payload_bid( } }; - debug!( - relay_id = relay.id.as_ref(), - header_size_bytes, - latency = ?request_latency, - version =? get_header_response.version, - trustless_bid_eth = format_gwei_as_eth(get_header_response.data.message.value), - execution_payment_eth = format_gwei_as_eth(get_header_response.data.message.execution_payment), - block_hash = %get_header_response.data.message.block_hash, - "received new header" - ); - - Ok(Some(get_header_response)) + Ok(Some(RelayBid { bid, body: response_bytes.into(), encoding: content_type })) } #[cfg(test)] diff --git a/crates/pbs/src/routes/submit_signed_beacon_block.rs b/crates/pbs/src/routes/submit_signed_beacon_block.rs index 8f60f1db2..75bb35542 100644 --- a/crates/pbs/src/routes/submit_signed_beacon_block.rs +++ b/crates/pbs/src/routes/submit_signed_beacon_block.rs @@ -77,7 +77,6 @@ pub async fn submit_signed_beacon_block( })) .await; let accepted = results - .into_iter() .zip(relays.iter()) .filter(|(res, relay)| match res { Ok(()) => true, diff --git a/crates/pbs/src/utils.rs b/crates/pbs/src/utils.rs index c07da9d40..77c03765d 100644 --- a/crates/pbs/src/utils.rs +++ b/crates/pbs/src/utils.rs @@ -4,8 +4,9 @@ use std::{ }; use alloy::primitives::utils::{ParseUnits, Unit}; +use axum::body::Bytes; use cb_common::{ - pbs::{ForkName, RelayClient, error::PbsError}, + pbs::{RelayClient, error::PbsError}, types::BlsPublicKey, wire::{ CONSENSUS_VERSION_HEADER, EncodingType, get_user_agent_with_version, @@ -15,7 +16,7 @@ use cb_common::{ use futures::future::join_all; use reqwest::{ StatusCode, - header::{CONTENT_TYPE, HeaderMap, HeaderValue, USER_AGENT}, + header::{CONTENT_TYPE, HeaderMap, USER_AGENT}, }; use tracing::{Instrument, debug, error, warn}; use url::Url; @@ -54,24 +55,17 @@ pub(crate) async fn send_to_relay( Ok((res, request_latency)) } -/// Count a request-rejection in `BEACON_NODE_STATUS` before it short-circuits -/// the handler, so rejected requests show up as 4xx in the endpoint counter. -pub(crate) fn record_client_error( - err: impl Into, - endpoint: &str, -) -> PbsClientError { - let err = err.into(); - BEACON_NODE_STATUS.with_label_values(&[err.status_code().as_str(), endpoint]).inc(); - err -} - pub(crate) fn record_beacon_status(code: &str, endpoint: &str) { BEACON_NODE_STATUS.with_label_values(&[code, endpoint]).inc(); } /// Logs and counts a failed ePBS request before it is returned to the beacon /// node. A 4xx is the caller's fault, not CB's: only a 5xx is an error. -pub(crate) fn record_request_failure(err: PbsClientError, endpoint: &str) -> PbsClientError { +pub(crate) fn record_request_failure( + err: impl Into, + endpoint: &str, +) -> PbsClientError { + let err = err.into(); if err.status_code().is_server_error() { error!(%err, "{endpoint} failed"); } else { @@ -84,7 +78,7 @@ pub(crate) fn record_request_failure(err: PbsClientError, endpoint: &str) -> Pbs /// Fans `sends` out on detached tasks and waits for all of them pub(crate) async fn join_detached_sends( sends: impl IntoIterator, -) -> Vec> +) -> impl Iterator> where F: Future> + Send + 'static, { @@ -94,7 +88,6 @@ where .await .into_iter() .map(|joined| joined.unwrap_or_else(|err| Err(PbsError::TokioJoinError(err)))) - .collect() } pub(crate) fn log_mux_selection( @@ -116,7 +109,7 @@ pub(crate) fn log_mux_selection( pub(crate) async fn post_ssz_expect_accepted( relay: &RelayClient, url: Url, - body: impl Into, + body: Bytes, headers: HeaderMap, timeout_ms: u64, tag: &str, @@ -160,18 +153,16 @@ pub(crate) fn builder_rejection(err: &PbsError) -> Option { } /// Headers every ePBS relay request carries: the versioned `User-Agent`, and -/// `Eth-Consensus-Version` since each body is a fork-versioned type. +/// the beacon node's `Eth-Consensus-Version`, which the route has validated pub(crate) fn epbs_base_send_headers(req_headers: &HeaderMap) -> Result { let mut headers = HeaderMap::new(); headers.insert( USER_AGENT, get_user_agent_with_version(req_headers).map_err(|_| PbsClientError::Internal)?, ); - headers.insert( - CONSENSUS_VERSION_HEADER, - HeaderValue::from_str(&ForkName::Gloas.to_string()) - .expect("fork name is always a valid header value"), - ); + if let Some(version) = req_headers.get(CONSENSUS_VERSION_HEADER) { + headers.insert(CONSENSUS_VERSION_HEADER, version.clone()); + } Ok(headers) } diff --git a/tests/tests/pbs_get_execution_payload_bid.rs b/tests/tests/pbs_get_execution_payload_bid.rs index b31aabbe1..0cc4fb2fa 100644 --- a/tests/tests/pbs_get_execution_payload_bid.rs +++ b/tests/tests/pbs_get_execution_payload_bid.rs @@ -4,8 +4,8 @@ use alloy::primitives::{B256, U256}; use cb_common::{ config::RuntimeMuxConfig, pbs::{ - GetExecutionPayloadBidInfo, GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, - HEADER_TIMEOUT_MS, SignedBuilderRequestAuth, SignedExecutionPayloadBid, + GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, + SignedBuilderRequestAuth, SignedExecutionPayloadBid, }, signer::random_secret, types::Chain, @@ -77,8 +77,8 @@ async fn test_get_execution_payload_bid() -> Result<()> { let res = serde_json::from_slice::(&res.bytes().await?)?; assert_eq!(res.version.to_string(), "gloas"); - assert_ne!(res.block_hash(), B256::ZERO); - assert_eq!(res.value(), 10); + assert_ne!(res.data.message.block_hash.0, B256::ZERO); + assert_eq!(res.data.message.value, 10); Ok(()) } @@ -327,8 +327,8 @@ async fn test_get_execution_payload_bid_spec_url() -> Result<()> { /// The response encoding follows the caller's Accept and defaults to JSON when /// none is sent (builder-specs), and the 200 carries Eth-Consensus-Version -/// either way. Default relays answer in SSZ, so the last row's relay serves -/// only JSON to cover the JSON decode on the relay leg too. +/// either way. Default relays answer in SSZ; the JSON-only relays cover the +/// relay's JSON passed through and converted to SSZ. #[tokio::test] async fn test_get_execution_payload_bid_response_encoding() -> Result<()> { let chain = Chain::Hoodi; @@ -340,6 +340,11 @@ async fn test_get_execution_payload_bid_response_encoding() -> Result<()> { MockRelayState::new(chain, random_secret()).with_json_only_response(), EncodingType::Json, ), + ( + vec![EncodingType::Ssz], + MockRelayState::new(chain, random_secret()).with_json_only_response(), + EncodingType::Ssz, + ), ]; for (accept, relay, expected) in cases { let (mock_validator, _) = setup_relays(chain, vec![relay]).await?; @@ -370,7 +375,7 @@ async fn test_get_execution_payload_bid_response_encoding() -> Result<()> { } EncodingType::Json => { let bid = serde_json::from_slice::(&body)?; - (bid.data.message.slot.as_u64(), bid.block_hash()) + (bid.data.message.slot.as_u64(), bid.data.message.block_hash.0) } }; assert_eq!(slot, TEST_SLOT, "{expected}"); From d95d41484e7c65b8d2a85c2a4a5b263f02690426 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 6 Oct 2026 11:39:21 -0700 Subject: [PATCH 11/24] test: wait for PBS's startup status check in test_mux to fix race condition and lint test --- tests/tests/pbs_mux.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/tests/pbs_mux.rs b/tests/tests/pbs_mux.rs index 7778fb456..6f01f502d 100644 --- a/tests/tests/pbs_mux.rs +++ b/tests/tests/pbs_mux.rs @@ -245,8 +245,15 @@ async fn test_mux() -> Result<()> { let state = PbsState::new(config, PathBuf::new()); tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); - // leave some time to start servers - tokio::time::sleep(Duration::from_millis(100)).await; + // PBS checks /status on every relay once at startup; wait for that check so + // the status count below holds only this test's request + for _ in 0..100 { + if mock_state.received_get_status() == 3 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!(mock_state.received_get_status(), 3); // Send default request without specifying a validator key let mock_validator = MockValidator::new(pbs_port)?; @@ -271,7 +278,7 @@ async fn test_mux() -> Result<()> { // Status requests should go to all relays info!("Sending get status"); assert_eq!(mock_validator.do_get_status().await?.status(), StatusCode::OK); - assert_eq!(mock_state.received_get_status(), 3); // default + 2 mux relays were used + assert_eq!(mock_state.received_get_status(), 6); // default + 2 mux relays were used // Register requests should go to all relays info!("Sending register validator"); From 1694f5457437836144b3198aef8a3601befa094d Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 5 Oct 2026 22:31:09 -0700 Subject: [PATCH 12/24] feat(pbs): dial builders outside the config for ePBS requests Auth data that is an http(s) URL now also routes to the relay entry with the same scheme, host and port. Either form may end in `?` and parameters for the builder: Commit-Boost routes on the part before `?` and forwards the signed auth, parameters included, unchanged. When no relay entry matches, Commit-Boost dials the builder the auth data names for a bid or preferences request: `https://`, or the URL. The target is resolved once, and the lookup and the dial share the request's budget. A name that does not resolve, or any address that is not public unicast, gets 400; a lookup or dial setup that runs out of time counts as a builder that did not answer. At most 32 lookups run at once, since a timed-out lookup keeps its blocking thread. The dial goes only to the checked addresses, with no proxy and no redirects. A request from a Commit-Boost never leads to a dial, so a dial that reaches a Commit-Boost, this one included, goes no further. Auth data, a dialed builder's error body and a dial target come from the request, so they are logged escaped, the body capped at 1 KiB and the target as its origin. The signed block gets 202 once it has been forwarded, whatever the builders answer, since the beacon node gossips it anyway. A block whose bid came from a dialed builder, which is not a relay entry, would otherwise always get 500. When no builder accepts the block, Commit-Boost logs a warning. --- crates/common/src/pbs/relay.rs | 41 ++- crates/pbs/Cargo.toml | 3 + crates/pbs/src/dial.rs | 337 ++++++++++++++++++ crates/pbs/src/error.rs | 10 +- crates/pbs/src/lib.rs | 3 + crates/pbs/src/routes/builder_preferences.rs | 14 +- .../pbs/src/routes/execution_payload_bid.rs | 57 ++- .../src/routes/submit_signed_beacon_block.rs | 12 +- crates/pbs/src/utils.rs | 97 +++-- docs/docs/get_started/epbs.md | 26 +- tests/Cargo.toml | 2 +- tests/src/mock_relay.rs | 26 +- tests/src/mock_validator.rs | 5 +- tests/tests/pbs_get_execution_payload_bid.rs | 117 +++++- tests/tests/pbs_submit_builder_preferences.rs | 29 +- tests/tests/pbs_submit_signed_beacon_block.rs | 39 +- 16 files changed, 686 insertions(+), 132 deletions(-) create mode 100644 crates/pbs/src/dial.rs diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index ae7f4b9aa..b877d8580 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -97,6 +97,14 @@ pub struct RelayClient { impl RelayClient { pub fn new(config: RelayConfig) -> eyre::Result { + Self::with_client_builder(config, reqwest::Client::builder()) + } + + /// [`Self::new`] on a client builder that already holds extra settings + pub fn with_client_builder( + config: RelayConfig, + builder: reqwest::ClientBuilder, + ) -> eyre::Result { let stream_url = match config.get_header { GetHeaderTransport::Http => None, GetHeaderTransport::Stream => Some(stream_url(&config.entry.url)?), @@ -131,10 +139,8 @@ impl RelayClient { } } - let client = reqwest::Client::builder() - .default_headers(headers.clone()) - .timeout(DEFAULT_REQUEST_TIMEOUT) - .build()?; + let client = + builder.default_headers(headers.clone()).timeout(DEFAULT_REQUEST_TIMEOUT).build()?; Ok(Self { id: Arc::new(config.id().to_owned()), @@ -255,6 +261,13 @@ impl RelayClient { } } +/// The builder URL in URL-form auth data. Only `http` and `https` count, since +/// opaque data such as `builder-a:prod` parses as a URL too. +pub fn decode_auth_data_url(data: &[u8]) -> Option { + let url = Url::parse(std::str::from_utf8(data).ok()?).ok()?; + matches!(url.scheme(), "http" | "https").then_some(url) +} + /// First 4 bytes of the value's SHA-256: enough to see a rotation /// across reloads, too short to identify a value on its own fn value_fingerprint(value: &str) -> String { @@ -268,12 +281,30 @@ mod tests { use alloy::primitives::B256; - use super::{GetHeaderRequest, RelayClient, RelayEntry, value_fingerprint}; + use super::{ + GetHeaderRequest, RelayClient, RelayEntry, decode_auth_data_url, value_fingerprint, + }; use crate::{ config::{GetHeaderTransport, RelayConfig, test_env::RELAY_URL}, utils::bls_pubkey_from_hex_unchecked, }; + #[test] + fn decode_auth_data_url_accepts_only_http_urls() { + let cases: [(&[u8], Option<&str>); 5] = [ + (b"https://Builder-A.example.com:443/eth", Some("https://builder-a.example.com/eth")), + (b"http://10.0.0.1:18550", Some("http://10.0.0.1:18550/")), + // Opaque data and host:port parse as URLs with their own scheme + (b"builder-a:prod", None), + (b"localhost:18550", None), + (b"builder-a.example.com", None), + ]; + for (data, expected) in cases { + let decoded = decode_auth_data_url(data).map(|url| url.to_string()); + assert_eq!(decoded.as_deref(), expected, "{}", String::from_utf8_lossy(data)); + } + } + #[test] fn test_relay_entry() { let pubkey = bls_pubkey_from_hex_unchecked( diff --git a/crates/pbs/Cargo.toml b/crates/pbs/Cargo.toml index 7497aa930..3a52e12ef 100644 --- a/crates/pbs/Cargo.toml +++ b/crates/pbs/Cargo.toml @@ -5,6 +5,9 @@ publish = false rust-version.workspace = true version.workspace = true +[features] +testing-flags = [] + [dependencies] alloy.workspace = true async-trait.workspace = true diff --git a/crates/pbs/src/dial.rs b/crates/pbs/src/dial.rs new file mode 100644 index 000000000..6b7fdfdd5 --- /dev/null +++ b/crates/pbs/src/dial.rs @@ -0,0 +1,337 @@ +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}, + sync::LazyLock, + time::Duration, +}; + +use cb_common::{ + config::{GetHeaderTransport, RelayConfig}, + pbs::{RelayClient, RelayEntry}, + types::BlsPublicKey, + utils::bls_pubkey_from_hex, +}; +use tokio::sync::Semaphore; +use tracing::{info, warn}; +use url::Url; + +use crate::error::PbsClientError; + +/// The `relay_id` of every dial: the target comes from request data, so a +/// per-host metric label would be unbounded +const DIAL_RELAY_ID: &str = "dial"; + +/// The BLS12-381 G1 generator, standing in for the pubkey `RelayEntry` requires +/// but only PBS reads. A workaround to leave `RelayClient` unchanged until PBS +/// is deprecated, when the pubkey can go. +static DIAL_PUBKEY: LazyLock = LazyLock::new(|| { + bls_pubkey_from_hex( + "0x97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb", + ) + .expect("the G1 generator is a valid pubkey") +}); + +/// The address in auth data, before the first `?`. Parameters after it are +/// for the builder, which gets them inside the signed auth +pub(crate) fn auth_data_address(data: &[u8]) -> &[u8] { + data.iter().position(|&byte| byte == b'?').map_or(data, |end| &data[..end]) +} + +/// A relay for the builder that an unmatched address names. The target is +/// untrusted, so it is resolved once and dialed only at the vetted addresses. +pub(crate) async fn dial_relay( + data_url: Option, + address: &[u8], + lookup_timeout: Duration, +) -> Result { + let url = dial_target(data_url, address)?; + let origin = url.origin().ascii_serialization(); + let addrs = resolve_dial_target(&url, &origin, lookup_timeout).await?; + info!(%origin, ?addrs, "auth data names a builder outside the config, dialing it"); + pinned_relay(url, &addrs).map_err(|err| { + warn!(%err, "failed to build the dial client"); + PbsClientError::Internal + }) +} + +/// `data_url`, else `https:///` for the builder-specs default auth +/// data, a bare hostname +fn dial_target(data_url: Option, address: &[u8]) -> Result { + data_url + .or_else(|| { + let url = + Url::parse(&format!("https://{}/", std::str::from_utf8(address).ok()?)).ok()?; + (url.host_str()?.as_bytes() == address).then_some(url) + }) + .ok_or_else(|| { + warn!( + address = ?String::from_utf8_lossy(address), + "auth data matches no configured relay and is neither a hostname nor a URL" + ); + PbsClientError::AuthDataMismatch + }) +} + +/// A lookup that times out keeps its blocking thread until the resolver +/// returns, and relays' own lookups share that pool, so few run at once +static DIAL_LOOKUPS: Semaphore = Semaphore::const_new(32); + +async fn resolve_dial_target( + url: &Url, + origin: &str, + lookup_timeout: Duration, +) -> Result, PbsClientError> { + let host_port = format!( + "{}:{}", + url.host_str().unwrap_or_default(), + url.port_or_known_default().unwrap_or_default() + ); + // An IP literal needs no lookup + if let Ok(addr) = host_port.parse::() { + vet_dial_addrs(origin, &[addr])?; + return Ok(vec![addr]); + } + let Ok(permit) = DIAL_LOOKUPS.try_acquire() else { + warn!(%origin, "too many dial lookups in flight, not dialing"); + return Err(PbsClientError::NoBuilderResponse); + }; + let lookup = tokio::task::spawn_blocking(move || { + // Held until the lookup returns, which a timeout does not stop + let _permit = permit; + host_port.to_socket_addrs().map(Iterator::collect::>) + }); + let addrs = match tokio::time::timeout(lookup_timeout, lookup).await { + Ok(Ok(Ok(addrs))) => addrs, + Ok(Ok(Err(err))) => { + warn!(%origin, %err, "dial target lookup failed, not dialing"); + return Err(PbsClientError::DialTargetBlocked); + } + Ok(Err(_)) => return Err(PbsClientError::Internal), + // Transient, unlike a failed lookup: the builder counts as not answering + Err(_) => { + warn!(%origin, ?lookup_timeout, "dial target lookup timed out, not dialing"); + return Err(PbsClientError::NoBuilderResponse); + } + }; + vet_dial_addrs(origin, &addrs)?; + Ok(addrs) +} + +/// Every address is checked, since the dial may connect to any of them +fn vet_dial_addrs(origin: &str, addrs: &[SocketAddr]) -> Result<(), PbsClientError> { + if dial_target_check_enabled() && addrs.iter().any(|addr| ip_is_disallowed(addr.ip())) { + warn!(%origin, "dial target resolves to a disallowed address, not dialing"); + return Err(PbsClientError::DialTargetBlocked); + } + Ok(()) +} + +/// A relay that dials `url` only at `addrs`: a system proxy would resolve the +/// host again, and a redirect target was never checked. +fn pinned_relay(url: Url, addrs: &[SocketAddr]) -> eyre::Result { + let mut client = + reqwest::Client::builder().no_proxy().redirect(reqwest::redirect::Policy::none()); + if let Some(domain) = url.domain() { + client = client.resolve_to_addrs(domain, addrs); + } + RelayClient::with_client_builder( + RelayConfig { + entry: RelayEntry { id: DIAL_RELAY_ID.to_string(), pubkey: DIAL_PUBKEY.clone(), url }, + id: None, + headers: None, + get_params: None, + get_header: GetHeaderTransport::Http, + enable_timing_games: false, + target_first_request_ms: None, + frequency_get_header_ms: None, + validator_registration_batch_size: None, + }, + client, + ) +} + +#[cfg(feature = "testing-flags")] +thread_local! { + static SKIP_DIAL_TARGET_CHECK: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Test-only: skips the dial target's address check on this thread, so a +/// test can dial a mock builder on loopback +#[cfg(feature = "testing-flags")] +pub fn set_skip_dial_target_check(skip: bool) { + SKIP_DIAL_TARGET_CHECK.with(|flag| flag.set(skip)); +} + +fn dial_target_check_enabled() -> bool { + #[cfg(feature = "testing-flags")] + { + !SKIP_DIAL_TARGET_CHECK.with(|flag| flag.get()) + } + #[cfg(not(feature = "testing-flags"))] + { + true + } +} + +/// Anything but public unicast: the special-purpose IPv4 ranges, also as the +/// IPv4 inside a v4-mapped, v4-compatible or NAT64 address, and IPv6 outside +/// 2000::/3 or in its documentation, Teredo and 6to4 ranges +fn ip_is_disallowed(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let [a, b, c, _] = v4.octets(); + v4.is_private() || + v4.is_loopback() || + v4.is_link_local() || + v4.is_multicast() || + v4.is_documentation() || + // std checks these only on nightly: 0.0.0.0/8, 100.64.0.0/10, + // 192.0.0.0/24, 198.18.0.0/15 and 240.0.0.0/4 + a == 0 || + (a == 100 && b & 0xc0 == 0x40) || + (a == 192 && b == 0 && c == 0) || + (a == 198 && b & 0xfe == 18) || + a >= 240 + } + IpAddr::V6(v6) => { + let s = v6.segments(); + let nat64 = s[..6] == [0x64, 0xff9b, 0, 0, 0, 0]; + let embedded = + if nat64 { Some(Ipv4Addr::from_bits(v6.to_bits() as u32)) } else { v6.to_ipv4() }; + match embedded { + // :: and ::1 land here as 0.0.0.0 and 0.0.0.1 + Some(v4) => ip_is_disallowed(IpAddr::V4(v4)), + None => { + s[0] & 0xe000 != 0x2000 || + (s[0] == 0x2001 && (s[1] == 0 || s[1] == 0x0db8)) || + s[0] == 0x2002 + } + } + } + } +} + +#[cfg(test)] +mod tests { + use axum::http::StatusCode; + use cb_common::pbs::decode_auth_data_url; + + use super::*; + + // Literal IPs keep the lookup off the network + #[tokio::test] + async fn dial_relay_targets() { + const MISMATCH: &str = "auth data does not match a configured builder"; + const BLOCKED: &str = "dial target does not resolve or resolves to a disallowed address"; + let cases: [(&[u8], Result<&str, &str>); 8] = [ + (b"http://1.1.1.1:8551", Ok("http://1.1.1.1:8551/")), + (b"1.1.1.1", Ok("https://1.1.1.1/")), + (b"1.1.1.1?ofac=1", Ok("https://1.1.1.1/")), + (b"[2606:4700:4700::1111]", Ok("https://[2606:4700:4700::1111]/")), + (b"10.0.0.1", Err(BLOCKED)), + (b"1.1.1.1:8551", Err(MISMATCH)), + (b"ftp://1.1.1.1", Err(MISMATCH)), + (&[0xde, 0xad], Err(MISMATCH)), + ]; + for (data, expected) in cases { + let address = auth_data_address(data); + let res = + dial_relay(decode_auth_data_url(address), address, Duration::from_secs(5)).await; + let case = String::from_utf8_lossy(data); + match (&res, expected) { + (Ok(relay), Ok(url)) => { + assert_eq!(relay.config.entry.url.as_str(), url, "{case}"); + assert_eq!(relay.id.as_str(), "dial", "{case}"); + } + (Err(err), Err(message)) => assert_eq!(err.to_string(), message, "{case}"), + _ => panic!("{case}: expected {expected:?}, got {:?}", res.map(|_| ())), + } + } + } + + // Holds every permit, so a hostname lookup in a unit test running beside it + // is refused + #[tokio::test] + async fn lookup_cap_refuses_hostnames_not_ip_literals() { + let _held = DIAL_LOOKUPS.try_acquire_many(32).unwrap(); + let dial = |address: &'static [u8]| dial_relay(None, address, Duration::from_secs(1)); + assert!(dial(b"1.1.1.1").await.is_ok()); + assert!(matches!(dial(b"builder.invalid").await, Err(PbsClientError::NoBuilderResponse))); + } + + #[test] + fn vet_dial_addrs_checks_every_address() { + let addr = |s: &str| s.parse::().unwrap(); + assert!(matches!( + vet_dial_addrs("https://builder.example.com", &[ + addr("1.1.1.1:443"), + addr("10.0.0.1:443") + ]), + Err(PbsClientError::DialTargetBlocked) + )); + } + + // A hostname target is dialed at the given address with no lookup + // (`.invalid` never resolves), and a redirect is not followed + #[tokio::test] + async fn pinned_relay_dials_the_given_address_and_refuses_redirects() -> eyre::Result<()> { + use axum::{Router, http::header::LOCATION, routing::post}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let builder = Router::new() + .route( + "/bid", + post(|| async { (StatusCode::TEMPORARY_REDIRECT, [(LOCATION, "/internal")]) }), + ) + .route("/internal", post(|| async { StatusCode::OK })); + tokio::spawn(async move { axum::serve(listener, builder).await }); + + let url = Url::parse(&format!("http://dial-target.invalid:{}/bid", addr.port()))?; + let res = pinned_relay(url.clone(), &[addr])?.client.post(url).send().await?; + assert_eq!(res.status(), StatusCode::TEMPORARY_REDIRECT); + Ok(()) + } + + #[test] + fn ip_is_disallowed_table() { + for (ip, disallowed) in [ + ("127.0.0.1", true), + ("0.0.0.0", true), + ("::1", true), + ("10.0.0.1", true), + ("169.254.1.1", true), + ("255.255.255.255", true), + ("100.64.0.1", true), + ("100.127.255.254", true), + ("100.63.0.1", false), + ("100.128.0.1", false), + ("fd12:3456::1", true), + ("fe80::1", true), + ("::ffff:127.0.0.1", true), + ("::10.0.0.1", true), + ("64:ff9b::a00:5", true), + ("::1.1.1.1", false), + ("64:ff9b::1.1.1.1", false), + ("::ffff:1.1.1.1", false), + ("0.1.2.3", true), + ("192.0.1.1", false), + ("198.19.255.255", true), + ("198.20.0.1", false), + ("224.0.0.1", true), + ("240.0.0.1", true), + ("192.0.0.192", true), + ("198.18.0.1", true), + ("203.0.113.1", true), + ("fec0::1", true), + ("ff02::1", true), + ("2001:db8::1", true), + ("64:ff9b:1::a00:1", true), + ("2002:a00:1::1", true), + ("2001::1", true), + ("1.1.1.1", false), + ("2606:4700:4700::1111", false), + ] { + assert_eq!(ip_is_disallowed(ip.parse().unwrap()), disallowed, "{ip}"); + } + } +} diff --git a/crates/pbs/src/error.rs b/crates/pbs/src/error.rs index 9b9aaf078..29f8dc4ca 100644 --- a/crates/pbs/src/error.rs +++ b/crates/pbs/src/error.rs @@ -20,12 +20,14 @@ struct ErrorResponse { pub enum PbsClientError { #[error("no response from relays")] NoResponse, - /// 500, not 502: 502 is in neither endpoint's builder-specs response set. - /// Legacy routes keep `NoResponse` -> 502. + /// 500, not 502: 502 is not in the preferences endpoint's builder-specs + /// response set. Legacy routes keep `NoResponse` -> 502. #[error("no builder accepted the submission")] NoBuilderResponse, #[error("auth data does not match a configured builder")] AuthDataMismatch, + #[error("dial target does not resolve or resolves to a disallowed address")] + DialTargetBlocked, #[error("missing or invalid timing headers")] MissingTimingHeader, #[error("auth slot does not match the request path")] @@ -50,6 +52,7 @@ impl PbsClientError { PbsClientError::NoResponse => StatusCode::BAD_GATEWAY, PbsClientError::NoBuilderResponse => StatusCode::INTERNAL_SERVER_ERROR, PbsClientError::AuthDataMismatch => StatusCode::BAD_REQUEST, + PbsClientError::DialTargetBlocked => StatusCode::BAD_REQUEST, PbsClientError::MissingTimingHeader => StatusCode::BAD_REQUEST, PbsClientError::AuthSlotMismatch => StatusCode::BAD_REQUEST, PbsClientError::BuilderRejected(code) => *code, @@ -73,6 +76,9 @@ impl IntoResponse for PbsClientError { PbsClientError::AuthDataMismatch => { "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder".to_string() } + PbsClientError::DialTargetBlocked => { + "Invalid SignedBuilderRequestAuth: the addressed builder's host does not resolve or resolves to a disallowed address".to_string() + } PbsClientError::MissingTimingHeader => { "Invalid request: Date-Milliseconds and X-Timeout-Ms headers are required".to_string() } diff --git a/crates/pbs/src/lib.rs b/crates/pbs/src/lib.rs index 8b4afdcfe..9c04224a8 100644 --- a/crates/pbs/src/lib.rs +++ b/crates/pbs/src/lib.rs @@ -1,5 +1,6 @@ mod api; mod constants; +mod dial; mod error; mod metrics; mod mev_boost; @@ -10,6 +11,8 @@ mod utils; pub use api::*; pub use constants::*; +#[cfg(feature = "testing-flags")] +pub use dial::set_skip_dial_target_check; pub use mev_boost::*; pub use service::PbsService; pub use state::{BuilderApiState, PbsState, PbsStateGuard}; diff --git a/crates/pbs/src/routes/builder_preferences.rs b/crates/pbs/src/routes/builder_preferences.rs index dc8596e72..cba99d0ef 100644 --- a/crates/pbs/src/routes/builder_preferences.rs +++ b/crates/pbs/src/routes/builder_preferences.rs @@ -65,17 +65,21 @@ pub async fn submit_builder_preferences( log_mux_selection(maybe_mux_id, relays.len(), ¶ms.proposer_pubkey); - let relay = resolve_addressed_relay(relays, request.auth.message.data.as_ref())?; + // Preferences are submitted an epoch ahead, so they share the registration + // timeout rather than the block-production one + let (relay, timeout_ms) = resolve_addressed_relay( + relays, + request.auth.message.data.as_ref(), + &req_headers, + pbs_config.timeout_register_validator_ms, + ) + .await?; let send_headers = epbs_base_send_headers(&req_headers)?; // SSZ on the relay hop let body = Bytes::from(request.as_ssz_bytes()); - // Preferences are submitted an epoch ahead, so they share the registration - // timeout rather than the block-production one - let timeout_ms = pbs_config.timeout_register_validator_ms; - let relay_id = relay.id.as_ref(); let sent = match relay.submit_builder_preferences_url(¶ms.proposer_pubkey) { Ok(url) => { diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index 1385ea3c5..34b40eb0c 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -16,7 +16,7 @@ use cb_common::{ wire::{ CONSENSUS_VERSION_HEADER, EncodingType, OUTBOUND_ACCEPT_SSZ_FIRST, decode_versioned_request_body, get_accept_types, get_user_agent, - parse_response_encoding_and_fork, safe_read_http_response, + parse_response_encoding_and_fork, read_chunked_body_with_max, safe_read_http_response, }, }; use reqwest::{ @@ -28,7 +28,9 @@ use tracing::{debug, error, info, warn}; use crate::{ PbsStateGuard, - constants::{GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE}, + constants::{ + GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, MAX_SIZE_DEFAULT, MAX_SIZE_GET_HEADER_RESPONSE, + }, error::PbsClientError, metrics::{RELAY_HEADER_VALUE, RELAY_LAST_SLOT}, state::{BuilderApiState, PbsState}, @@ -134,8 +136,6 @@ pub async fn get_execution_payload_bid( return Err(PbsClientError::AuthSlotMismatch); } - let relay = resolve_addressed_relay(relays, auth.message.data.as_ref())?; - // The beacon node's deadline, not timeout_get_header_ms, bounds the request let slot_ms = state.config.chain.slot_time_sec().saturating_mul(1000); let budget_ms = request_budget_ms(&req_headers, utcnow_ms(), slot_ms)?; @@ -153,6 +153,20 @@ pub async fn get_execution_payload_bid( return Ok(None); } + let (relay, max_timeout_ms) = match resolve_addressed_relay( + relays, + auth.message.data.as_ref(), + &req_headers, + max_timeout_ms, + ) + .await + { + // A dial with no time left, or refused for too many lookups, is a builder + // that did not answer + Err(PbsClientError::NoBuilderResponse) => return Ok(None), + resolved => resolved?, + }; + let mut send_headers = epbs_base_send_headers(&req_headers)?; // SSZ is smaller and reaches a beacon node that asks for SSZ unchanged @@ -178,7 +192,7 @@ pub async fn get_execution_payload_bid( Ok(None) } Err(err) => { - error!(%err, %relay_id); + error!(err = ?err, %relay_id); builder_rejection(&err).map_or(Ok(None), Err) } } @@ -238,17 +252,18 @@ async fn send_get_execution_payload_bid( let (res, request_latency) = send_to_relay(request, &relay, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG).await?; let code = res.status(); + if !code.is_success() { + // Read after the status check, so a body over the cap still reports the + // builder's status + let url = res.url().to_string(); + let error_msg = match read_chunked_body_with_max(res, MAX_SIZE_DEFAULT, &url).await { + Ok(body) => String::from_utf8_lossy(&body).into_owned(), + Err(err) => err.to_string(), + }; + return Err(PbsError::RelayResponse { error_msg, code: code.as_u16() }); + } - // Parse the negotiated Content-Type (and optional fork) before the body is - // consumed. Only successful responses carry a meaningful encoding; on - // non-success we fall through to safe_read_http_response's NonSuccess error, - // so these values are never consumed. - let (content_type, fork) = if code.is_success() { - parse_response_encoding_and_fork(res.headers(), code.as_u16())? - } else { - (EncodingType::Json, None) - }; - + let (content_type, fork) = parse_response_encoding_and_fork(res.headers(), code.as_u16())?; let response_bytes = safe_read_http_response(res, MAX_SIZE_GET_HEADER_RESPONSE).await?; if code == StatusCode::NO_CONTENT { debug!( @@ -262,9 +277,15 @@ async fn send_get_execution_payload_bid( } let bid = match content_type { - EncodingType::Json => serde_json::from_slice(&response_bytes).map_err(|err| { - PbsError::JsonDecode { err, raw: String::from_utf8_lossy(&response_bytes).into_owned() } - })?, + EncodingType::Json => { + serde_json::from_slice(&response_bytes).map_err(|err| PbsError::JsonDecode { + err, + raw: String::from_utf8_lossy( + &response_bytes[..response_bytes.len().min(MAX_SIZE_DEFAULT)], + ) + .into_owned(), + })? + } EncodingType::Ssz => { // SSZ requires the fork from Eth-Consensus-Version; its absence is a // relay protocol violation. diff --git a/crates/pbs/src/routes/submit_signed_beacon_block.rs b/crates/pbs/src/routes/submit_signed_beacon_block.rs index 75bb35542..f8471bade 100644 --- a/crates/pbs/src/routes/submit_signed_beacon_block.rs +++ b/crates/pbs/src/routes/submit_signed_beacon_block.rs @@ -4,7 +4,7 @@ use cb_common::wire::{ require_consensus_version_header, }; use reqwest::StatusCode; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use crate::{ PbsStateGuard, @@ -35,7 +35,8 @@ pub async fn handle_submit_signed_beacon_block( /// Implements https://ethereum.github.io/builder-specs/?urls.primaryName=dev#/Builder/submitSignedBeaconBlock /// Forwards the block bytes, undecoded, to every configured builder, since CB -/// keeps no auction state to know which bid won. Returns 202 if one accepts +/// keeps no auction state to know which bid won. Returns 202 whatever they +/// answer: the beacon node gossips the block anyway pub async fn submit_signed_beacon_block( body: Bytes, req_headers: HeaderMap, @@ -86,11 +87,10 @@ pub async fn submit_signed_beacon_block( } }) .count(); - - // Only the winner accepts, so one 202 across the broadcast is success if accepted == 0 { - return Err(PbsClientError::NoBuilderResponse); + warn!(addressed = relays.len(), "no builder accepted the signed beacon block"); + } else { + info!(accepted, addressed = relays.len(), "signed beacon block accepted"); } - info!(accepted, addressed = relays.len(), "signed beacon block submitted"); Ok(()) } diff --git a/crates/pbs/src/utils.rs b/crates/pbs/src/utils.rs index 77c03765d..04a8dad51 100644 --- a/crates/pbs/src/utils.rs +++ b/crates/pbs/src/utils.rs @@ -6,7 +6,7 @@ use std::{ use alloy::primitives::utils::{ParseUnits, Unit}; use axum::body::Bytes; use cb_common::{ - pbs::{RelayClient, error::PbsError}, + pbs::{HEADER_VERSION_KEY, RelayClient, decode_auth_data_url, error::PbsError}, types::BlsPublicKey, wire::{ CONSENSUS_VERSION_HEADER, EncodingType, get_user_agent_with_version, @@ -23,6 +23,7 @@ use url::Url; use crate::{ constants::{MAX_SIZE_DEFAULT, TIMEOUT_ERROR_CODE_STR}, + dial::{auth_data_address, dial_relay}, error::PbsClientError, metrics::{BEACON_NODE_STATUS, RELAY_LATENCY, RELAY_STATUS_CODE}, }; @@ -188,25 +189,51 @@ pub fn check_gas_limit(gas_limit: u64, parent_gas_limit: u64) -> bool { true } -/// The relay an ePBS request addresses: the first whose URL hostname equals -/// `auth_data`. No match is a 400. -pub(crate) fn resolve_addressed_relay( +/// The first configured relay that the address in `auth_data` names, by +/// hostname or, for a URL, by origin; otherwise a relay that dials it. Also +/// returns what `timeout_ms` leaves after setting up that dial. +pub(crate) async fn resolve_addressed_relay( relays: &[RelayClient], auth_data: &[u8], -) -> Result { - let addressed = relays.iter().find(|relay| { - relay.config.entry.url.host_str().is_some_and(|host| host.as_bytes() == auth_data) + req_headers: &HeaderMap, + timeout_ms: u64, +) -> Result<(RelayClient, u64), PbsClientError> { + let address = auth_data_address(auth_data); + let by_host = relays.iter().find(|relay| { + relay.config.entry.url.host_str().is_some_and(|host| host.as_bytes() == address) }); - match addressed { - Some(relay) => Ok(relay.clone()), - None => { - warn!( - auth_data = %String::from_utf8_lossy(auth_data), - "auth data matches no configured relay" - ); - Err(PbsClientError::AuthDataMismatch) - } + if let Some(relay) = by_host { + return Ok((relay.clone(), timeout_ms)); + } + let data_url = decode_auth_data_url(address); + let by_origin = data_url.as_ref().and_then(|data_url| { + relays.iter().find(|relay| { + let url = &relay.config.entry.url; + url.scheme() == data_url.scheme() && + url.host() == data_url.host() && + url.port_or_known_default() == data_url.port_or_known_default() + }) + }); + if let Some(relay) = by_origin { + return Ok((relay.clone(), timeout_ms)); + } + // Every Commit-Boost dial carries this header, so a request dialed back into + // a Commit-Boost, this one included, goes no further + if req_headers.contains_key(HEADER_VERSION_KEY) { + warn!( + auth_data = ?String::from_utf8_lossy(auth_data), + "auth data matches no configured relay and the request came from a Commit-Boost, not dialing on" + ); + return Err(PbsClientError::AuthDataMismatch); + } + let started = Instant::now(); + let relay = dial_relay(data_url, address, Duration::from_millis(timeout_ms)).await?; + let left_ms = timeout_ms.saturating_sub(started.elapsed().as_millis() as u64); + // A request with no time left would still reach the builder + if left_ms == 0 { + return Err(PbsClientError::NoBuilderResponse); } + Ok((relay, left_ms)) } #[cfg(test)] @@ -239,24 +266,38 @@ mod tests { RelayClient::new(config).unwrap() } - #[test] - fn resolve_relay_by_hostname() { + #[tokio::test] + async fn resolve_relay_by_auth_data() { let relays = vec![ test_relay("https://0xdeadbeef@builder-a.example.com"), test_relay("https://builder-b.example.com:8443/eth"), test_relay("http://[::1]:18550"), ]; - let host = |data: &[u8]| -> Option { - resolve_addressed_relay(&relays, data) + // From a Commit-Boost, so a miss is an error, not a dial + let mut from_cb = HeaderMap::new(); + from_cb.insert(HEADER_VERSION_KEY, reqwest::header::HeaderValue::from_static("test")); + let host = async |data: &[u8]| -> Option { + resolve_addressed_relay(&relays, data, &from_cb, 0) + .await .ok() - .map(|relay| relay.config.entry.url.host_str().unwrap().to_string()) + .map(|(relay, _)| relay.config.entry.url.host_str().unwrap().to_string()) }; - assert_eq!(host(b"builder-a.example.com").as_deref(), Some("builder-a.example.com")); - assert_eq!(host(b"builder-b.example.com").as_deref(), Some("builder-b.example.com")); - assert_eq!(host(b"[::1]").as_deref(), Some("[::1]")); - assert!(host(b"Builder-A.example.com").is_none()); - assert!(host(b"builder-a.example.com:443").is_none()); - // a URL is not a hostname - assert!(host(b"https://builder-a.example.com").is_none()); + assert_eq!(host(b"builder-a.example.com").await.as_deref(), Some("builder-a.example.com")); + assert_eq!(host(b"builder-b.example.com").await.as_deref(), Some("builder-b.example.com")); + assert_eq!(host(b"[::1]").await.as_deref(), Some("[::1]")); + assert!(host(b"Builder-A.example.com").await.is_none()); + assert!(host(b"builder-a.example.com:443").await.is_none()); + // A URL matches on scheme, host and port + assert_eq!( + host(b"https://builder-a.example.com:443/eth").await.as_deref(), + Some("builder-a.example.com") + ); + assert!(host(b"http://builder-a.example.com").await.is_none()); + assert!(host(b"https://builder-b.example.com").await.is_none()); + // Parameters after `?` are for the builder and do not affect routing + assert_eq!( + host(b"builder-a.example.com?ofac=1").await.as_deref(), + Some("builder-a.example.com") + ); } } diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 27cc0f75d..df2c577d3 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -33,7 +33,7 @@ A `get_header = "stream"` relay is asked for ePBS bids over plain HTTP. From the fork, each validator key has a builder config: a list of entries, each a `url` and an `auth_data`. For every entry the beacon node sends a bid request to the entry's `url`, carrying its `auth_data` in a `SignedBuilderRequestAuth` that the validator signs. `auth_data` tells a builder that a request was meant for it, so a request signed for one builder is rejected by another. Unless the proposer and builder agree on another value, it is the builder's hostname. -To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and its `auth_data` is the hostname of the builder it stands for. Commit-Boost reads the `auth_data` of each request, finds the relay entry with that hostname and forwards the request there. Builder preferences are routed the same way. The signed block goes to every builder, since only the one whose bid won accepts it. +To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and its `auth_data` is the hostname of the builder it stands for. Commit-Boost reads the `auth_data` of each request, finds the relay entry with that hostname and forwards the request there. If no relay entry has it, Commit-Boost dials the builder the `auth_data` names ([builders outside your config](#builders-outside-your-config)). Builder preferences are routed the same way. The signed block goes to every builder in your config, since only the one whose bid won accepts it. ## Setup @@ -43,7 +43,7 @@ List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mu ### 2. Point each validator key at Commit-Boost {#validator-builder-config} -Write each key's builder config through its validator client's keymanager API, which must implement the builder config endpoint ([keymanager-APIs #88](https://github.com/ethereum/keymanager-APIs/pull/88)). Add one entry per builder: `url` is Commit-Boost's URL, and `auth_data` is the hex of the builder's hostname. A key without builder config sends the default auth data of Commit-Boost's own URL, which matches no builder, so every bid request gets `400`. +Write each key's builder config through its validator client's keymanager API, which must implement the builder config endpoint ([keymanager-APIs #88](https://github.com/ethereum/keymanager-APIs/pull/88)). Add one entry per builder: `url` is Commit-Boost's URL, and `auth_data` is the hex of the builder's hostname. A key without builder config gets no bids through Commit-Boost ([builders outside your config](#builders-outside-your-config)). With the relay entry `url = "https://0xa1ce...@builder-a.example.com"`, Commit-Boost listening at `http://cb.example.com:18550`, the keymanager API at `$KEYMANAGER_URL`, its token in `$TOKEN` and the validator key in `$PUBKEY`: @@ -72,7 +72,17 @@ The call is `POST /eth/v1/validator/{pubkey}/builder_config`, authenticated with ## Routing by auth data -Commit-Boost sends a bid or preferences request to the first builder serving the proposer's key (the relays of its `[[mux]]`, otherwise `[[relays]]`) whose URL hostname equals the request's `auth_data`, byte for byte. That is the [builder specs](https://github.com/ethereum/builder-specs/blob/main/specs/gloas/validator.md#default-auth-data) default auth data: the lowercase hostname, without scheme, port or path, with an IPv6 address in brackets, such as `[::1]`. Builders on one host share it, so only the first is asked. Commit-Boost routes by hostname only, so a value agreed with a builder works through it only if it is that hostname. When no builder matches, the request gets `400`. +Commit-Boost sends a bid or preferences request to the first builder serving the proposer's key (the relays of its `[[mux]]`, otherwise `[[relays]]`) whose URL hostname equals the request's `auth_data`, byte for byte. That is the [builder specs](https://github.com/ethereum/builder-specs/blob/main/specs/gloas/validator.md#default-auth-data) default auth data: the lowercase hostname, without scheme, port or path, with an IPv6 address in brackets, such as `[::1]`. Builders on one host share it, so only the first is asked. + +`auth_data` can also be an `http` or `https` URL, a form some builders agree out of band. It then matches the first builder whose `url` has the same scheme, host and port; the path and the pubkey in the relay URL do not count. Any other value agreed with a builder does not route through Commit-Boost. When no builder matches, Commit-Boost [dials the builder](#builders-outside-your-config) the `auth_data` names, and answers `400` when the `auth_data` is neither a hostname nor an `http(s)` URL. + +Either form may end in `?` and form-encoded parameters for the builder, such as `builder-a.example.com?ofac=1`. Commit-Boost routes on the part before `?`. The parameters reach the builder inside the signed auth, so set them only for a builder that accepts them; one that does not answers `400`. + +## Builders outside your config {#builders-outside-your-config} + +A key's builder config may name a builder you have not added as a relay entry. Commit-Boost dials that builder anyway: a hostname at `https://` on the default port, and a URL at its scheme, host and port. It answers `400` instead when the host does not resolve, or resolves to any address that is not public unicast, such as a loopback, private, link-local or `100.64.0.0/10` address, so a builder on your own network has to be a relay entry. These requests connect directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`, follow no redirects and do not carry your relay `headers`. A builder reached this way gets the signed block over gossip, not from Commit-Boost. Neither `[[relays]]` nor a mux's relays limit which builders a key can reach this way, and anyone who can reach Commit-Boost's port can make it dial a public host, so keep that port private. Commit-Boost looks up at most 32 of these hosts at once and answers further requests as if the builder had not responded. + +A key without builder config sends Commit-Boost's own hostname as its `auth_data`, so it gets no bids. Commit-Boost answers `400` when that hostname resolves to a loopback or private address, as it usually does, and otherwise dials `https://` on port 443, where Commit-Boost itself does not listen. Commit-Boost never dials for a request that came from another Commit-Boost, so chained Commit-Boosts route only through relay entries. ## Timing @@ -88,8 +98,8 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Question | Series | |---|---| -| What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences and the signed block, `4xx` for a rejected request, `500` when no builder accepted preferences or the signed block | -| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure) | +| What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences, `202` for the signed block whatever the builders answer, `4xx` for a rejected request, `500` when no builder accepted the preferences | +| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"` | | How fast are builders? | `cb_pbs_relay_latency`, by `relay_id` | | What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from each bid a builder serves | @@ -97,7 +107,9 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Symptom | Cause | Fix | |---|---|---| -| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data matches no builder serving the key. Usually the key has no builder config, so its auth data is Commit-Boost's own hostname | Write the key's [builder config](#validator-builder-config), or add the builder it names as a relay entry for the key | -| A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than its hostname | Have the builder accept its hostname, the only auth data Commit-Boost [routes by](#routing-by-auth-data) | +| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a hostname nor an `http(s)` URL, or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | +| Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config). A key without builder config sends Commit-Boost's own hostname, which usually does | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | +| A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | | The signed block gets `415` | The beacon node sends it as JSON | Configure the beacon node to send SSZ | +| Commit-Boost logs "no builder accepted the signed beacon block" | The winning bid came from a builder outside your config, which gets the block over gossip, or your builders did not accept it. The beacon node gossips the block either way | If the bid came from a configured builder, check its answer in `cb_pbs_relay_status_code_total{endpoint="submit_signed_beacon_block"}` | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/tests/Cargo.toml b/tests/Cargo.toml index c8503378c..772066868 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -8,7 +8,7 @@ version.workspace = true alloy.workspace = true axum.workspace = true cb-common.workspace = true -cb-pbs.workspace = true +cb-pbs = { workspace = true, features = ["testing-flags"] } cb-signer.workspace = true eyre.workspace = true ethereum_ssz.workspace = true diff --git a/tests/src/mock_relay.rs b/tests/src/mock_relay.rs index a05dcbeab..3d679d6fb 100644 --- a/tests/src/mock_relay.rs +++ b/tests/src/mock_relay.rs @@ -127,6 +127,11 @@ pub struct MockRelayState { trustless_bid_gwei: u64, // default 10 /// When true, an SSZ bid is served without `Eth-Consensus-Version` epbs_omit_consensus_version: bool, + /// When true, a JSON bid is pretty-printed, so it differs from the compact + /// form a re-encode produces + pretty_json_bid: bool, + /// The body of the last bid served + served_bid: RwLock>, } impl MockRelayState { @@ -184,6 +189,9 @@ impl MockRelayState { pub fn received_auth_data(&self) -> Option> { self.received_auth.read().unwrap().as_ref().map(|a| a.message.data.to_vec()) } + pub fn served_bid(&self) -> Option { + self.served_bid.read().unwrap().clone() + } pub fn large_body(&self) -> bool { self.large_body } @@ -246,6 +254,8 @@ impl MockRelayState { api_keys_seen: RwLock::new(HashMap::new()), trustless_bid_gwei: 10, epbs_omit_consensus_version: false, + pretty_json_bid: false, + served_bid: RwLock::new(None), supported_content_types: Arc::new( [EncodingType::Json, EncodingType::Ssz].iter().cloned().collect(), ), @@ -323,6 +333,10 @@ impl MockRelayState { pub fn with_epbs_omit_consensus_version(self) -> Self { Self { epbs_omit_consensus_version: true, ..self } } + + pub fn with_pretty_json_bid(self) -> Self { + Self { pretty_json_bid: true, ..self } + } } pub fn mock_relay_app_router(state: Arc) -> Router { @@ -419,6 +433,10 @@ async fn handle_get_execution_payload_bid( // fail on the bid endpoint (its bid is then dropped by PBS). The request was // already counted above. if let Some(status) = *state.response_override.read().unwrap() { + // An error body over PBS's 1 KiB read cap + if state.large_body() { + return (status, "x".repeat(2048)).into_response(); + } return status.into_response(); } @@ -477,9 +495,15 @@ async fn handle_get_execution_payload_bid( data, metadata: Default::default(), }; - serde_json::to_vec(&versioned).unwrap() + if state.pretty_json_bid { + serde_json::to_vec_pretty(&versioned).unwrap() + } else { + serde_json::to_vec(&versioned).unwrap() + } } }; + let response_body = axum::body::Bytes::from(response_body); + *state.served_bid.write().unwrap() = Some(response_body.clone()); let mut response = (StatusCode::OK, response_body).into_response(); // A real builder tags the 200 with the fork so a client can decode the diff --git a/tests/src/mock_validator.rs b/tests/src/mock_validator.rs index 2059a2456..c433d322f 100644 --- a/tests/src/mock_validator.rs +++ b/tests/src/mock_validator.rs @@ -30,7 +30,10 @@ impl MockValidator { let pubkey = bls_pubkey_from_hex( "0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae", )?; - Ok(Self { comm_boost: generate_mock_relay(port, pubkey)? }) + let mut comm_boost = generate_mock_relay(port, pubkey)?; + // Like a beacon node, without X-CommitBoost-Version, which stops a dial + comm_boost.client = reqwest::Client::new(); + Ok(Self { comm_boost }) } pub async fn do_get_header( diff --git a/tests/tests/pbs_get_execution_payload_bid.rs b/tests/tests/pbs_get_execution_payload_bid.rs index 0cc4fb2fa..561d11300 100644 --- a/tests/tests/pbs_get_execution_payload_bid.rs +++ b/tests/tests/pbs_get_execution_payload_bid.rs @@ -4,7 +4,7 @@ use alloy::primitives::{B256, U256}; use cb_common::{ config::RuntimeMuxConfig, pbs::{ - GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, + GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, SignedExecutionPayloadBid, }, signer::random_secret, @@ -121,9 +121,8 @@ async fn test_get_execution_payload_bid_shared_auth_data_asks_the_first() -> Res } /// The spec default auth data, the builder's hostname, routes to the relay on -/// that host, and the auth reaches the relay unchanged. A hostname no relay -/// has is a 400 with the builder's data-mismatch message, and no relay is -/// contacted. +/// that host, and the auth reaches the relay unchanged. An unconfigured +/// `localhost` resolves to loopback, so it is refused with 400 and not dialed. #[tokio::test] async fn test_get_execution_payload_bid_demux_by_hostname() -> Result<()> { let chain = Chain::Hoodi; @@ -148,11 +147,40 @@ async fn test_get_execution_payload_bid_demux_by_hostname() -> Result<()> { assert_eq!(body["code"], 400); assert_eq!( body["message"], - "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder" + "Invalid SignedBuilderRequestAuth: the addressed builder's host does not resolve or resolves to a disallowed address" ); Ok(()) } +/// Auth data that matches no relay entry is dialed, and the builder gets the +/// auth, `?` parameters included, unchanged. A request from a Commit-Boost is +/// not dialed, though a configured relay still serves it. +#[tokio::test] +async fn test_get_execution_payload_bid_dial() -> Result<()> { + // The mock builder listens on an address the dial check refuses + cb_pbs::set_skip_dial_target_check(true); + let chain = Chain::Hoodi; + let (mut mock_validator, cfg_state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let (dial_state, dial_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let dial_auth_data = format!("http://0.0.0.0:{dial_port}/?ofac=1").into_bytes(); + + let res = get_json_bid(&mock_validator, &opaque_auth(&dial_auth_data, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(dial_state.received_auth_data(), Some(dial_auth_data.clone())); + + // RelayClient::new's client sends the Commit-Boost version header + mock_validator.comm_boost = + RelayClient::new(mock_validator.comm_boost.config.as_ref().clone())?; + let res = get_json_bid(&mock_validator, &opaque_auth(&dial_auth_data, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(dial_state.received_execution_payload_bid(), 1); + assert_eq!(cfg_state.received_execution_payload_bid(), 1); + Ok(()) +} + /// A key listed in a `[[mux]]` is served by that mux's relays: its bid request /// reaches the mux relay and not the default `[[relays]]` one. #[tokio::test] @@ -274,7 +302,8 @@ async fn test_get_execution_payload_bid_rejected_before_relays() -> Result<()> { } /// A deadline that has already passed means there is no time to serve the -/// request: CB returns 204 rather than calling a relay it cannot beat. +/// request: CB returns 204 rather than calling a relay it cannot beat, or +/// looking up a dial target (`localhost` would be refused with 400). #[tokio::test] async fn test_get_execution_payload_bid_expired_deadline_204() -> Result<()> { let (mock_validator, mock_state) = @@ -285,18 +314,20 @@ async fn test_get_execution_payload_bid_expired_deadline_204() -> Result<()> { &B256::ZERO, &random_secret().public_key(), )?; - let res = mock_validator - .comm_boost - .client - .post(url) - .header(HEADER_START_TIME_UNIX_MS, utcnow_ms() - 5_000) - .header(HEADER_TIMEOUT_MS, 1_000u64) - .header("Eth-Consensus-Version", "gloas") - .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) - .body(opaque_auth(TEST_AUTH_DATA, TEST_SLOT).as_ssz_bytes()) - .send() - .await?; - assert_eq!(res.status(), StatusCode::NO_CONTENT); + for auth_data in [TEST_AUTH_DATA, b"localhost"] { + let res = mock_validator + .comm_boost + .client + .post(url.clone()) + .header(HEADER_START_TIME_UNIX_MS, utcnow_ms() - 5_000) + .header(HEADER_TIMEOUT_MS, 1_000u64) + .header("Eth-Consensus-Version", "gloas") + .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .body(opaque_auth(auth_data, TEST_SLOT).as_ssz_bytes()) + .send() + .await?; + assert_eq!(res.status(), StatusCode::NO_CONTENT); + } assert_eq!(mock_state.received_execution_payload_bid(), 0, "no relay call past the deadline"); Ok(()) } @@ -384,6 +415,22 @@ async fn test_get_execution_payload_bid_response_encoding() -> Result<()> { Ok(()) } +/// A beacon node that asks for the relay's encoding gets the relay's body byte +/// for byte; a re-encode would drop the pretty-printing +#[tokio::test] +async fn test_get_execution_payload_bid_json_passthrough() -> Result<()> { + let chain = Chain::Hoodi; + let relay = MockRelayState::new(chain, random_secret()) + .with_json_only_response() + .with_pretty_json_bid(); + let (mock_validator, states) = setup_relays(chain, vec![relay]).await?; + + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(Some(res.bytes().await?), states[0].served_bid()); + Ok(()) +} + /// An SSZ bid without `Eth-Consensus-Version` cannot be forwarded, since CB's /// 200 must name the fork: that relay contributes no bid, so the request is a /// 204. The relay is still contacted, so the 204 proves the drop. @@ -421,6 +468,21 @@ async fn test_get_execution_payload_bid_relay_status() -> Result<()> { Ok(()) } +/// An error body over PBS's 1 KiB read cap still reports the builder's status. +#[tokio::test] +async fn test_get_execution_payload_bid_large_error_body() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, states) = + setup_relays(chain, vec![MockRelayState::new(chain, random_secret()).with_large_body()]) + .await?; + for status in [StatusCode::BAD_REQUEST, StatusCode::UNAUTHORIZED] { + states[0].set_response_override(status); + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), status); + } + Ok(()) +} + /// A relay slower than the proposer's `X-Timeout-Ms` is dropped, so the request /// degrades to 204 rather than waiting the relay out. #[tokio::test] @@ -482,3 +544,22 @@ async fn test_get_execution_payload_bid_relay_timing_headers() -> Result<()> { assert!(0 < timeout_ms && timeout_ms <= 11_900, "relay saw X-Timeout-Ms {timeout_ms}"); Ok(()) } + +/// Auth data naming Commit-Boost's own URL is dialed once, and that hop, a +/// request from a Commit-Boost, is not dialed on: the builder's 400 +#[tokio::test] +async fn test_get_execution_payload_bid_dial_self_400() -> Result<()> { + // As if Commit-Boost's own address were public + cb_pbs::set_skip_dial_target_check(true); + let (mock_validator, cfg_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + let own = &mock_validator.comm_boost.config.entry.url; + let own = format!("http://{}:{}/", own.host_str().unwrap(), own.port().unwrap()); + + let res = get_json_bid(&mock_validator, &opaque_auth(own.as_bytes(), TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(body["message"], "The addressed builder rejected the request with status 400"); + assert_eq!(cfg_state.received_execution_payload_bid(), 0); + Ok(()) +} diff --git a/tests/tests/pbs_submit_builder_preferences.rs b/tests/tests/pbs_submit_builder_preferences.rs index b1079e7e4..302fece01 100644 --- a/tests/tests/pbs_submit_builder_preferences.rs +++ b/tests/tests/pbs_submit_builder_preferences.rs @@ -9,7 +9,7 @@ use cb_tests::{ mock_relay::MockRelayState, utils::{ TEST_AUTH_DATA, generate_mock_relay, opaque_auth, setup_relay, setup_relays, - setup_relays_on_hosts, + setup_relays_on_hosts, spawn_mock_relay, }, }; use eyre::Result; @@ -173,10 +173,9 @@ async fn test_submit_builder_preferences_past_slot_forwarded() -> Result<()> { Ok(()) } -/// Auth data matching no relay's hostname is a 400 with the builder's -/// data-mismatch message: unmatched means CB has no builder to proxy to, and -/// proposer-private preferences must never broadcast to builders the proposer -/// did not address. +/// Auth data that is neither a hostname nor a URL is a 400 with the builder's +/// data-mismatch message: CB has no builder to proxy to, and proposer-private +/// preferences must never broadcast to builders the proposer did not address. #[tokio::test] async fn test_submit_builder_preferences_unmatched_auth_data_400() -> Result<()> { let chain = Chain::Hoodi; @@ -270,3 +269,23 @@ async fn test_submit_builder_preferences_two_relays_addressed_one_only() -> Resu assert_eq!(states[1].received_builder_preferences(), 1, "the addressed builder is asked"); Ok(()) } + +/// Preferences whose auth data names a builder outside the config are sent +/// to it by a dial and accepted with its 202 +#[tokio::test] +async fn test_submit_builder_preferences_dial() -> Result<()> { + // The mock builder listens on an address the dial check refuses + cb_pbs::set_skip_dial_target_check(true); + let chain = Chain::Hoodi; + let (mock_validator, _) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let (dial_state, dial_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + + let auth = opaque_auth(format!("http://0.0.0.0:{dial_port}/").as_bytes(), TEST_SLOT); + let request = preferences(auth, TEST_MAX_EXECUTION_PAYMENT); + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + assert_eq!(res.status(), StatusCode::ACCEPTED); + assert_eq!(dial_state.received_builder_preferences(), 1); + Ok(()) +} diff --git a/tests/tests/pbs_submit_signed_beacon_block.rs b/tests/tests/pbs_submit_signed_beacon_block.rs index a5e474a16..2e05c2e3e 100644 --- a/tests/tests/pbs_submit_signed_beacon_block.rs +++ b/tests/tests/pbs_submit_signed_beacon_block.rs @@ -56,18 +56,16 @@ async fn test_submit_signed_beacon_block_broadcasts_to_all_relays() -> Result<() Ok(()) } -/// The block is broadcast; if every builder rejects it, PBS maps the all-reject -/// outcome to a 500 (no builder accepted). +/// Every builder rejecting the block is still a 202: the beacon node gossips +/// the block anyway, so a builder's answer is not a failure. Each is asked. #[tokio::test] -async fn test_submit_signed_beacon_block_broadcast_all_reject_500() -> Result<()> { +async fn test_submit_signed_beacon_block_broadcast_all_reject_202() -> Result<()> { let chain = Chain::Hoodi; let (mock_validator, states) = setup_relays(chain, vec![ MockRelayState::new(chain, random_secret()), MockRelayState::new(chain, random_secret()), ]) .await?; - - // Make every builder reject the broadcast for state in &states { state.set_response_override(StatusCode::INTERNAL_SERVER_ERROR); } @@ -75,36 +73,7 @@ async fn test_submit_signed_beacon_block_broadcast_all_reject_500() -> Result<() let block = gloas_block(TEST_SLOT); let res = mock_validator.do_submit_signed_beacon_block(&block, EncodingType::Ssz).await?; - assert_eq!(res.status(), StatusCode::INTERNAL_SERVER_ERROR); - assert_eq!(states[0].received_signed_beacon_block(), 1, "every builder is asked"); - assert_eq!(states[1].received_signed_beacon_block(), 1, "every builder is asked"); - Ok(()) -} - -/// A broadcast where one builder accepts and the other rejects is still a 202: -/// one acceptance across the broadcast is success. This is the core of the -/// stateless broadcast model, distinct from the all-accept and all-reject -/// extremes the other tests cover. -#[tokio::test] -async fn test_submit_signed_beacon_block_broadcast_one_accepts_202() -> Result<()> { - let chain = Chain::Hoodi; - let (mock_validator, states) = setup_relays(chain, vec![ - MockRelayState::new(chain, random_secret()), - MockRelayState::new(chain, random_secret()), - ]) - .await?; - - // Only the second builder accepts; the first rejects. PBS must still 202. - states[0].set_response_override(StatusCode::INTERNAL_SERVER_ERROR); - - let block = gloas_block(TEST_SLOT); - let res = mock_validator.do_submit_signed_beacon_block(&block, EncodingType::Ssz).await?; - - assert_eq!( - res.status(), - StatusCode::ACCEPTED, - "one accepting builder makes the broadcast a success" - ); + assert_eq!(res.status(), StatusCode::ACCEPTED); assert_eq!(states[0].received_signed_beacon_block(), 1, "every builder is asked"); assert_eq!(states[1].received_signed_beacon_block(), 1, "every builder is asked"); Ok(()) From 70acaee0683358b6bb87d46dc9de98b141d18136 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Wed, 7 Oct 2026 11:30:27 -0700 Subject: [PATCH 13/24] normalize gloas version header --- crates/common/src/wire.rs | 2 ++ .../pbs/src/routes/execution_payload_bid.rs | 19 ++++----------- crates/pbs/src/utils.rs | 10 ++++---- tests/src/mock_relay.rs | 8 +++++++ tests/tests/pbs_get_execution_payload_bid.rs | 24 +++++++++++++++++++ 5 files changed, 43 insertions(+), 20 deletions(-) diff --git a/crates/common/src/wire.rs b/crates/common/src/wire.rs index 48f3be15a..4355c3da6 100644 --- a/crates/common/src/wire.rs +++ b/crates/common/src/wire.rs @@ -280,6 +280,8 @@ fn essence_encoding(mt: &MediaType) -> Option { pub static OUTBOUND_ACCEPT_SSZ_FIRST: HeaderValue = HeaderValue::from_static("application/octet-stream;q=1.0,application/json;q=0.9"); +pub static GLOAS_CONSENSUS_VERSION: HeaderValue = HeaderValue::from_static("gloas"); + pub fn get_content_type(req_headers: &HeaderMap) -> EncodingType { EncodingType::from_str( req_headers diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index 1385ea3c5..9e1b9e369 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -14,7 +14,7 @@ use cb_common::{ }, utils::{ms_into_slot, utcnow_ms}, wire::{ - CONSENSUS_VERSION_HEADER, EncodingType, OUTBOUND_ACCEPT_SSZ_FIRST, + CONSENSUS_VERSION_HEADER, EncodingType, GLOAS_CONSENSUS_VERSION, OUTBOUND_ACCEPT_SSZ_FIRST, decode_versioned_request_body, get_accept_types, get_user_agent, parse_response_encoding_and_fork, safe_read_http_response, }, @@ -62,14 +62,10 @@ pub async fn handle_get_execution_payload_bid( info!(ua, ms_into_slot, "new request"); - let version = req_headers.get(CONSENSUS_VERSION_HEADER).cloned(); match get_execution_payload_bid(params, auth, req_headers, state).await { - Ok(Some(bid)) => Ok(encode_bid_response( - bid, - response_encoding, - version, - GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, - )), + Ok(Some(bid)) => { + Ok(encode_bid_response(bid, response_encoding, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG)) + } Ok(None) => { info!("no bid for slot"); record_beacon_status("204", GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG); @@ -87,12 +83,9 @@ pub(crate) struct RelayBid { encoding: EncodingType, } -/// `version` is the beacon node's own `Eth-Consensus-Version`: the 200 names -/// the fork it asked for fn encode_bid_response( RelayBid { bid, body, encoding }: RelayBid, response_encoding: EncodingType, - version: Option, endpoint: &str, ) -> Response { let message = &bid.data.message; @@ -111,9 +104,7 @@ fn encode_bid_response( EncodingType::Ssz => (content_type, bid.data.as_ssz_bytes()).into_response(), EncodingType::Json => axum::Json(bid).into_response(), }; - if let Some(version) = version { - res.headers_mut().insert(CONSENSUS_VERSION_HEADER, version); - } + res.headers_mut().insert(CONSENSUS_VERSION_HEADER, GLOAS_CONSENSUS_VERSION.clone()); res } diff --git a/crates/pbs/src/utils.rs b/crates/pbs/src/utils.rs index 77c03765d..023dc6bb0 100644 --- a/crates/pbs/src/utils.rs +++ b/crates/pbs/src/utils.rs @@ -9,8 +9,8 @@ use cb_common::{ pbs::{RelayClient, error::PbsError}, types::BlsPublicKey, wire::{ - CONSENSUS_VERSION_HEADER, EncodingType, get_user_agent_with_version, - read_chunked_body_with_max, + CONSENSUS_VERSION_HEADER, EncodingType, GLOAS_CONSENSUS_VERSION, + get_user_agent_with_version, read_chunked_body_with_max, }, }; use futures::future::join_all; @@ -153,16 +153,14 @@ pub(crate) fn builder_rejection(err: &PbsError) -> Option { } /// Headers every ePBS relay request carries: the versioned `User-Agent`, and -/// the beacon node's `Eth-Consensus-Version`, which the route has validated +/// `Eth-Consensus-Version`, which the route has validated as Gloas pub(crate) fn epbs_base_send_headers(req_headers: &HeaderMap) -> Result { let mut headers = HeaderMap::new(); headers.insert( USER_AGENT, get_user_agent_with_version(req_headers).map_err(|_| PbsClientError::Internal)?, ); - if let Some(version) = req_headers.get(CONSENSUS_VERSION_HEADER) { - headers.insert(CONSENSUS_VERSION_HEADER, version.clone()); - } + headers.insert(CONSENSUS_VERSION_HEADER, GLOAS_CONSENSUS_VERSION.clone()); Ok(headers) } diff --git a/tests/src/mock_relay.rs b/tests/src/mock_relay.rs index a05dcbeab..5826a1d8b 100644 --- a/tests/src/mock_relay.rs +++ b/tests/src/mock_relay.rs @@ -115,6 +115,8 @@ pub struct MockRelayState { received_auth: RwLock>, /// The `X-Timeout-Ms` of the last bid request received_bid_timeout_ms: RwLock>, + /// The raw `Eth-Consensus-Version` of the last bid request + received_bid_consensus_version: RwLock>, response_override: RwLock>, bid_value: RwLock, /// The raw `Accept` header PBS sent on the most recent get_header request, @@ -181,6 +183,9 @@ impl MockRelayState { pub fn received_bid_timeout_ms(&self) -> Option { *self.received_bid_timeout_ms.read().unwrap() } + pub fn received_bid_consensus_version(&self) -> Option { + self.received_bid_consensus_version.read().unwrap().clone() + } pub fn received_auth_data(&self) -> Option> { self.received_auth.read().unwrap().as_ref().map(|a| a.message.data.to_vec()) } @@ -240,6 +245,7 @@ impl MockRelayState { bid_delay_ms: None, received_auth: RwLock::new(None), received_bid_timeout_ms: RwLock::new(None), + received_bid_consensus_version: RwLock::new(None), response_override: RwLock::new(None), bid_value: RwLock::new(U256::from(10)), received_get_header_accept: RwLock::new(None), @@ -391,6 +397,8 @@ async fn handle_get_execution_payload_bid( .into_response(); }; *state.received_bid_timeout_ms.write().unwrap() = Some(timeout_ms); + *state.received_bid_consensus_version.write().unwrap() = + headers.get(CONSENSUS_VERSION_HEADER).and_then(|v| v.to_str().ok()).map(str::to_owned); // Decode the request auth the way a real builder does: Content-Type // selects JSON vs SSZ. The wire type is fork-versioned per builder-specs, diff --git a/tests/tests/pbs_get_execution_payload_bid.rs b/tests/tests/pbs_get_execution_payload_bid.rs index 0cc4fb2fa..3eca13e22 100644 --- a/tests/tests/pbs_get_execution_payload_bid.rs +++ b/tests/tests/pbs_get_execution_payload_bid.rs @@ -325,6 +325,30 @@ async fn test_get_execution_payload_bid_spec_url() -> Result<()> { Ok(()) } +/// The beacon node's `Eth-Consensus-Version` parses in any case, and CB sends +/// the spec's lowercase name on to the relay and back in the 200 +#[tokio::test] +async fn test_get_execution_payload_bid_sends_the_spec_consensus_version() -> Result<()> { + let (mock_validator, mock_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + let res = mock_validator + .comm_boost + .client + .post(bid_url(&mock_validator)) + .header(HEADER_START_TIME_UNIX_MS, utcnow_ms()) + .header(HEADER_TIMEOUT_MS, 60_000u64) + .header(CONSENSUS_VERSION_HEADER, "GLOAS") + .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .body(opaque_auth(TEST_AUTH_DATA, TEST_SLOT).as_ssz_bytes()) + .send() + .await?; + assert_eq!(res.status(), StatusCode::OK); + let echoed = res.headers().get(CONSENSUS_VERSION_HEADER).and_then(|v| v.to_str().ok()); + assert_eq!(echoed, Some("gloas")); + assert_eq!(mock_state.received_bid_consensus_version().as_deref(), Some("gloas")); + Ok(()) +} + /// The response encoding follows the caller's Accept and defaults to JSON when /// none is sent (builder-specs), and the 200 carries Eth-Consensus-Version /// either way. Default relays answer in SSZ; the JSON-only relays cover the From 02ec95244eaac027b51063d07968a84f575414f3 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Wed, 7 Oct 2026 15:34:47 -0700 Subject: [PATCH 14/24] fix(pbs): require an ePBS deadline buffer above 0 proposer_deadline_buffer_ms must now be greater than 0 as well as less than one slot. With 0 the builder gets the whole deadline, so a bid it sends at the deadline reaches the beacon node late. The range is one check in PbsConfig::validate, so the service, a hot reload and a custom PBS module's loader all refuse it. The test fixtures use the default, 50. The metrics catalog documents the ePBS endpoint values, the codes the ePBS endpoints return to the beacon node, the relay_id="dial" label, and that ePBS bids set the relay gauges. The ePBS page corrects two builder config bullets: a top-level min_bid does not floor an entry that sets its own, and the per-client max_execution_payment defaults are dropped. A builder Commit-Boost refuses to dial is not counted under relay_id="dial". --- config.example.toml | 2 +- crates/common/src/config/pbs.rs | 45 ++++++++++++++++--- docs/docs/get_started/epbs.md | 11 ++--- .../get_started/running/metrics-catalog.md | 12 ++--- tests/src/utils.rs | 2 +- tests/tests/pbs_cfg_file_update.rs | 2 +- 6 files changed, 55 insertions(+), 19 deletions(-) diff --git a/config.example.toml b/config.example.toml index 4ce750cb9..1c6354e60 100644 --- a/config.example.toml +++ b/config.example.toml @@ -31,7 +31,7 @@ wait_all_registrations = true # this should be lower than that, leaving some margin for overhead # OPTIONAL, DEFAULT: 950 timeout_get_header_ms = 950 -# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline, must be under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing) +# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline, must be above 0 and under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing) # OPTIONAL, DEFAULT: 50 # proposer_deadline_buffer_ms = 50 # Timeout in milliseconds for the `submit_blinded_block` call to relays. diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index 22f7a1eda..294c5b535 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -250,13 +250,12 @@ impl PbsConfig { ); ensure!(self.late_in_slot_time_ms > 0, "late_in_slot_time_ms must be greater than 0"); - // The buffer comes out of the proposer's deadline, which is clamped to - // one slot, so a buffer of a slot or more leaves no time for the bid - // request. 0 is allowed (no reserve). + // 0 leaves the bid no time to get back to the beacon node, and a slot or + // more leaves the builder none, as the proposer's deadline is clamped to a slot let slot_time_ms = chain.slot_time_sec().saturating_mul(1000); ensure!( - self.proposer_deadline_buffer_ms < slot_time_ms, - "proposer_deadline_buffer_ms must be less than one slot ({slot_time_ms} ms)" + (1..slot_time_ms).contains(&self.proposer_deadline_buffer_ms), + "proposer_deadline_buffer_ms must be greater than 0 and less than one slot ({slot_time_ms} ms)" ); ensure!( @@ -557,6 +556,42 @@ mod tests { use super::*; use crate::config::test_env::{RELAY_URL, with_env}; + fn config_with_buffer(buffer: u64) -> String { + format!( + "chain = \"Holesky\"\n[pbs]\nproposer_deadline_buffer_ms = {buffer}\n\ + [[relays]]\nurl = \"{RELAY_URL}\"\n" + ) + } + + #[tokio::test] + async fn test_deadline_buffer_range() { + for (buffer, accepted) in [(0, false), (1, true), (11999, true), (12000, false)] { + let config: CommitBoostConfig = toml::from_str(&config_with_buffer(buffer)).unwrap(); + assert_eq!(config.validate().await.is_ok(), accepted, "{buffer}"); + } + } + + // The service and a custom PBS module load [pbs] through different functions + #[test] + fn test_loaders_reject_zero_deadline_buffer() { + #[derive(Deserialize)] + struct NoExtra {} + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cb-config.toml"); + std::fs::write(&path, config_with_buffer(0)).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let service = runtime.block_on(load_pbs_config(Some(path.clone()))).map(|_| ()); + let custom = with_env(&[(CONFIG_ENV, path.to_str())], || { + runtime.block_on(load_pbs_custom_config::()).map(|_| ()) + }); + for result in [service, custom] { + let err = result.expect_err("a zero deadline buffer loaded"); + let expected = + "proposer_deadline_buffer_ms must be greater than 0 and less than one slot"; + assert!(format!("{err:#}").contains(expected), "{err:#}"); + } + } + fn relay_with_headers(headers: &str) -> Result { toml::from_str(&format!("url = \"{RELAY_URL}\"\nheaders = {headers}\n")) } diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index df2c577d3..1c14de6c8 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -39,7 +39,7 @@ To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and i ### 1. Add the builders to Commit-Boost -List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be under one slot, `12000` on mainnet. +List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be above `0` and under one slot, `12000` on mainnet. ### 2. Point each validator key at Commit-Boost {#validator-builder-config} @@ -66,8 +66,8 @@ curl -X POST "$KEYMANAGER_URL/eth/v1/validator/$PUBKEY/builder_config" \ The call is `POST /eth/v1/validator/{pubkey}/builder_config`, authenticated with the keymanager API's bearer token. The body replaces the key's config in full, and the validator client answers `202` once it is stored. Besides `url` and `auth_data`, an entry or the top level can set: -- `min_bid` and `builder_boost_factor` are optional. At the top level they apply to every entry, and to p2p bids, unless an entry sets its own. A top-level `min_bid` also floors the bids that come through Commit-Boost, so a value above your builders' bids sends every proposal to p2p bids or a local block. -- `max_execution_payment`, when an entry omits it, gets the validator client's default, which differs between clients (Lodestar stores `0`, Nimbus the maximum). Lodestar accepts a nonzero value only when started with `--allowDangerousTrustedPayments`. +- `min_bid` and `builder_boost_factor` are optional. At the top level they apply to p2p bids and to every entry that does not set its own, so a top-level `min_bid` above your builders' bids sends every proposal to a p2p bid or a local block, unless an entry sets a lower one of its own. +- `max_execution_payment`, when an entry omits it, gets the validator client's default, which differs between clients and can be `0`, so no execution payment counts. Lodestar accepts a nonzero value only when started with `--allowDangerousTrustedPayments`. - `builder_pubkeys` limits which builder keys' bids the beacon node accepts; leave it empty to accept any. Don't copy the pubkey from the relay URL: that is the relay's key, not the builder's bid-signing key. ## Routing by auth data @@ -88,7 +88,7 @@ A key without builder config sends Commit-Boost's own hostname as its `auth_data The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts your `proposer_deadline_buffer_ms` from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. -Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon node's remaining time: enough for the bid to get back from Commit-Boost to the beacon node and for the beacon node to process it. If no time is left, Commit-Boost does not ask the builder. +Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon node's remaining time: enough for the bid to get back from Commit-Boost to the beacon node and for the beacon node to process it. If no time is left, Commit-Boost does not ask the builder. With `0`, Commit-Boost refuses to start or reload: the builder gets the whole deadline, so a bid it sends at the deadline reaches the beacon node late. Builder preferences use `timeout_register_validator_ms`, and the signed block `timeout_get_payload_ms`. @@ -99,7 +99,7 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Question | Series | |---|---| | What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences, `202` for the signed block whatever the builders answer, `4xx` for a rejected request, `500` when no builder accepted the preferences | -| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"` | +| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"`, except one Commit-Boost [refuses to dial](#builders-outside-your-config), which gets no request | | How fast are builders? | `cb_pbs_relay_latency`, by `relay_id` | | What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from each bid a builder serves | @@ -110,6 +110,7 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a hostname nor an `http(s)` URL, or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | | Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config). A key without builder config sends Commit-Boost's own hostname, which usually does | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | +| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | | The signed block gets `415` | The beacon node sends it as JSON | Configure the beacon node to send SSZ | | Commit-Boost logs "no builder accepted the signed beacon block" | The winning bid came from a builder outside your config, which gets the block over gossip, or your builders did not accept it. The beacon node gossips the block either way | If the bid came from a configured builder, check its answer in `cb_pbs_relay_status_code_total{endpoint="submit_signed_beacon_block"}` | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/docs/get_started/running/metrics-catalog.md b/docs/docs/get_started/running/metrics-catalog.md index fe44cc3fc..4db35ea41 100644 --- a/docs/docs/get_started/running/metrics-catalog.md +++ b/docs/docs/get_started/running/metrics-catalog.md @@ -10,19 +10,19 @@ Every metric emitted by the Commit-Boost PBS and Signer services, together with ## PBS metrics -PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. The registry is created via `Registry::new_custom(Some("cb_pbs"), None)` in `crates/pbs/src/metrics.rs`. All wire names shown below include this prefix. +PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. The registry is created via `Registry::new_custom(Some("cb_pbs"), None)` in `crates/pbs/src/metrics.rs`. All wire names shown below include this prefix. (unreleased, from v0.12.0-rc1) ePBS requests to a builder outside your config are labeled `relay_id="dial"`. | Metric name (wire) | Type | Labels | Description | |---|---|---|---| -| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when the request hit its deadline, and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`. | -| `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`. Under `get_header_stream` the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | -| `cb_pbs_relay_last_slot` | Gauge | `relay_id` | Latest slot for which a relay delivered a header. Only updated in the `get_header` handler. Set to the current slot on each successful header from that relay. | -| `cb_pbs_relay_header_value` | Gauge | `relay_id` | Header value in gwei delivered by a relay. Converted from raw wei (รท 1e9) in the `get_header` handler. | +| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (timeout, DNS or connection failure), and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. | +| `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Under `get_header_stream` the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | +| `cb_pbs_relay_last_slot` | Gauge | `relay_id` | Latest slot for which a relay delivered a bid. Set to the current slot on each successful `get_header` bid and (unreleased, from v0.12.0-rc1) each ePBS bid served from that relay. | +| `cb_pbs_relay_header_value` | Gauge | `relay_id` | Value in gwei of the bid a relay delivered: for `get_header`, the bid's value converted from wei (รท 1e9). (unreleased, from v0.12.0-rc1) For ePBS, the returned bid's `value`, without its execution payment. | | `cb_pbs_relay_stream_connect_latency` | Histogram | `relay_id` | Websocket handshake latency in seconds, for a relay with `get_header = "stream"`. Observed once per successful handshake; a handshake that fails or runs out the bid window records nothing here. Custom buckets from 5 ms to 1 s. | | `cb_pbs_relay_stream_updates` | Histogram | `relay_id` | Bid updates received per stream window. Observed once per `get_header` served by the stream, including windows that received nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. | | `cb_pbs_relay_stream_invalid_frames_total` | Counter | `relay_id` | Websocket frames that could not be parsed as a bid. Incremented only for windows that saw at least one, so the series is absent while zero. | | `cb_pbs_relay_stream_fallback_total` | Counter | `relay_id` | Stream attempts that fell back to a plain HTTP `get_header`. Only a handshake failure with bid window left to retry counts here; one that consumed the window shows on the status series alone. The fallback's own result is recorded under `endpoint="get_header"`. | -| `cb_pbs_beacon_node_status_code_total` | Counter | `http_status_code`, `endpoint` | HTTP status code returned to the beacon node. Tracks what status codes the PBS returns for beacon node-facing requests. Endpoint values: `get_header`, `register_validator`, `submit_blinded_block`, `status`, `reload`. Error status codes (`502` for `NoResponse`/`NoPayload`, `500` for `Internal`) are set via `PbsClientError`. The handlers also record `406` directly when the request's `Accept` header offers no supported encoding, `204` when no bid is available on `get_header`, and `202` for accepted v2 `submit_blinded_block` requests. | +| `cb_pbs_beacon_node_status_code_total` | Counter | `http_status_code`, `endpoint` | HTTP status code returned to the beacon node. Tracks what status codes the PBS returns for beacon node-facing requests. Endpoint values: `get_header`, `register_validator`, `submit_blinded_block`, `status`, `reload`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Error status codes (`502` for `NoResponse`/`NoPayload`, `500` for `Internal`) are set via `PbsClientError`. The handlers also record `406` directly when the request's `Accept` header offers no supported encoding, `204` when no bid is available on `get_header`, and `202` for accepted v2 `submit_blinded_block` requests. (unreleased, from v0.12.0-rc1) The ePBS endpoints record `200` for a bid, `204` for none, `202` for accepted preferences and for every signed block it forwards, `400` for a request Commit-Boost refuses, a builder's own `400` or `401`, `406` for a bid request whose `Accept` offers no supported encoding, `415` for an unsupported `Content-Type`, and `500` when no builder accepts the preferences. | | `cb_pbs_pbs_submit_block_v2_unsupported_total` | Counter | `relay_id` | Count of v2 `submit_blinded_block` requests a relay could not serve because it returned 404 on the v2 endpoint. A non-zero value means the relay does not support `submitBlindedBlockV2` and those blocks were not submitted via that relay. The double `pbs` in the wire name comes from the registry prefix plus the metric name `pbs_submit_block_v2_unsupported_total`. | --- diff --git a/tests/src/utils.rs b/tests/src/utils.rs index d3c924166..1839b3c9c 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -138,7 +138,7 @@ pub fn get_pbs_config(port: u16) -> PbsConfig { skip_sigverify: false, min_bid_wei: U256::ZERO, late_in_slot_time_ms: u64::MAX, - proposer_deadline_buffer_ms: 0, + proposer_deadline_buffer_ms: 50, extra_validation_enabled: false, ssv_node_api_url: Url::parse("http://localhost:0").unwrap(), diff --git a/tests/tests/pbs_cfg_file_update.rs b/tests/tests/pbs_cfg_file_update.rs index 23d59e94e..c4c02ce17 100644 --- a/tests/tests/pbs_cfg_file_update.rs +++ b/tests/tests/pbs_cfg_file_update.rs @@ -66,7 +66,7 @@ async fn test_cfg_file_update() -> Result<()> { min_bid_wei: U256::ZERO, late_in_slot_time_ms: u64::MAX / 2, /* serde gets very upset about serializing u64::MAX * or anything close to it */ - proposer_deadline_buffer_ms: 0, + proposer_deadline_buffer_ms: 50, extra_validation_enabled: false, rpc_url: None, ssv_node_api_url: Url::parse("http://example.com").unwrap(), From 8aa4118cdd54b097dc8979ae897af61dc522a05b Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Thu, 8 Oct 2026 08:24:48 -0700 Subject: [PATCH 15/24] perf(pbs): share one client across dials A client per dial existed only because resolve_to_addrs pins addresses per client. The shared client's resolver now checks every address it resolves, so a dial still connects only to checked addresses, and a second dial to a builder reuses the first one's connection instead of a new TCP and TLS handshake. Dial hosts are chosen by the request, so the client keeps at most one idle connection per host, for one slot. The check before the dial is unchanged, so a blocked target still gets 400 before anything is sent. --- crates/pbs/Cargo.toml | 3 + crates/pbs/src/dial.rs | 205 ++++++++++++++++++++++++++++++++--------- 2 files changed, 162 insertions(+), 46 deletions(-) diff --git a/crates/pbs/Cargo.toml b/crates/pbs/Cargo.toml index 3a52e12ef..7037a566f 100644 --- a/crates/pbs/Cargo.toml +++ b/crates/pbs/Cargo.toml @@ -38,3 +38,6 @@ url.workspace = true uuid.workspace = true webpki-roots.workspace = true thiserror.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/pbs/src/dial.rs b/crates/pbs/src/dial.rs index 6b7fdfdd5..d5a75a007 100644 --- a/crates/pbs/src/dial.rs +++ b/crates/pbs/src/dial.rs @@ -1,6 +1,7 @@ use std::{ + io, net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}, - sync::LazyLock, + sync::{Arc, LazyLock, OnceLock}, time::Duration, }; @@ -10,7 +11,8 @@ use cb_common::{ types::BlsPublicKey, utils::bls_pubkey_from_hex, }; -use tokio::sync::Semaphore; +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; +use tokio::{sync::Semaphore, task::JoinHandle}; use tracing::{info, warn}; use url::Url; @@ -36,8 +38,8 @@ pub(crate) fn auth_data_address(data: &[u8]) -> &[u8] { data.iter().position(|&byte| byte == b'?').map_or(data, |end| &data[..end]) } -/// A relay for the builder that an unmatched address names. The target is -/// untrusted, so it is resolved once and dialed only at the vetted addresses. +/// A relay for the builder an unmatched address names, checked before the dial +/// and, for a hostname, again on each new connection pub(crate) async fn dial_relay( data_url: Option, address: &[u8], @@ -47,10 +49,7 @@ pub(crate) async fn dial_relay( let origin = url.origin().ascii_serialization(); let addrs = resolve_dial_target(&url, &origin, lookup_timeout).await?; info!(%origin, ?addrs, "auth data names a builder outside the config, dialing it"); - pinned_relay(url, &addrs).map_err(|err| { - warn!(%err, "failed to build the dial client"); - PbsClientError::Internal - }) + dial_client(url) } /// `data_url`, else `https:///` for the builder-specs default auth @@ -90,15 +89,10 @@ async fn resolve_dial_target( vet_dial_addrs(origin, &[addr])?; return Ok(vec![addr]); } - let Ok(permit) = DIAL_LOOKUPS.try_acquire() else { + let Some(lookup) = spawn_lookup(host_port) else { warn!(%origin, "too many dial lookups in flight, not dialing"); return Err(PbsClientError::NoBuilderResponse); }; - let lookup = tokio::task::spawn_blocking(move || { - // Held until the lookup returns, which a timeout does not stop - let _permit = permit; - host_port.to_socket_addrs().map(Iterator::collect::>) - }); let addrs = match tokio::time::timeout(lookup_timeout, lookup).await { Ok(Ok(Ok(addrs))) => addrs, Ok(Ok(Err(err))) => { @@ -116,6 +110,16 @@ async fn resolve_dial_target( Ok(addrs) } +/// `None` when every lookup slot is taken +fn spawn_lookup(host_port: String) -> Option>>> { + let permit = DIAL_LOOKUPS.try_acquire().ok()?; + Some(tokio::task::spawn_blocking(move || { + // Held until the lookup returns, which a timeout does not stop + let _permit = permit; + host_port.to_socket_addrs().map(Iterator::collect::>) + })) +} + /// Every address is checked, since the dial may connect to any of them fn vet_dial_addrs(origin: &str, addrs: &[SocketAddr]) -> Result<(), PbsClientError> { if dial_target_check_enabled() && addrs.iter().any(|addr| ip_is_disallowed(addr.ip())) { @@ -125,30 +129,75 @@ fn vet_dial_addrs(origin: &str, addrs: &[SocketAddr]) -> Result<(), PbsClientErr Ok(()) } -/// A relay that dials `url` only at `addrs`: a system proxy would resolve the -/// host again, and a redirect target was never checked. -fn pinned_relay(url: Url, addrs: &[SocketAddr]) -> eyre::Result { - let mut client = - reqwest::Client::builder().no_proxy().redirect(reqwest::redirect::Policy::none()); - if let Some(domain) = url.domain() { - client = client.resolve_to_addrs(domain, addrs); - } +/// Dial hosts are chosen by the request, so an idle connection is reused for at +/// most one slot. The pool sweeps at this interval, so it closes within two. +const DIAL_IDLE_TIMEOUT: Duration = Duration::from_secs(12); + +/// Lets a builder dialed again reuse its connection. A failed build is not +/// kept, so the next dial tries again. +static DIAL_CLIENT: OnceLock = OnceLock::new(); + +/// No proxy, which would resolve the host itself, and no redirects, since one +/// to an IP address would skip the resolver +fn build_dial_client() -> eyre::Result { RelayClient::with_client_builder( - RelayConfig { - entry: RelayEntry { id: DIAL_RELAY_ID.to_string(), pubkey: DIAL_PUBKEY.clone(), url }, - id: None, - headers: None, - get_params: None, - get_header: GetHeaderTransport::Http, - enable_timing_games: false, - target_first_request_ms: None, - frequency_get_header_ms: None, - validator_registration_batch_size: None, - }, - client, + dial_config(Url::parse("https://dial.invalid/").expect("a valid URL")), + reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .dns_resolver(DialResolver) + .pool_idle_timeout(DIAL_IDLE_TIMEOUT) + .pool_max_idle_per_host(1), ) } +/// The shared dial client, addressed to `url` +fn dial_client(url: Url) -> Result { + let shared = match DIAL_CLIENT.get() { + Some(shared) => shared, + None => { + let built = build_dial_client().map_err(|err| { + warn!(%err, "failed to build the dial client"); + PbsClientError::Internal + })?; + DIAL_CLIENT.get_or_init(|| built) + } + }; + let mut relay = shared.clone(); + relay.config = Arc::new(dial_config(url)); + Ok(relay) +} + +fn dial_config(url: Url) -> RelayConfig { + RelayConfig { + entry: RelayEntry { id: DIAL_RELAY_ID.to_string(), pubkey: DIAL_PUBKEY.clone(), url }, + id: None, + headers: None, + get_params: None, + get_header: GetHeaderTransport::Http, + enable_timing_games: false, + target_first_request_ms: None, + frequency_get_header_ms: None, + validator_registration_batch_size: None, + } +} + +/// Run for each new connection to a hostname (an IP address skips it): refuses +/// it while every lookup slot is taken or if any address is disallowed +struct DialResolver; + +impl Resolve for DialResolver { + fn resolve(&self, name: Name) -> Resolving { + Box::pin(async move { + let lookup = spawn_lookup(format!("{}:0", name.as_str())) + .ok_or("too many dial lookups in flight")?; + let addrs = lookup.await??; + vet_dial_addrs(name.as_str(), &addrs)?; + Ok(Box::new(addrs.into_iter()) as Addrs) + }) + } +} + #[cfg(feature = "testing-flags")] thread_local! { static SKIP_DIAL_TARGET_CHECK: std::cell::Cell = const { std::cell::Cell::new(false) }; @@ -212,6 +261,8 @@ fn ip_is_disallowed(ip: IpAddr) -> bool { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use axum::http::StatusCode; use cb_common::pbs::decode_auth_data_url; @@ -248,10 +299,13 @@ mod tests { } } - // Holds every permit, so a hostname lookup in a unit test running beside it - // is refused + /// Taken by every test that holds or takes a lookup slot, since the slots + /// are shared by the whole test binary + static LOOKUP_SLOTS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + #[tokio::test] async fn lookup_cap_refuses_hostnames_not_ip_literals() { + let _slots = LOOKUP_SLOTS.lock().await; let _held = DIAL_LOOKUPS.try_acquire_many(32).unwrap(); let dial = |address: &'static [u8]| dial_relay(None, address, Duration::from_secs(1)); assert!(dial(b"1.1.1.1").await.is_ok()); @@ -270,25 +324,84 @@ mod tests { )); } - // A hostname target is dialed at the given address with no lookup - // (`.invalid` never resolves), and a redirect is not followed - #[tokio::test] - async fn pinned_relay_dials_the_given_address_and_refuses_redirects() -> eyre::Result<()> { - use axum::{Router, http::header::LOCATION, routing::post}; + /// A builder on loopback whose `/bid` redirects to `/internal` and whose + /// `/slow` answers after 50 ms, and its count of accepted connections + async fn mock_builder() -> eyre::Result<(u16, Arc)> { + use axum::{Router, http::header::LOCATION, routing::post, serve::ListenerExt}; + let connections = Arc::new(AtomicUsize::new(0)); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; - let addr = listener.local_addr()?; + let port = listener.local_addr()?.port(); + let accepted = connections.clone(); + let listener = listener.tap_io(move |_| { + accepted.fetch_add(1, Ordering::Relaxed); + }); let builder = Router::new() .route( "/bid", post(|| async { (StatusCode::TEMPORARY_REDIRECT, [(LOCATION, "/internal")]) }), ) - .route("/internal", post(|| async { StatusCode::OK })); + .route("/internal", post(|| async { StatusCode::OK })) + .route( + "/slow", + post(|| async { + tokio::time::sleep(Duration::from_millis(50)).await; + StatusCode::OK + }), + ); tokio::spawn(async move { axum::serve(listener, builder).await }); + Ok((port, connections)) + } + + // An IP literal skips the resolver, which would refuse loopback + #[tokio::test] + async fn dials_share_connections_and_refuse_redirects() -> eyre::Result<()> { + let (port, connections) = mock_builder().await?; + let url = Url::parse(&format!("http://127.0.0.1:{port}/bid"))?; + for _ in 0..2 { + let res = dial_client(url.clone())?.client.post(url.clone()).send().await?; + assert_eq!(res.status(), StatusCode::TEMPORARY_REDIRECT); + } + assert_eq!(connections.load(Ordering::Relaxed), 1); - let url = Url::parse(&format!("http://dial-target.invalid:{}/bid", addr.port()))?; - let res = pinned_relay(url.clone(), &[addr])?.client.post(url).send().await?; - assert_eq!(res.status(), StatusCode::TEMPORARY_REDIRECT); + tokio::time::pause(); + tokio::time::advance(DIAL_IDLE_TIMEOUT + Duration::from_secs(1)).await; + tokio::time::resume(); + dial_client(url.clone())?.client.post(url).send().await?; + assert_eq!(connections.load(Ordering::Relaxed), 2); + Ok(()) + } + + // Two requests at once need two connections, and only one stays idle + #[tokio::test] + async fn dial_client_keeps_one_idle_connection_per_host() -> eyre::Result<()> { + let (port, connections) = mock_builder().await?; + let url = Url::parse(&format!("http://127.0.0.1:{port}/slow"))?; + let client = dial_client(url.clone())?.client; + for _ in 0..2 { + tokio::try_join!(client.post(url.clone()).send(), client.post(url.clone()).send())?; + } + assert_eq!(connections.load(Ordering::Relaxed), 3); + Ok(()) + } + + // No check runs before these dials, so only the resolver refuses + // `localhost`, which resolves to loopback + #[cfg(feature = "testing-flags")] + #[tokio::test] + async fn dial_client_checks_the_addresses_it_resolves() -> eyre::Result<()> { + let _slots = LOOKUP_SLOTS.lock().await; + let (port, _) = mock_builder().await?; + let url = Url::parse(&format!("http://localhost:{port}/bid"))?; + let dial = || dial_client(url.clone()).map(|relay| relay.client.post(url.clone()).send()); + assert!(dial()?.await.is_err()); + set_skip_dial_target_check(true); + let held = DIAL_LOOKUPS.try_acquire_many(32)?; + assert!(dial()?.await.is_err()); + drop(held); + let res = dial()?.await; + set_skip_dial_target_check(false); + assert_eq!(res?.status(), StatusCode::TEMPORARY_REDIRECT); Ok(()) } From 6d11cdc8aa19adf6e2e546c0689baae335c06566 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 5 Oct 2026 23:17:50 -0700 Subject: [PATCH 16/24] feat(pbs): stream the ePBS bid alongside the HTTP bid request A relay with get_header = "stream" now also streams its ePBS bids. Each bid request addressed to it opens ws(s):///eth/v1/builder/execution_payload_bid_stream/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey} on the PBS stream reader, with the SignedBuilderRequestAuth as padded standard base64 of its SSZ in X-Request-Auth, and races it against the relay's HTTP request. The bid with the higher value + execution_payment wins, unclamped, and the stream's bid wins a tie. A builder's HTTP 400 or 401 reaches the proposer only when neither leg has a bid. A builder Commit-Boost dials is asked over HTTP only. The stream's bid keeps its SSZ frame, so a beacon node that asks for SSZ gets it unchanged. Each frame is decoded on arrival, so the stream holds only its newest bid. A streamed request can last until the deadline less proposer_deadline_buffer_ms, which #506 requires to be above 0. A failed stream is logged and counted in one place, and the race's docs mark what is unreleased. The stream reports the bid stream series under endpoint="get_execution_payload_bid_stream". The ePBS page gets a Bid streaming section and two troubleshooting rows: a relay that streams get_header but not ePBS bids, and a stream that times out or cannot connect. --- Cargo.lock | 2 + config.example.toml | 11 +- crates/common/src/config/pbs.rs | 2 +- crates/common/src/pbs/constants.rs | 4 + crates/common/src/pbs/relay.rs | 54 ++- crates/pbs/Cargo.toml | 1 + crates/pbs/src/bid_stream.rs | 47 ++- crates/pbs/src/constants.rs | 1 + crates/pbs/src/dial.rs | 2 + crates/pbs/src/metrics.rs | 8 +- crates/pbs/src/mev_boost/get_header.rs | 122 ++++-- crates/pbs/src/mev_boost/get_header_ws.rs | 13 +- .../pbs/src/routes/execution_payload_bid.rs | 399 ++++++++++++++++-- .../src/routes/execution_payload_bid_ws.rs | 98 +++++ crates/pbs/src/routes/mod.rs | 1 + docs/docs/get_started/configuration.md | 26 +- docs/docs/get_started/epbs.md | 23 +- .../get_started/running/metrics-catalog.md | 14 +- docs/docs/get_started/running/metrics.md | 20 +- tests/Cargo.toml | 1 + tests/src/mock_relay.rs | 49 ++- tests/src/mock_ws_relay.rs | 167 ++++++-- .../tests/pbs_get_execution_payload_bid_ws.rs | 193 +++++++++ tests/tests/pbs_get_header_ws.rs | 54 +-- 24 files changed, 1102 insertions(+), 210 deletions(-) create mode 100644 crates/pbs/src/routes/execution_payload_bid_ws.rs create mode 100644 tests/tests/pbs_get_execution_payload_bid_ws.rs diff --git a/Cargo.lock b/Cargo.lock index 140c33039..32e64a92d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1992,6 +1992,7 @@ dependencies = [ "async-trait", "axum 0.8.9", "axum-extra", + "base64 0.22.1", "cb-common", "cb-metrics", "ethereum_serde_utils 0.7.0", @@ -2056,6 +2057,7 @@ version = "0.11.0" dependencies = [ "alloy", "axum 0.8.9", + "base64 0.22.1", "cb-common", "cb-pbs", "cb-signer", diff --git a/config.example.toml b/config.example.toml index 96f0cdcfd..f4a177bff 100644 --- a/config.example.toml +++ b/config.example.toml @@ -100,16 +100,17 @@ headers = { X-MyCustomHeader = "MyCustomValue" } # GET parameters to add to each request URL for this relay # OPTIONAL get_params = { param1 = "value1", param2 = "value2" } -# How to fetch headers from this relay. +# How to fetch bids from this relay. # "http" -> one request per get_header, at the relay url above # "stream" -> websocket stream of bid updates, only for relays that support it. Connects to -# ws(s):///eth/v1/builder/header_stream/{slot}/{parent_hash}/{pubkey}, -# and sends the relay's normal HTTP request alongside, timing games included. The -# stream's latest valid bid and the HTTP bid are separate candidates in the -# auction, so a stream that fails leaves the HTTP bid in the auction. +# ws(s):///eth/v1/builder/header_stream/{slot}/{parent_hash}/{pubkey}. +# (unreleased, from v0.12.0-rc1) The relay's normal HTTP request runs alongside, +# timing games included, and the better bid wins, so a failed stream still +# leaves the HTTP bid. # Requires a UUID api key, set it in `headers` field above under the `X-Api-Key` # name; it is sent on the websocket handshake. Can use # https://www.uuidgenerator.net/version4 to generate one. +# (unreleased, from v0.12.0-rc1) After Gloas this also streams ePBS bids (https://commit-boost.github.io/commit-boost-client/get_started/epbs#bid-streaming) # OPTIONAL, DEFAULT: "http" get_header = "http" # Whether to enable timing games, as tuned by `target_first_request_ms` and `frequency_get_header_ms`. diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index 28f3b90f3..00df7f6ee 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -140,7 +140,7 @@ pub struct RelayConfig { pub headers: Option>, /// Optional GET parameters to add to each request pub get_params: Option>, - /// How to fetch headers from this relay + /// How to fetch bids from this relay #[serde(default)] pub get_header: GetHeaderTransport, /// Whether to enable timing games diff --git a/crates/common/src/pbs/constants.rs b/crates/common/src/pbs/constants.rs index e76395301..111d77e9c 100644 --- a/crates/common/src/pbs/constants.rs +++ b/crates/common/src/pbs/constants.rs @@ -13,6 +13,7 @@ pub const RELOAD_PATH: &str = "/reload"; pub const GET_EXECUTION_PAYLOAD_BID_PATH: &str = "/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}"; +pub const GET_EXECUTION_PAYLOAD_BID_STREAM_PATH: &str = "/execution_payload_bid_stream"; pub const SUBMIT_BUILDER_PREFERENCES_PATH: &str = "/builder_preferences/{proposer_pubkey}"; pub const SUBMIT_SIGNED_BEACON_BLOCK_PATH: &str = "/beacon_blocks"; @@ -24,6 +25,9 @@ pub const HEADER_VERSION_KEY: &str = "X-CommitBoost-Version"; pub const HEADER_VERSION_VALUE: &str = COMMIT_BOOST_VERSION; pub const HEADER_START_TIME_UNIX_MS: &str = "Date-Milliseconds"; pub const HEADER_TIMEOUT_MS: &str = "X-Timeout-Ms"; +/// The SSZ `SignedBuilderRequestAuth` as padded standard base64, on the ePBS +/// bid stream handshake: a websocket upgrade has no body +pub const HEADER_REQUEST_AUTH: &str = "X-Request-Auth"; pub const HEADER_API_KEY: &str = "X-Api-Key"; pub const HEADER_CONSENSUS_VERSION: &str = "Eth-Consensus-Version"; diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index 75ec1bbd0..4a708b143 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -11,8 +11,8 @@ use url::Url; use super::{ HEADER_VERSION_KEY, HEADER_VERSION_VALUE, constants::{ - GET_HEADER_STREAM_PATH, GET_STATUS_PATH, REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH, - SUBMIT_SIGNED_BEACON_BLOCK_PATH, + GET_EXECUTION_PAYLOAD_BID_STREAM_PATH, GET_HEADER_STREAM_PATH, GET_STATUS_PATH, + REGISTER_VALIDATOR_PATH, SUBMIT_BLOCK_PATH, SUBMIT_SIGNED_BEACON_BLOCK_PATH, }, error::PbsError, }; @@ -96,9 +96,10 @@ pub struct RelayClient { pub id: Arc, /// HTTP client to send requests pub client: reqwest::Client, - /// Base url of the get_header stream, `Some` only when the relay streams. + /// Base url of the get_header stream, whose origin the ePBS bid stream + /// shares. `Some` only when the relay streams. stream_url: Option, - /// Baseline headers for the get_header stream handshake. + /// Baseline headers for the bid stream handshakes. stream_headers: Arc, /// Configuration of the relay pub config: Arc, @@ -246,6 +247,24 @@ impl RelayClient { ) } + /// The ePBS bid stream, at the get_header stream's origin. `None` when the + /// relay does not stream. + pub fn get_execution_payload_bid_stream_url( + &self, + slot: u64, + parent_hash: &B256, + parent_root: &B256, + validator_pubkey: &BlsPublicKey, + ) -> Option { + let mut url = self.stream_url.clone()?; + url.set_path(&format!( + "{}{GET_EXECUTION_PAYLOAD_BID_STREAM_PATH}/{slot}/{parent_hash}/{parent_root}/{validator_pubkey}", + BuilderApiVersion::V1.path() + )); + self.append_get_params(&mut url); + Some(url) + } + /// builder-API: POST /eth/v1/builder/builder_preferences/{proposer_pubkey} pub fn submit_builder_preferences_url( &self, @@ -391,9 +410,10 @@ mod tests { } #[test] - fn test_get_header_stream_url() { + fn test_stream_urls() { let slot = 0; let parent_hash = B256::ZERO; + let parent_root = B256::repeat_byte(1); let validator_pubkey = bls_pubkey_from_hex_unchecked( "0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae", ); @@ -406,6 +426,16 @@ mod tests { // Default transport: no stream let relay = RelayClient::new(base_config.clone()).unwrap(); assert!(relay.get_header_stream_url(slot, &parent_hash, &validator_pubkey).is_none()); + assert!( + relay + .get_execution_payload_bid_stream_url( + slot, + &parent_hash, + &parent_root, + &validator_pubkey + ) + .is_none() + ); // Streaming: the relay url over ws, at the fixed stream path, with the // pubkey credentials dropped @@ -434,6 +464,20 @@ mod tests { "wss://abc.xyz:4444/eth/v1/builder/header_stream/{slot}/{parent_hash}/{validator_pubkey}?token=abc" ) ); + let url = relay + .get_execution_payload_bid_stream_url( + slot, + &parent_hash, + &parent_root, + &validator_pubkey, + ) + .unwrap(); + assert_eq!( + url.to_string(), + format!( + "wss://abc.xyz:4444/eth/v1/builder/execution_payload_bid_stream/{slot}/{parent_hash}/{parent_root}/{validator_pubkey}?token=abc" + ) + ); // A relay url we can't stream over is rejected at construction let mut config = base_config; diff --git a/crates/pbs/Cargo.toml b/crates/pbs/Cargo.toml index 7037a566f..cdc46a194 100644 --- a/crates/pbs/Cargo.toml +++ b/crates/pbs/Cargo.toml @@ -13,6 +13,7 @@ alloy.workspace = true async-trait.workspace = true axum.workspace = true axum-extra.workspace = true +base64.workspace = true cb-common.workspace = true cb-metrics.workspace = true ethereum_serde_utils.workspace = true diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs index 6250b2280..6dd630024 100644 --- a/crates/pbs/src/bid_stream.rs +++ b/crates/pbs/src/bid_stream.rs @@ -50,9 +50,9 @@ const FRAME_PREFIX_LEN: usize = 2; const MSG_BID: u8 = 0x01; -/// A stream holds only its newest frames, which bounds its memory and the -/// validation a caller leaves until the deadline -const MAX_HELD_FRAMES: usize = 8; +/// A PBS stream holds only its newest frames, which bounds its memory and the +/// validation it leaves until the deadline +pub(crate) const MAX_HELD_FRAMES: usize = 8; /// One bid update, its bid still SSZ-encoded pub(crate) struct Frame { @@ -62,8 +62,8 @@ pub(crate) struct Frame { /// What a stream held when its window closed pub(crate) struct Held { - /// The newest `MAX_HELD_FRAMES` frames the parse closure accepted, oldest - /// first. Empty when the relay had no bid. + /// The newest frames the parse closure accepted, up to the caller's cap, + /// oldest first pub(crate) frames: VecDeque, /// Frames the parse closure accepted pub(crate) updates: usize, @@ -72,8 +72,8 @@ pub(crate) struct Held { pub(crate) invalid_frames: usize, } -/// The handshake for `url`, with the same headers an HTTP request carries. -/// `read_bid_stream` adds the timing headers once connected. +/// The handshake for `url`, with the caller's user agent and the relay's +/// headers pub(crate) fn handshake_request( url: &Url, relay: &RelayClient, @@ -96,15 +96,14 @@ pub(crate) fn handshake_request( Ok(request) } -/// Opens the stream and holds the newest frames `parse` accepts, until -/// `deadline` or the relay ends the stream. A frame `parse` rejects counts as -/// invalid. On failure, returns the status the failure records under. The -/// stream metrics are recorded under `endpoint`. +/// Holds the newest `max_held` frames `parse` accepts until `deadline` or the +/// end of the stream. An error carries the status the failure records under. pub(crate) async fn read_bid_stream( request: Request<()>, deadline: Instant, relay: &RelayClient, endpoint: &str, + max_held: usize, mut parse: impl FnMut(Frame) -> Result, ) -> Result, (StatusCode, PbsError)> { let start_request = Instant::now(); @@ -132,7 +131,7 @@ pub(crate) async fn read_bid_stream( let timer = sleep_until(deadline); tokio::pin!(timer); - let mut frames = VecDeque::with_capacity(MAX_HELD_FRAMES); + let mut frames = VecDeque::with_capacity(max_held); let mut updates = 0usize; let mut first_frame_latency = None; let mut invalid_frames = 0usize; @@ -150,7 +149,7 @@ pub(crate) async fn read_bid_stream( Some(Ok(Message::Close(_))) | None => break, Some(Ok(_)) => continue, Some(Err(err)) => { - warn!(relay_id = relay.id.as_ref(), %err, "ws stream error"); + debug!(relay_id = relay.id.as_ref(), %err, "ws stream error"); stream_error = Some(PbsError::WebSocket(format!("stream error: {err}"))); break; } @@ -160,7 +159,7 @@ pub(crate) async fn read_bid_stream( Ok(frame) => { first_frame_latency.get_or_insert_with(|| start_request.elapsed()); updates += 1; - if frames.len() == MAX_HELD_FRAMES { + if frames.len() == max_held { frames.pop_front(); } frames.push_back(frame); @@ -192,10 +191,8 @@ pub(crate) async fn read_bid_stream( Ok(Held { frames, updates, connect_latency, first_frame_latency, invalid_frames }) } -/// Opens TCP first, so the handshake can ask the relay to end its stream one -/// downlink trip before `deadline`: a relay honouring the timing headers ends -/// it at Date-Milliseconds + X-Timeout-Ms, and a frame sent then still has to -/// travel. None when no time would be left. +/// Opens TCP first and times it, so `X-Timeout-Ms` ends the relay's stream one +/// downlink trip before `deadline`. None when no time would be left. async fn connect( mut request: Request<()>, deadline: Instant, @@ -259,7 +256,8 @@ fn relay_timeout_ms(time_left: Duration, rtt: Duration) -> Option { (timeout_ms > 0).then_some(timeout_ms) } -pub(crate) fn record_stream_fallback(endpoint: &str, relay_id: &str) { +pub(crate) fn record_stream_failure(endpoint: &str, relay_id: &str, err: &PbsError) { + warn!(%err, relay_id, endpoint, "stream failed"); RELAY_STREAM_FALLBACK.with_label_values(&[endpoint, relay_id]).inc(); } @@ -373,6 +371,17 @@ mod tests { assert_eq!(relay_timeout_ms(ms(40), ms(100)), None); } + // The host of an IPv6 url keeps its brackets, which name resolution rejects + #[tokio::test] + async fn test_connect_tcp_to_an_ipv6_literal() { + let Ok(listener) = tokio::net::TcpListener::bind("[::1]:0").await else { + return; // no IPv6 loopback on this host + }; + let port = listener.local_addr().unwrap().port(); + let uri: Uri = format!("ws://[::1]:{port}/").parse().unwrap(); + assert!(connect_tcp(&uri).await.is_ok()); + } + #[test] fn test_parse_frame() { assert!(matches!( diff --git a/crates/pbs/src/constants.rs b/crates/pbs/src/constants.rs index 5790af416..a48e07e35 100644 --- a/crates/pbs/src/constants.rs +++ b/crates/pbs/src/constants.rs @@ -9,6 +9,7 @@ pub const SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG: &str = "submit_builder_prefer pub const SUBMIT_SIGNED_BEACON_BLOCK_ENDPOINT_TAG: &str = "submit_signed_beacon_block"; pub const RELOAD_ENDPOINT_TAG: &str = "reload"; pub const GET_HEADER_STREAM_ENDPOINT_TAG: &str = "get_header_stream"; +pub const GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG: &str = "get_execution_payload_bid_stream"; /// For metrics recorded when a request times out pub const TIMEOUT_ERROR_CODE: u16 = 555; diff --git a/crates/pbs/src/dial.rs b/crates/pbs/src/dial.rs index d5a75a007..b93938bef 100644 --- a/crates/pbs/src/dial.rs +++ b/crates/pbs/src/dial.rs @@ -174,6 +174,7 @@ fn dial_config(url: Url) -> RelayConfig { id: None, headers: None, get_params: None, + // HTTP only: a stream's connection skips DialResolver's address check get_header: GetHeaderTransport::Http, enable_timing_games: false, target_first_request_ms: None, @@ -292,6 +293,7 @@ mod tests { (Ok(relay), Ok(url)) => { assert_eq!(relay.config.entry.url.as_str(), url, "{case}"); assert_eq!(relay.id.as_str(), "dial", "{case}"); + assert_eq!(relay.config.get_header, GetHeaderTransport::Http, "{case}"); } (Err(err), Err(message)) => assert_eq!(err.to_string(), message, "{case}"), _ => panic!("{case}: expected {expected:?}, got {:?}", res.map(|_| ())), diff --git a/crates/pbs/src/metrics.rs b/crates/pbs/src/metrics.rs index 47e93e36b..9745d53e4 100644 --- a/crates/pbs/src/metrics.rs +++ b/crates/pbs/src/metrics.rs @@ -52,9 +52,9 @@ lazy_static! { .unwrap(); - // THE WEBSOCKET BID STREAM + // THE WEBSOCKET BID STREAMS // Outcome and time-to-first-bid ride RELAY_STATUS_CODE / RELAY_LATENCY - // under `get_header_stream` + // under `get_header_stream` and `get_execution_payload_bid_stream` /// Websocket handshake latency by relay pub static ref RELAY_STREAM_CONNECT_LATENCY: HistogramVec = register_histogram_vec_with_registry!( @@ -67,8 +67,8 @@ lazy_static! { .unwrap(); /// Stream frames accepted on arrival per window, by relay - // What a frame must pass on arrival is the endpoint's: get_header_stream - // reads only its prefix, and validates the bids it holds at the deadline + // On arrival get_header_stream checks only the frame prefix, while + // get_execution_payload_bid_stream decodes the bid pub static ref RELAY_STREAM_UPDATES: HistogramVec = register_histogram_vec_with_registry!( "relay_stream_updates", "Stream frames accepted on arrival per window, by relay", diff --git a/crates/pbs/src/mev_boost/get_header.rs b/crates/pbs/src/mev_boost/get_header.rs index 4372f13bd..19067d521 100644 --- a/crates/pbs/src/mev_boost/get_header.rs +++ b/crates/pbs/src/mev_boost/get_header.rs @@ -41,7 +41,7 @@ use url::Url; use super::get_header_ws::get_header_ws; use crate::{ - bid_stream::record_stream_fallback, + bid_stream::record_stream_failure, constants::{ GET_HEADER_ENDPOINT_TAG, GET_HEADER_STREAM_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, TIMEOUT_ERROR_CODE, TIMEOUT_ERROR_CODE_STR, @@ -218,13 +218,10 @@ pub async fn get_header( } Ok(None) => {} Err(err) if transport == GetHeaderTransport::Stream => { - warn!(%err, relay_id, "stream failed"); - record_stream_fallback(GET_HEADER_STREAM_ENDPOINT_TAG, relay_id); + record_stream_failure(GET_HEADER_STREAM_ENDPOINT_TAG, relay_id, &err) } - Err(err) if err.is_timeout() => { - error!(err = "Timed Out", relay_id, transport = transport.as_str()) - } - Err(err) => error!(%err, relay_id, transport = transport.as_str()), + Err(err) if err.is_timeout() => error!(err = "Timed Out", relay_id), + Err(err) => error!(%err, relay_id), } } for (relay_id, value) in best_per_relay { @@ -751,10 +748,7 @@ mod tests { }; use futures::SinkExt; use ssz::Encode; - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - }; + use tokio::net::TcpListener; use tokio_tungstenite::{accept_async, tungstenite::Message}; use super::{validate_header_data, *}; @@ -895,21 +889,19 @@ mod tests { } } - /// A relay that answers every request, the stream handshake included, with - /// `status` and no body + /// A relay that answers the stream handshake with `status`, and its HTTP + /// get_header with 204, so only the stream can count as failed async fn start_status_relay(status: StatusCode) -> SocketAddr { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - while let Ok((mut socket, _)) = listener.accept().await { - tokio::spawn(async move { - let mut request = [0u8; 4096]; - let _ = socket.read(&mut request).await; - let response = format!("HTTP/1.1 {status}\r\ncontent-length: 0\r\n\r\n"); - let _ = socket.write_all(response.as_bytes()).await; - }); + let relay = axum::Router::new().fallback(move |headers: HeaderMap| async move { + if headers.contains_key(reqwest::header::UPGRADE) { + status + } else { + StatusCode::NO_CONTENT } }); + tokio::spawn(async move { axum::serve(listener, relay).await }); addr } @@ -938,9 +930,13 @@ mod tests { addr } - /// get_header against one stream relay at `addr`. Returns the relay's id, - /// which labels its metrics, and the bid. - async fn get_header_from_stream_relay(addr: SocketAddr) -> (String, Option) { + /// get_header for `slot` and `parent_hash` against one stream relay at + /// `addr`. Returns the relay's id, which labels its metrics, and the bid. + async fn get_header_from_stream_relay( + addr: SocketAddr, + slot: u64, + parent_hash: B256, + ) -> (String, Option) { let pubkey = BlsSecretKey::random().public_key(); let relay_id = format!("stream_relay_{}", addr.port()); let relay = RelayClient::new(RelayConfig { @@ -961,9 +957,11 @@ mod tests { .unwrap(); let mut pbs_config: PbsConfig = serde_json::from_str("{}").unwrap(); pbs_config.late_in_slot_time_ms = u64::MAX; + // The fixture bids are not signed by the relay's key + pbs_config.skip_sigverify = true; let state = PbsState::new( PbsModuleConfig { - chain: Chain::Hoodi, + chain: Chain::Holesky, endpoint: addr, pbs_config: Arc::new(pbs_config), relays: vec![relay.clone()], @@ -974,7 +972,7 @@ mod tests { }, Default::default(), ); - let params = GetHeaderParams { slot: 1, parent_hash: B256::ZERO, pubkey }; + let params = GetHeaderParams { slot, parent_hash, pubkey }; let bid = get_header(params, HeaderMap::new(), state).await.unwrap(); (relay_id, bid) @@ -984,7 +982,7 @@ mod tests { async fn test_stream_fallback_counts_a_refused_handshake_not_a_204() { for (status, fallbacks) in [(StatusCode::NO_CONTENT, 0), (StatusCode::NOT_FOUND, 1)] { let addr = start_status_relay(status).await; - let (relay_id, bid) = get_header_from_stream_relay(addr).await; + let (relay_id, bid) = get_header_from_stream_relay(addr, 1, B256::ZERO).await; assert!(bid.is_none()); assert_eq!( RELAY_STATUS_CODE @@ -1012,7 +1010,7 @@ mod tests { #[tokio::test] async fn test_stream_fallback_counts_a_window_with_no_valid_bid() { let addr = start_invalid_bid_stream_relay(10).await; - let (relay_id, bid) = get_header_from_stream_relay(addr).await; + let (relay_id, bid) = get_header_from_stream_relay(addr, 1, B256::ZERO).await; assert!(bid.is_none()); let status = RELAY_STATUS_CODE.with_label_values(&[ @@ -1025,4 +1023,72 @@ mod tests { assert_eq!(RELAY_STREAM_FALLBACK.with_label_values(&stream).get(), 1); assert_eq!(RELAY_STREAM_UPDATES.with_label_values(&stream).get_sample_sum(), 10.0); } + + /// The slot whose start is the fixture bid's timestamp, on Holesky + const FIXTURE_SLOT: u64 = 1_714_953; + + /// The fixture's get_header response bidding `gwei`, and its parent hash + fn fixture_bid_json(gwei: u64) -> (B256, Vec) { + let json = fs::read("../../tests/data/get_header/fulu.json").unwrap(); + let mut json: serde_json::Value = serde_json::from_slice(&json).unwrap(); + let wei = U256::from(gwei) * U256::from(1_000_000_000); + json["data"]["message"]["value"] = wei.to_string().into(); + let parent_hash = json["data"]["message"]["header"]["parent_hash"].as_str().unwrap(); + (parent_hash.parse().unwrap(), serde_json::to_vec(&json).unwrap()) + } + + /// A relay serving `http_body` as its HTTP get_header and `frame` on its + /// stream, both on one port + async fn start_dual_relay(http_body: Vec, frame: Vec) -> SocketAddr { + let http = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let http_addr = http.local_addr().unwrap(); + let relay = axum::Router::new().fallback(move || { + let body = http_body.clone(); + async move { ([(reqwest::header::CONTENT_TYPE, "application/json")], body) } + }); + tokio::spawn(async move { axum::serve(http, relay).await }); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let frame = frame.clone(); + tokio::spawn(async move { + let mut head = [0; 2048]; + let n = socket.peek(&mut head).await.unwrap_or_default(); + let head = String::from_utf8_lossy(&head[..n]).to_ascii_lowercase(); + if !head.contains("upgrade: websocket") { + let Ok(mut http) = tokio::net::TcpStream::connect(http_addr).await else { + return; + }; + let _ = tokio::io::copy_bidirectional(&mut socket, &mut http).await; + return; + } + let Ok(mut stream) = accept_async(socket).await else { return }; + let _ = stream.send(Message::Binary(frame.into())).await; + let _ = stream.close(None).await; + }); + } + }); + addr + } + + // A relay's gauges record the better of its HTTP and stream bids + #[tokio::test] + async fn test_relay_gauges_take_the_better_leg() { + for (http_gwei, stream_gwei) in [(60, 50), (50, 60)] { + let (parent_hash, http_bid) = fixture_bid_json(http_gwei); + let (_, stream_bid) = fixture_bid_json(stream_gwei); + // Message type bid, fork fulu + let mut frame = vec![0x01, 6]; + frame.extend(decode_json_payload(&stream_bid).unwrap().data.as_ssz_bytes()); + let addr = start_dual_relay(http_bid, frame).await; + + let (relay_id, bid) = + get_header_from_stream_relay(addr, FIXTURE_SLOT, parent_hash).await; + assert_eq!(*bid.expect("a bid").value(), U256::from(60_000_000_000u64)); + assert_eq!(RELAY_HEADER_VALUE.with_label_values(&[&relay_id]).get(), 60); + assert_eq!(RELAY_LAST_SLOT.with_label_values(&[&relay_id]).get(), FIXTURE_SLOT as i64); + } + } } diff --git a/crates/pbs/src/mev_boost/get_header_ws.rs b/crates/pbs/src/mev_boost/get_header_ws.rs index 52a7b5f25..3fd9bd20f 100644 --- a/crates/pbs/src/mev_boost/get_header_ws.rs +++ b/crates/pbs/src/mev_boost/get_header_ws.rs @@ -14,7 +14,7 @@ use url::Url; use super::get_header::{RequestInfo, decode_ssz_payload, validate_get_header_response}; use crate::{ - bid_stream::{Frame, Held, handshake_request, read_bid_stream}, + bid_stream::{Frame, Held, MAX_HELD_FRAMES, handshake_request, read_bid_stream}, constants::{GET_HEADER_STREAM_ENDPOINT_TAG, TRANSPORT_ERROR_STATUS}, metrics::RELAY_STATUS_CODE, }; @@ -49,7 +49,16 @@ async fn stream_header( }; let Held { frames, updates, connect_latency, first_frame_latency, invalid_frames } = - match read_bid_stream(request, deadline, relay, GET_HEADER_STREAM_ENDPOINT_TAG, Ok).await { + match read_bid_stream( + request, + deadline, + relay, + GET_HEADER_STREAM_ENDPOINT_TAG, + MAX_HELD_FRAMES, + Ok, + ) + .await + { Ok(held) => held, Err((status, err)) => return (status, Err(err)), }; diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index affdfcf10..31f7afe7e 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -7,10 +7,11 @@ use axum::{ response::{IntoResponse, Response}, }; use cb_common::{ + config::GetHeaderTransport, pbs::{ - GetExecutionPayloadBidParams, GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, - HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, SignedExecutionPayloadBid, - error::PbsError, + ForkName, GetExecutionPayloadBidParams, GetExecutionPayloadBidResponse, + HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, + SignedExecutionPayloadBid, error::PbsError, }, utils::{ms_into_slot, utcnow_ms}, wire::{ @@ -26,6 +27,7 @@ use reqwest::{ use ssz::{Decode, Encode}; use tracing::{debug, error, info, warn}; +use super::execution_payload_bid_ws::get_execution_payload_bid_ws; use crate::{ PbsStateGuard, constants::{ @@ -166,26 +168,78 @@ pub async fn get_execution_payload_bid( let slot = params.slot; let relay_id = relay.id.clone(); + let stream_url = relay.get_execution_payload_bid_stream_url( + params.slot, + ¶ms.parent_hash, + ¶ms.parent_root, + ¶ms.proposer_pubkey, + ); + let streams = stream_url.is_some(); + let (http, stream) = match stream_url { + None => ( + send_get_execution_payload_bid(params, body, relay, send_headers, max_timeout_ms).await, + None, + ), + Some(url) => tokio::join!( + send_get_execution_payload_bid( + params, + body.clone(), + relay.clone(), + send_headers.clone(), + max_timeout_ms, + ), + get_execution_payload_bid_ws(&body, &relay, &send_headers, url, max_timeout_ms) + ), + }; + // A relay that errors or times out contributes no bid: 204, never a 502. - // The builder's own 400 and 401 still reach the proposer. - match send_get_execution_payload_bid(params, body, relay, send_headers, max_timeout_ms).await { - Ok(Some(relay_bid)) => { - RELAY_LAST_SLOT.with_label_values(&[relay_id.as_str()]).set(slot as i64); - // The bid's value is already gwei, the gauge's unit, so it is set unscaled - RELAY_HEADER_VALUE - .with_label_values(&[relay_id.as_str()]) - .set(i64::try_from(relay_bid.bid.data.message.value).unwrap_or_default()); - Ok(Some(relay_bid)) - } - Ok(None) => Ok(None), + // The builder's own 400 and 401 reach the proposer when no bid does. + let http = match http { + Ok(bid) => bid, Err(err) if err.is_timeout() => { error!(err = "Timed Out", %relay_id, timeout_ms = max_timeout_ms); - Ok(None) + None } Err(err) => { error!(err = ?err, %relay_id); - builder_rejection(&err).map_or(Ok(None), Err) + match builder_rejection(&err) { + Some(rejection) if stream.is_none() => return Err(rejection), + _ => None, + } + } + }; + + let selected = select_bid(http, stream); + if streams && let Some((_, transport)) = &selected { + info!(transport = transport.as_str(), "selected bid"); + } + let relay_bid = selected.map(|(relay_bid, _)| relay_bid); + if let Some(relay_bid) = &relay_bid { + RELAY_LAST_SLOT.with_label_values(&[relay_id.as_str()]).set(slot as i64); + // The bid's value is already gwei, the gauge's unit, so it is set unscaled + RELAY_HEADER_VALUE + .with_label_values(&[relay_id.as_str()]) + .set(i64::try_from(relay_bid.bid.data.message.value).unwrap_or_default()); + } + Ok(relay_bid) +} + +/// The higher `value + execution_payment` and the leg it came from, the +/// stream's bid on a tie. Unclamped: the beacon node applies its own cap. +fn select_bid( + http: Option, + stream: Option, +) -> Option<(RelayBid, GetHeaderTransport)> { + let total = |relay_bid: &RelayBid| { + let bid = &relay_bid.bid.data.message; + bid.value.saturating_add(bid.execution_payment) + }; + match (http, stream) { + (Some(http), Some(stream)) if total(&http) > total(&stream) => { + Some((http, GetHeaderTransport::Http)) } + (http, None) => http.map(|bid| (bid, GetHeaderTransport::Http)), + (_, stream) => stream.map(|bid| (bid, GetHeaderTransport::Stream)), } } @@ -267,15 +321,17 @@ async fn send_get_execution_payload_bid( return Ok(None); } - let bid = match content_type { + let relay_bid = match content_type { EncodingType::Json => { - serde_json::from_slice(&response_bytes).map_err(|err| PbsError::JsonDecode { - err, - raw: String::from_utf8_lossy( - &response_bytes[..response_bytes.len().min(MAX_SIZE_DEFAULT)], - ) - .into_owned(), - })? + let bid = + serde_json::from_slice(&response_bytes).map_err(|err| PbsError::JsonDecode { + err, + raw: String::from_utf8_lossy( + &response_bytes[..response_bytes.len().min(MAX_SIZE_DEFAULT)], + ) + .into_owned(), + })?; + RelayBid { bid, body: response_bytes.into(), encoding: EncodingType::Json } } EncodingType::Ssz => { // SSZ requires the fork from Eth-Consensus-Version; its absence is a @@ -285,24 +341,48 @@ async fn send_get_execution_payload_bid( .to_string(), code: code.as_u16(), })?; - let data = - SignedExecutionPayloadBid::from_ssz_bytes(&response_bytes).map_err(|err| { - PbsError::SSZDecode { - err: format!("error decoding relay payload: {err:?}"), - fork, - } - })?; - GetExecutionPayloadBidResponse { version: fork, data, metadata: Default::default() } + decode_ssz_bid(response_bytes.into(), fork)? } }; - Ok(Some(RelayBid { bid, body: response_bytes.into(), encoding: content_type })) + Ok(Some(relay_bid)) +} + +pub(super) fn decode_ssz_bid(body: Bytes, fork: ForkName) -> Result { + let data = SignedExecutionPayloadBid::from_ssz_bytes(&body).map_err(|err| { + PbsError::SSZDecode { err: format!("error decoding relay payload: {err:?}"), fork } + })?; + let bid = GetExecutionPayloadBidResponse { version: fork, data, metadata: Default::default() }; + Ok(RelayBid { bid, body, encoding: EncodingType::Ssz }) } #[cfg(test)] mod tests { + use std::{net::SocketAddr, sync::Arc}; + + use alloy::primitives::B256; + use cb_common::{ + config::{PbsModuleConfig, RelayConfig}, + pbs::{BuilderRequestAuth, ExecutionPayloadBid, RelayEntry}, + types::{BlsSecretKey, BlsSignature, Chain}, + }; + use futures::SinkExt; + use lh_types::Slot; + use prometheus::TextEncoder; + use tokio::net::TcpListener; + use tokio_tungstenite::{accept_async, tungstenite::Message}; use super::*; + use crate::{ + constants::{ + GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, TIMEOUT_ERROR_STATUS, + TRANSPORT_ERROR_STATUS, + }, + metrics::{ + PBS_METRICS_REGISTRY, RELAY_LATENCY, RELAY_STATUS_CODE, RELAY_STREAM_CONNECT_LATENCY, + RELAY_STREAM_FALLBACK, RELAY_STREAM_INVALID_FRAMES, RELAY_STREAM_UPDATES, + }, + }; #[test] fn test_request_budget_ms() { @@ -355,4 +435,257 @@ mod tests { )); } } + + fn relay_bid(value: u64, execution_payment: u64) -> RelayBid { + let data = SignedExecutionPayloadBid { + message: ExecutionPayloadBid { value, execution_payment, ..Default::default() }, + signature: BlsSignature::empty(), + }; + let bid = GetExecutionPayloadBidResponse { + version: ForkName::Gloas, + data, + metadata: Default::default(), + }; + RelayBid { bid, body: Bytes::new(), encoding: EncodingType::Ssz } + } + + // (http, stream, served): each bid a `(value, execution_payment)` + #[test] + fn test_select_bid() { + for (http, stream, served) in [ + // The HTTP bid's execution payment counts as much as the stream's + (Some((10, 60)), Some((50, 0)), Some((GetHeaderTransport::Http, (10, 60)))), + // The stream wins a tie + (Some((50, 0)), Some((10, 40)), Some((GetHeaderTransport::Stream, (10, 40)))), + // A total past u64::MAX saturates rather than wrapping to a low one + (Some((u64::MAX, 1)), Some((1, 0)), Some((GetHeaderTransport::Http, (u64::MAX, 1)))), + (None, Some((1, 0)), Some((GetHeaderTransport::Stream, (1, 0)))), + (Some((1, 0)), None, Some((GetHeaderTransport::Http, (1, 0)))), + (None, None, None), + ] { + let selected = select_bid( + http.map(|(v, p)| relay_bid(v, p)), + stream.map(|(v, p)| relay_bid(v, p)), + ) + .map(|(relay_bid, transport)| { + let bid = &relay_bid.bid.data.message; + (transport, (bid.value, bid.execution_payment)) + }); + assert_eq!(selected, served, "http {http:?}, stream {stream:?}"); + } + } + + /// A relay that answers every request, the stream handshake included, with + /// `status` and no body + async fn start_status_relay(status: StatusCode) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let relay = axum::Router::new().fallback(move || async move { status }); + tokio::spawn(async move { axum::serve(listener, relay).await }); + addr + } + + /// A relay whose stream sends `messages`, `gap` apart, then closes, or with + /// `close` false drops the connection with no close frame. Its HTTP request + /// fails. + async fn start_stream_relay(messages: Vec, gap: Duration, close: bool) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + while let Ok((socket, _)) = listener.accept().await { + let messages = messages.clone(); + tokio::spawn(async move { + let Ok(mut stream) = accept_async(socket).await else { return }; + for message in messages { + let _ = stream.send(message).await; + tokio::time::sleep(gap).await; + } + if close { + let _ = stream.close(None).await; + } + }); + } + }); + addr + } + + /// A relay that accepts the connection and never answers the handshake + async fn start_silent_relay() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((socket, _)) = listener.accept().await { + held.push(socket); + } + }); + addr + } + + fn bid_frame(value: u64, execution_payment: u64) -> Message { + let data = relay_bid(value, execution_payment).bid.data; + // Message type bid, fork gloas + let mut frame = vec![0x01, 7]; + frame.extend(data.as_ssz_bytes()); + Message::Binary(frame.into()) + } + + /// An ePBS bid request for slot 1 addressed to one stream relay at `addr`, + /// with `budget_ms` until the beacon node's deadline. Returns the relay's + /// id, which labels its metrics, and the bid. + async fn bid_from_stream_relay(addr: SocketAddr, budget_ms: u64) -> (String, Option) { + let pubkey = BlsSecretKey::random().public_key(); + let relay_id = format!("epbs_stream_relay_{}", addr.port()); + let relay = RelayClient::new(RelayConfig { + entry: RelayEntry { + id: relay_id.clone(), + pubkey: pubkey.clone(), + url: format!("http://{pubkey}@{addr}").parse().unwrap(), + }, + id: None, + headers: None, + get_params: None, + get_header: GetHeaderTransport::Stream, + enable_timing_games: false, + target_first_request_ms: None, + frequency_get_header_ms: None, + validator_registration_batch_size: None, + }) + .unwrap(); + let state = PbsState::new( + PbsModuleConfig { + chain: Chain::Hoodi, + endpoint: addr, + pbs_config: Arc::new(serde_json::from_str("{}").unwrap()), + relays: vec![relay.clone()], + all_relays: vec![relay], + signer_client: None, + registry_muxes: None, + mux_lookup: None, + }, + Default::default(), + ); + let params = GetExecutionPayloadBidParams { + slot: 1, + parent_hash: B256::ZERO, + parent_root: B256::ZERO, + proposer_pubkey: pubkey, + }; + let auth = SignedBuilderRequestAuth { + message: BuilderRequestAuth { + data: b"127.0.0.1".to_vec().try_into().unwrap(), + slot: Slot::new(1), + }, + signature: BlsSignature::empty(), + }; + let mut req_headers = HeaderMap::new(); + req_headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); + req_headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(budget_ms)); + + let bid = get_execution_payload_bid(params, auth, req_headers, state).await.unwrap(); + (relay_id, bid) + } + + const STREAM: &str = GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG; + const ZERO: Duration = Duration::ZERO; + + fn status_count(status: StatusCode, relay_id: &str) -> u64 { + RELAY_STATUS_CODE.with_label_values(&[status.as_str(), STREAM, relay_id]).get() + } + + fn failed_streams(relay_id: &str) -> u64 { + RELAY_STREAM_FALLBACK.with_label_values(&[STREAM, relay_id]).get() + } + + // A handshake answered 204 is no bid. A refused handshake, one that runs + // out the window, a stream that breaks before a bid, and a frame over the + // size cap are failed streams. + #[tokio::test] + async fn test_epbs_stream_failures_by_status() { + let oversized = Message::Binary(vec![0; MAX_SIZE_GET_HEADER_RESPONSE + 1].into()); + for (relay, status, failed) in [ + (start_status_relay(StatusCode::NO_CONTENT).await, StatusCode::NO_CONTENT, 0), + (start_status_relay(StatusCode::NOT_FOUND).await, StatusCode::NOT_FOUND, 1), + (start_silent_relay().await, TIMEOUT_ERROR_STATUS, 1), + (start_stream_relay(vec![], ZERO, false).await, TRANSPORT_ERROR_STATUS, 1), + (start_stream_relay(vec![oversized], ZERO, true).await, TRANSPORT_ERROR_STATUS, 1), + ] { + let (relay_id, bid) = bid_from_stream_relay(relay, 300).await; + assert!(bid.is_none(), "{status}"); + assert_eq!(status_count(status, &relay_id), 1, "{status}"); + assert_eq!(failed_streams(&relay_id), failed, "{status}"); + } + } + + // A window whose every bid fails to decode is no bid, not a failed stream, + // and counts each of those bids as an invalid frame + #[tokio::test] + async fn test_epbs_stream_undecodable_bids_are_no_bid() { + for n_frames in [1, 3] { + let undecodable = Message::Binary(vec![0x01, 7, 1, 2, 3].into()); + let relay = start_stream_relay(vec![undecodable; n_frames], ZERO, true).await; + let (relay_id, bid) = bid_from_stream_relay(relay, 1_000).await; + assert!(bid.is_none()); + + assert_eq!(status_count(StatusCode::NO_CONTENT, &relay_id), 1); + assert_eq!(failed_streams(&relay_id), 0); + let stream = [STREAM, relay_id.as_str()]; + assert_eq!( + RELAY_STREAM_INVALID_FRAMES.with_label_values(&stream).get(), + n_frames as u64 + ); + assert_eq!(RELAY_STREAM_UPDATES.with_label_values(&stream).get_sample_sum(), 0.0); + } + } + + // Messages other than bids do not end the stream. The served bid keeps the + // frame's fork and sets the relay's gauges: its value without the execution + // payment, and the slot. The latency series times the first bid. + #[tokio::test] + async fn test_epbs_stream_bid_after_other_messages_is_served_and_recorded() { + let messages = vec![ + bid_frame(30, 0), + Message::Text("hello".into()), + Message::Ping(vec![1].into()), + bid_frame(77, 5), + ]; + let relay = start_stream_relay(messages, Duration::from_millis(150), true).await; + let (relay_id, bid) = bid_from_stream_relay(relay, 2_000).await; + let bid = bid.expect("the stream's bid").bid; + assert_eq!(bid.version, ForkName::Gloas); + let bid = &bid.data.message; + assert_eq!((bid.value, bid.execution_payment), (77, 5)); + + assert_eq!(status_count(StatusCode::OK, &relay_id), 1); + assert_eq!(RELAY_HEADER_VALUE.with_label_values(&[&relay_id]).get(), 77); + assert_eq!(RELAY_LAST_SLOT.with_label_values(&[&relay_id]).get(), 1); + let stream = [STREAM, relay_id.as_str()]; + assert_eq!(RELAY_STREAM_UPDATES.with_label_values(&stream).get_sample_sum(), 2.0); + assert_eq!(RELAY_STREAM_CONNECT_LATENCY.with_label_values(&stream).get_sample_count(), 1); + // The last bid arrives 450 ms in + let latency = RELAY_LATENCY.with_label_values(&stream); + assert_eq!(latency.get_sample_count(), 1); + assert!(latency.get_sample_sum() < 0.3, "{}", latency.get_sample_sum()); + } + + // Dashboards and the docs select the stream series by label name + #[tokio::test] + async fn test_epbs_stream_series_are_labeled_by_name() { + let messages = vec![Message::Binary(vec![0x01, 7, 1].into()), bid_frame(1, 0)]; + let (relay_id, _) = + bid_from_stream_relay(start_stream_relay(messages, ZERO, true).await, 1_000).await; + let (failed_id, _) = + bid_from_stream_relay(start_status_relay(StatusCode::NOT_FOUND).await, 300).await; + + let scrape = TextEncoder::new().encode_to_string(&PBS_METRICS_REGISTRY.gather()).unwrap(); + for (series, relay_id) in [ + ("relay_stream_connect_latency_count", &relay_id), + ("relay_stream_updates_sum", &relay_id), + ("relay_stream_invalid_frames_total", &relay_id), + ("relay_stream_fallback_total", &failed_id), + ] { + let series = format!(r#"{series}{{endpoint="{STREAM}",relay_id="{relay_id}"}}"#); + assert!(scrape.contains(&series), "{series}"); + } + } } diff --git a/crates/pbs/src/routes/execution_payload_bid_ws.rs b/crates/pbs/src/routes/execution_payload_bid_ws.rs new file mode 100644 index 000000000..fb5987df9 --- /dev/null +++ b/crates/pbs/src/routes/execution_payload_bid_ws.rs @@ -0,0 +1,98 @@ +//! The ePBS bid over the bid stream of a relay with `get_header = "stream"`. +//! Each frame carries an SSZ `SignedExecutionPayloadBid`. + +use std::time::Duration; + +use axum::http::{HeaderMap, HeaderValue}; +use base64::{Engine, prelude::BASE64_STANDARD}; +use cb_common::{ + pbs::{HEADER_REQUEST_AUTH, RelayClient, error::PbsError}, + wire::CONSENSUS_VERSION_HEADER, +}; +use reqwest::StatusCode; +use tokio::time::Instant; +use tracing::debug; +use url::Url; + +use super::execution_payload_bid::{RelayBid, decode_ssz_bid}; +use crate::{ + bid_stream::{Frame, Held, handshake_request, read_bid_stream, record_stream_failure}, + constants::{GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, TRANSPORT_ERROR_STATUS}, + metrics::RELAY_STATUS_CODE, +}; + +type StreamOutcome = (StatusCode, Result, PbsError>); + +/// The relay's latest bid at the deadline. A failed stream is no bid. +pub(super) async fn get_execution_payload_bid_ws( + auth_ssz: &[u8], + relay: &RelayClient, + send_headers: &HeaderMap, + url: Url, + timeout_ms: u64, +) -> Option { + let (status, res) = + stream_execution_payload_bid(auth_ssz, relay, send_headers, url, timeout_ms).await; + let endpoint = GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG; + RELAY_STATUS_CODE.with_label_values(&[status.as_str(), endpoint, &relay.id]).inc(); + res.unwrap_or_else(|err| { + record_stream_failure(endpoint, &relay.id, &err); + None + }) +} + +async fn stream_execution_payload_bid( + auth_ssz: &[u8], + relay: &RelayClient, + send_headers: &HeaderMap, + url: Url, + timeout_ms: u64, +) -> StreamOutcome { + let deadline = Instant::now() + Duration::from_millis(timeout_ms); + let mut request = match handshake_request(&url, relay, send_headers) { + Ok(request) => request, + Err(err) => return (TRANSPORT_ERROR_STATUS, Err(err)), + }; + let headers = request.headers_mut(); + if let Some(fork) = send_headers.get(CONSENSUS_VERSION_HEADER) { + headers.insert(CONSENSUS_VERSION_HEADER, fork.clone()); + } + let mut request_auth = HeaderValue::try_from(BASE64_STANDARD.encode(auth_ssz)) + .expect("base64 is always a valid header value"); + request_auth.set_sensitive(true); + headers.insert(HEADER_REQUEST_AUTH, request_auth); + + // Each frame is decoded on arrival, so the newest is the latest bid that + // decodes and the only one to hold + let parse = |frame: Frame| decode_ssz_bid(frame.bid, frame.fork); + let Held { mut frames, updates, connect_latency, first_frame_latency, invalid_frames } = + match read_bid_stream( + request, + deadline, + relay, + GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, + 1, + parse, + ) + .await + { + Ok(held) => held, + Err((status, err)) => return (status, Err(err)), + }; + + let Some(bid) = frames.pop_back() else { + debug!(relay_id = relay.id.as_ref(), ?connect_latency, invalid_frames, "no bid"); + return (StatusCode::NO_CONTENT, Ok(None)); + }; + + debug!( + relay_id = relay.id.as_ref(), + ?connect_latency, + first_bid_latency = ?first_frame_latency, + updates, + invalid_frames, + "received new bid from ws stream" + ); + + (StatusCode::OK, Ok(Some(bid))) +} diff --git a/crates/pbs/src/routes/mod.rs b/crates/pbs/src/routes/mod.rs index 595552f58..174b5d08a 100644 --- a/crates/pbs/src/routes/mod.rs +++ b/crates/pbs/src/routes/mod.rs @@ -1,5 +1,6 @@ mod builder_preferences; mod execution_payload_bid; +mod execution_payload_bid_ws; mod get_header; mod register_validator; mod reload; diff --git a/docs/docs/get_started/configuration.md b/docs/docs/get_started/configuration.md index 2a55d7e2a..c5d486168 100644 --- a/docs/docs/get_started/configuration.md +++ b/docs/docs/get_started/configuration.md @@ -75,7 +75,7 @@ Each `[[relays]]` entry supports, besides `id` and `url`: - `headers`: custom headers sent with every request to this relay, which is how a relay API key is supplied. See [Relay API keys](#relay-api-keys). - `get_params`: optional GET parameters to add to each request URL for this relay. -- `get_header`: how bids are fetched from this relay, either `"http"` (one request per `get_header`) or `"stream"` (that request plus a websocket stream of bid updates, for relays that support it). Default: `"http"`. See [Bid streaming](#bid-streaming). +- `get_header`: how bids are fetched from this relay: `"http"`, one request per `get_header`, or `"stream"`, a websocket stream of bid updates for relays that support it. (unreleased, from v0.12.0-rc1) A streaming relay also gets the HTTP request. Default: `"http"`; details under [Bid streaming](#bid-streaming). - `enable_timing_games`: whether to enable timing games for this relay, as tuned by `target_first_request_ms` and `frequency_get_header_ms`. If neither of those is set, this flag has no effect. Advanced users only: misconfiguration can result in e.g. fetching a lower header value or missing a slot (caveats and worked examples in the annotated config example). Default: `false`. - `target_first_request_ms`: target time in the slot, in milliseconds, at which to send the first `get_header` request. - `frequency_get_header_ms`: frequency, in milliseconds, at which to send `get_header` requests. @@ -110,8 +110,20 @@ Under Docker, `commit-boost init` mounts each `file` path into the container at ### Bid streaming +:::info Unreleased +From v0.12.0-rc1, compared with v0.11.0: +- The relay's HTTP request runs alongside every stream. In v0.11.0 it runs only when the handshake fails with time left. +- The stream serves its latest valid bid. In v0.11.0 only the last bid is validated. +- A `204` answer to the handshake is no bid. In v0.11.0 it records `556` and retries over HTTP. +- `X-Timeout-Ms` on the handshake leaves out half the TCP connect time. In v0.11.0 it is the whole window. +- A failed stream logs `stream failed`. In v0.11.0 it logs `stream failed, falling back to http get_header`, and only when it retries. +- A relay also listed with `get_header = "http"`, as the v0.11.0 docs advised, gets two HTTP requests per `get_header`, and after the Gloas fork only the first of the two serves ePBS; remove the `"http"` entry. +::: + `get_header = "stream"` adds a websocket stream of bid updates to the relay's per-slot HTTP request. It is set per relay, on `[[relays]]` and `[[mux.relays]]` entries alike. [`examples/configs/pbs_bid_stream.toml`](https://github.com/Commit-Boost/commit-boost-client/blob/main/examples/configs/pbs_bid_stream.toml) streams from two default relays and two mux relays. +(unreleased, from v0.12.0-rc1) After the Gloas fork the same setting also streams [ePBS bids](./epbs.md#bid-streaming). + #### Connection For each `get_header` call PBS dials: @@ -120,7 +132,7 @@ For each `get_header` call PBS dials: ws(s):///eth/v1/builder/header_stream/{slot}/{parent_hash}/{pubkey} ``` -The scheme and host come from the relay's `url`: `https` becomes `wss`, `http` becomes `ws`, any other scheme fails at startup. The pubkey is dropped from the userinfo and `get_params` are appended as usual. +The scheme and host come from the relay's `url`: `https` becomes `wss`, `http` becomes `ws`, any other scheme fails at startup. The pubkey is dropped from the userinfo and `get_params` are appended as usual. The stream connects directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`. The handshake carries what the HTTP request would: slot, parent hash and pubkey in the path, plus `Date-Milliseconds`, `X-Timeout-Ms`, `X-CommitBoost-Version` and every configured header. `X-Timeout-Ms` is the time left in the bid window minus half the TCP connect time, so the relay's last update arrives before the window closes. @@ -128,7 +140,7 @@ The stream needs the same API key as the HTTP path, sent on the handshake with t #### Bid window -Once connected, the relay sends one binary frame per bid update. When the window ends or the relay closes the stream, PBS returns the latest update that passes validation, checking the held updates newest first. Validation is the same as on the HTTP path, so `skip_sigverify`, `min_bid_eth` and `extra_validation_enabled` apply unchanged. Frames that do not parse as a bid are skipped and counted. +Once connected, the relay sends one binary frame per bid update. PBS keeps the last 8 bids and, when the window ends or the relay closes the stream, returns the newest one that passes validation. Validation is the same as on the HTTP path, so `skip_sigverify`, `min_bid_eth` and `extra_validation_enabled` apply unchanged. Frames whose message type or fork byte cannot be read are skipped and counted. The window is the deadline the HTTP path already computes: `timeout_get_header_ms`, capped by the time left until `late_in_slot_time_ms`, and capped again by the CL's `X-Timeout-Ms` if it sends one. An HTTP `get_header` normally returns early, while a stream is held to the end, so on a streaming relay `timeout_get_header_ms` sets how long PBS holds the CL's request. A CL request arriving at or after `late_in_slot_time_ms` still skips relays entirely and forces local building. @@ -136,13 +148,13 @@ The timing-game options have no effect on the stream, which already delivers eve #### HTTP request -PBS sends the relay its normal HTTP `get_header` every slot, alongside the stream. The stream's bid and the HTTP bid are separate candidates and the better one wins, so a failed stream leaves the HTTP bid in the auction. The `auction winner` log line names the winning `transport`. +PBS sends the relay its normal HTTP `get_header` alongside the stream. The stream's bid and the HTTP bid are separate candidates and the better one wins, so a failed stream leaves the HTTP bid in the auction. The `auction winner` log line names the winning `transport`. -A stream fails when the handshake times out or does not upgrade, when it breaks before any bid arrives, or when every bid it held is invalid. If it breaks after a bid arrived, PBS validates the held bids as usual. A `204` answer to the handshake means the relay has no bid, not a failure. +A stream fails when it cannot connect, when the handshake times out or the relay refuses it, when the connection drops before any bid arrives, or when none of the bids PBS kept passes validation. Commit-Boost logs each failed stream as `stream failed` with the reason, and counts it in `cb_pbs_relay_stream_fallback_total`. A drop after a bid arrived is not a failure: PBS validates the bids it kept as usual. A `204` answer to the handshake means the relay has no bid. -#### Metrics +#### Stream metrics -Stream and HTTP outcomes are recorded separately. `cb_pbs_relay_status_code_total` and `cb_pbs_relay_latency` carry `endpoint="get_header_stream"` for the stream and `endpoint="get_header"` for the HTTP request, and four `cb_pbs_relay_stream_*` series cover handshake latency, updates per window, unparseable frames and failed streams. See [Metrics > Bid stream](./running/metrics.md#bid-stream). +Stream and HTTP outcomes are recorded separately. `cb_pbs_relay_status_code_total` and `cb_pbs_relay_latency` carry `endpoint="get_header_stream"` for the stream and `endpoint="get_header"` for the HTTP request, and four `cb_pbs_relay_stream_*` series cover handshake latency, updates per window, unparseable frames and failed streams; [Metrics > Bid stream](./running/metrics.md#bid-stream) says what each value means. ### SSZ support diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 1c14de6c8..b763c0dae 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -27,7 +27,7 @@ The ePBS bid endpoint does not use these PBS options, which will be deprecated a - `skip_sigverify`, `min_bid_eth` and `extra_validation_enabled`: the beacon node checks the bid against the on-chain builder registry and applies the `min_bid` from its builder config. - `timeout_get_header_ms`, `late_in_slot_time_ms`, their `[[mux]]` overrides and the timing-games options: the beacon node's deadline bounds the request (see [Timing](#timing)). -A `get_header = "stream"` relay is asked for ePBS bids over plain HTTP. +For a builder whose relay entry sets `get_header = "stream"`, Commit-Boost also reads its bids from a [stream](#bid-streaming). ## How a bid request reaches a builder @@ -92,16 +92,29 @@ Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon nod Builder preferences use `timeout_register_validator_ms`, and the signed block `timeout_get_payload_ms`. +## Bid streaming + +A builder answers an HTTP bid request once, so a better bid it makes after answering never reaches the beacon node. Over a websocket stream it can keep sending bids until just before the beacon node's deadline. + +Ask the builder whether it serves the ePBS bid stream, at `ws(s):///eth/v1/builder/execution_payload_bid_stream/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}`. If it does, set `get_header = "stream"` on its relay entry. Before the fork the same setting reads the builder's `get_header` bids from its [PBS stream](./configuration.md#bid-streaming), so for a builder that serves only one of the two, Commit-Boost logs a failed stream on each request to the other and uses the HTTP bid. The handshake carries the relay entry's `headers`, so a builder that requires an [API key](./configuration.md#relay-api-keys) gets it. + +For each bid request to a streaming builder, Commit-Boost sends the usual HTTP request and opens the stream at the same time. It returns the HTTP bid or the latest streamed bid that decodes, whichever has the higher `value + execution_payment`, and the streamed bid on a tie. Commit-Boost does not apply the key's `max_execution_payment` to this choice; the beacon node applies it afterwards. If the stream fails, the HTTP bid still reaches the beacon node. A builder [outside your config](#builders-outside-your-config) is always asked over HTTP. + +Streaming makes bid requests slower. Commit-Boost replies to the beacon node once the builder's HTTP response is in and the stream has ended: when the builder closes it, when it fails, or at the beacon node's deadline less `proposer_deadline_buffer_ms`. An HTTP builder is answered as soon as it responds. `timeout_get_header_ms` does not shorten the wait. If the beacon node reports bid requests timing out on streaming builders, raise `proposer_deadline_buffer_ms`. + +The stream [connects directly](./configuration.md#connection), without your proxy settings, while the HTTP request uses them. If your host reaches builders only through a proxy, keep them on `"http"`. + ## Metrics -With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoint` labels `get_execution_payload_bid`, `submit_builder_preferences` and `submit_signed_beacon_block`. +With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoint` labels `get_execution_payload_bid`, `submit_builder_preferences` and `submit_signed_beacon_block`, and a [streaming](#bid-streaming) builder's stream uses `get_execution_payload_bid_stream`. | Question | Series | |---|---| | What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences, `202` for the signed block whatever the builders answer, `4xx` for a rejected request, `500` when no builder accepted the preferences | | What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"`, except one Commit-Boost [refuses to dial](#builders-outside-your-config), which gets no request | | How fast are builders? | `cb_pbs_relay_latency`, by `relay_id` | -| What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from each bid a builder serves | +| Is a builder's bid stream working? | `cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid_stream"}`: `200` when the stream supplied a bid, `204` when it had none, any other code a failed stream: `555` the deadline ran out before the handshake completed, `556` a connection that failed or dropped before a bid, or the builder's own refusal such as `404`. When Commit-Boost returns a streaming builder's bid, it also logs `selected bid`, with `transport` set to `stream` or `http` for where that bid came from | +| What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from the bid Commit-Boost returns for each builder; with a streaming builder, whichever of its two bids it chose | ## Troubleshooting @@ -110,7 +123,9 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a hostname nor an `http(s)` URL, or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | | Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config). A key without builder config sends Commit-Boost's own hostname, which usually does | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | +| Bid requests to a streaming builder log "stream failed" with "rejected with 404", and `cb_pbs_relay_stream_fallback_total{endpoint="get_execution_payload_bid_stream"}` rises | The builder does not serve the ePBS bid stream. Its HTTP bid still reaches the beacon node | Set `get_header = "http"` on its relay entry | | Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | +| Bid requests to a streaming builder log "stream failed" with "websocket timed out" or a connection error | Commit-Boost could not open the stream before the deadline, for example because your network allows only proxied connections, which the stream does not use. The HTTP bid still reaches the beacon node | Keep the builder on `get_header = "http"`, or allow direct connections to it | +| Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | | The signed block gets `415` | The beacon node sends it as JSON | Configure the beacon node to send SSZ | | Commit-Boost logs "no builder accepted the signed beacon block" | The winning bid came from a builder outside your config, which gets the block over gossip, or your builders did not accept it. The beacon node gossips the block either way | If the bid came from a configured builder, check its answer in `cb_pbs_relay_status_code_total{endpoint="submit_signed_beacon_block"}` | -| Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/docs/get_started/running/metrics-catalog.md b/docs/docs/get_started/running/metrics-catalog.md index ca49cec50..325c1850c 100644 --- a/docs/docs/get_started/running/metrics-catalog.md +++ b/docs/docs/get_started/running/metrics-catalog.md @@ -14,14 +14,14 @@ PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. Th | Metric name (wire) | Type | Labels | Description | |---|---|---|---| -| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (timeout, DNS or connection failure), and `"556"` for a bid-stream transport failure (connect failed, the stream broke before a bid, or the handshake was answered with a `2xx` other than `204` instead of the upgrade). Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. | -| `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Under `get_header_stream` the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | +| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (over HTTP a timeout, DNS or connection failure; on a stream, the bid window ran out before the handshake completed), and `"556"` for a bid-stream transport failure (connect failed, the stream broke before a bid, or the handshake was answered with a `2xx` other than `204`). (unreleased, from v0.12.0-rc1) A `204` answer to the handshake records `204`. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. | +| `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. Under the two stream endpoints the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | | `cb_pbs_relay_last_slot` | Gauge | `relay_id` | Latest slot for which a relay delivered a bid. Set to the current slot on each successful `get_header` bid and (unreleased, from v0.12.0-rc1) each ePBS bid served from that relay. | -| `cb_pbs_relay_header_value` | Gauge | `relay_id` | Value in gwei of the bid a relay delivered: for `get_header`, the bid's value converted from wei (รท 1e9). (unreleased, from v0.12.0-rc1) For ePBS, the returned bid's `value`, without its execution payment. | -| `cb_pbs_relay_stream_connect_latency` | Histogram | `endpoint`, `relay_id` | Websocket handshake latency in seconds, for a relay with `get_header = "stream"`. Observed once per successful handshake; a handshake that fails or runs out the bid window records nothing here. Custom buckets from 5 ms to 1 s. Endpoint value: `get_header_stream`. | -| `cb_pbs_relay_stream_updates` | Histogram | `endpoint`, `relay_id` | Bid updates received per stream window. Observed once per `get_header` served by the stream, including windows that received nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. Endpoint value: `get_header_stream`. | -| `cb_pbs_relay_stream_invalid_frames_total` | Counter | `endpoint`, `relay_id` | Websocket frames whose message type or fork byte cannot be read. Incremented only for windows that saw at least one, so the series is absent while zero. Endpoint value: `get_header_stream`. | -| `cb_pbs_relay_stream_fallback_total` | Counter | `endpoint`, `relay_id` | Bid streams that failed while the relay's HTTP `get_header` ran alongside: a handshake that timed out or did not upgrade, a stream that broke before a bid, or a window in which every held bid was invalid. A `204` answer to the handshake is no bid and does not count. Counts every failed stream, whether or not the HTTP request returned a bid; the HTTP result is recorded under `endpoint="get_header"`. Endpoint value: `get_header_stream`. | +| `cb_pbs_relay_header_value` | Gauge | `relay_id` | Value in gwei of the bid a relay delivered: for `get_header`, the bid's value converted from wei (รท 1e9). (unreleased, from v0.12.0-rc1) For a streaming relay, the better of its HTTP and stream bids. For ePBS, the returned bid's `value`, without its execution payment. | +| `cb_pbs_relay_stream_connect_latency` | Histogram | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Websocket handshake latency in seconds, for a relay with `get_header = "stream"`. Observed once per successful handshake; a handshake that fails, runs out the bid window, or is answered `204` records nothing here. Custom buckets from 5 ms to 1 s. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | +| `cb_pbs_relay_stream_updates` | Histogram | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Bid updates accepted on arrival per stream: on `get_header_stream`, every frame whose message type and fork byte can be read; on `get_execution_payload_bid_stream`, every bid that decodes. Observed once per stream that connected, including streams that received nothing; a handshake that fails, times out or is answered `204` records nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | +| `cb_pbs_relay_stream_invalid_frames_total` | Counter | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Websocket frames whose message type or fork byte cannot be read; on the ePBS stream, also bids that fail to decode. Incremented only for windows that saw at least one, so the series is absent while zero. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | +| `cb_pbs_relay_stream_fallback_total` | Counter | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | (unreleased, from v0.12.0-rc1; in v0.11.0, handshake failures retried over HTTP) Bid streams that failed while the relay's HTTP request ran alongside: a connection or handshake that failed or timed out, a stream that broke before a bid, or, on `get_header_stream`, a window in which every held bid was invalid. A `204` answer to the handshake is no bid and does not count. Counts every failed stream, whether or not the HTTP request returned a bid; the HTTP result is recorded under `endpoint="get_header"` (or `get_execution_payload_bid`). Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | | `cb_pbs_beacon_node_status_code_total` | Counter | `http_status_code`, `endpoint` | HTTP status code returned to the beacon node. Tracks what status codes the PBS returns for beacon node-facing requests. Endpoint values: `get_header`, `register_validator`, `submit_blinded_block`, `status`, `reload`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Error status codes (`502` for `NoResponse`/`NoPayload`, `500` for `Internal`) are set via `PbsClientError`. The handlers also record `406` directly when the request's `Accept` header offers no supported encoding, `204` when no bid is available on `get_header`, and `202` for accepted v2 `submit_blinded_block` requests. (unreleased, from v0.12.0-rc1) The ePBS endpoints record `200` for a bid, `204` for none, `202` for accepted preferences and for every signed block it forwards, `400` for a request Commit-Boost refuses, a builder's own `400` or `401`, `406` for a bid request whose `Accept` offers no supported encoding, `415` for an unsupported `Content-Type`, and `500` when no builder accepts the preferences. | | `cb_pbs_pbs_submit_block_v2_unsupported_total` | Counter | `relay_id` | Count of v2 `submit_blinded_block` requests a relay could not serve because it returned 404 on the v2 endpoint. A non-zero value means the relay does not support `submitBlindedBlockV2` and those blocks were not submitted via that relay. The double `pbs` in the wire name comes from the registry prefix plus the metric name `pbs_submit_block_v2_unsupported_total`. | diff --git a/docs/docs/get_started/running/metrics.md b/docs/docs/get_started/running/metrics.md index f0a37c18d..1d5efdb6e 100644 --- a/docs/docs/get_started/running/metrics.md +++ b/docs/docs/get_started/running/metrics.md @@ -112,15 +112,25 @@ Once Grafana is running, you can [import](https://grafana.com/docs/grafana/lates ## Bid stream -For a relay with `get_header = "stream"` (see [Configuration > Bid streaming](../configuration.md#bid-streaming)), every series below is labeled by `relay_id` and `endpoint`. The stream's outcome and time-to-first-bid share the two general relay series under `endpoint="get_header_stream"`, and the HTTP request that runs alongside it every slot keeps `endpoint="get_header"`. +:::info Unreleased +From v0.12.0-rc1, compared with v0.11.0: +- The `cb_pbs_relay_stream_*` series carry an `endpoint` label. In v0.11.0 they carry `relay_id` only. +- `cb_pbs_relay_stream_fallback_total` counts every failed stream. In v0.11.0 it counts handshake failures retried over HTTP. +- A handshake answered `204` records `204`. In v0.11.0 it records `556`. +- A streaming relay's HTTP request records under `endpoint="get_header"` on every request. In v0.11.0 it records only on a fallback. +::: + +For a relay with `get_header = "stream"` (see [Configuration > Bid streaming](../configuration.md#bid-streaming)), every series below is labeled by `relay_id` and `endpoint`. The stream's outcome and time-to-first-bid share the two general relay series under `endpoint="get_header_stream"`, and the HTTP request that runs alongside it keeps `endpoint="get_header"`. | Question | Series | |---|---| -| Is the stream serving bids? | `cb_pbs_relay_status_code_total{endpoint="get_header_stream"}`: `200` a bid was delivered, `204` no bid (the handshake was answered `204`, or no bid arrived before the deadline), `555` the bid window ran out during the handshake, `556` a transport error (connect failed, the stream broke before a bid, or the handshake was answered with a `2xx` other than `204` instead of the upgrade), any other code the relay's own refusal of the upgrade | +| Is the stream serving bids? | `cb_pbs_relay_status_code_total{endpoint="get_header_stream"}`: `200` a bid was delivered, `204` no bid (the handshake was answered `204`, or the stream ended or the deadline passed without one), `555` the bid window ran out during the handshake, `556` a transport error (connect failed, the stream broke before a bid, or the handshake was answered with a `2xx` other than `204` instead of the upgrade), any other code the relay's own refusal of the upgrade | | How fast does the first bid arrive? | `cb_pbs_relay_latency{endpoint="get_header_stream"}` | -| Is the stream failing? | `cb_pbs_relay_stream_fallback_total{endpoint="get_header_stream"}`: streams that failed while the relay's HTTP request ran alongside: a handshake that could not connect, timed out or was refused, a stream that broke before a bid, or a window in which every held bid failed decoding or validation. A `204` is no bid, not a failure. It counts every failed stream, whether or not the HTTP request returned a bid; the HTTP results are under `endpoint="get_header"`. For a steady rate, the status series above gives the reason | +| Is the stream failing? | `cb_pbs_relay_stream_fallback_total{endpoint="get_header_stream"}`: [failed streams](../configuration.md#http-request), counted whether or not the HTTP request returned a bid. For a steady rate, the status series above and the `stream failed` log line give the reason | | Is the handshake slow? | `cb_pbs_relay_stream_connect_latency` | -| Is it actually streaming? | `cb_pbs_relay_stream_updates`: frames accepted per window, which is every frame whose message type and fork byte can be read; their bids are decoded and validated only when the window closes. A healthy relay sends several; windows that carry at most one update mean the stream connects but does not stream | +| Is it actually streaming? | `cb_pbs_relay_stream_updates`: bids received per request. On `get_header_stream` every frame with a readable message type and fork byte counts, before its bid is checked. A healthy relay sends several; at most one per request means the stream connects but does not stream | | Are the frames usable? | `cb_pbs_relay_stream_invalid_frames_total`: frames dropped on arrival because their message type or fork byte cannot be read, absent while zero. A bid that fails decoding or validation is not counted here | -When the window closes, the stream serves the latest bid that passes decoding and validation, from the newest 8 it holds. A failing bid newer than the one served is not counted as a failure; the error is in the logs. A window in which every held bid fails counts as `200` on the status series, the same as an invalid bid over HTTP, and as one failed stream. A request the beacon node abandons mid-window records no outcome at all. +When the newest bid fails validation and an earlier one is served, the failure is logged, not counted. A window in which every kept bid fails counts as `200` on the status series, the same as an invalid bid over HTTP, and as one failed stream. If the beacon node drops the request before the deadline, no status is recorded. + +The [ePBS bid stream](../epbs.md#bid-streaming) (unreleased, from v0.12.0-rc1) reports the same series under `endpoint="get_execution_payload_bid_stream"`, and its HTTP request under `endpoint="get_execution_payload_bid"`. It decodes each bid as it arrives and does not validate it, so `cb_pbs_relay_stream_updates` counts only bids that decode, and `cb_pbs_relay_stream_invalid_frames_total` also counts those that do not. A stream that ends with only undecodable bids is no bid (`204`), not a failed stream; if its connection drops instead, it records `556` and a failed stream. diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 772066868..6e7c876ac 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -7,6 +7,7 @@ version.workspace = true [dependencies] alloy.workspace = true axum.workspace = true +base64.workspace = true cb-common.workspace = true cb-pbs = { workspace = true, features = ["testing-flags"] } cb-signer.workspace = true diff --git a/tests/src/mock_relay.rs b/tests/src/mock_relay.rs index c1ad4b813..0a6505123 100644 --- a/tests/src/mock_relay.rs +++ b/tests/src/mock_relay.rs @@ -328,6 +328,10 @@ impl MockRelayState { } } + pub fn with_trustless_bid_gwei(self, trustless_bid_gwei: u64) -> Self { + Self { trustless_bid_gwei, ..self } + } + /// Hold every bid request `delay_ms` before answering, so a caller with a /// shorter `X-Timeout-Ms` times out. pub fn with_bid_delay_ms(self, delay_ms: u64) -> Self { @@ -394,6 +398,32 @@ pub fn mock_signed_builder_bid( SignedBuilderBid { message, signature } } +/// CB does not verify bid signatures (the beacon node does), so the mock +/// serves an unsigned bid +pub fn mock_execution_payload_bid( + slot: u64, + parent_hash: B256, + parent_root: B256, + value: u64, + execution_payment: u64, +) -> SignedExecutionPayloadBid { + let mut block_hash = B256::ZERO; + block_hash.0[0] = 1; + + let message = ExecutionPayloadBid { + parent_block_hash: parent_hash.into(), + parent_block_root: parent_root, + block_hash: block_hash.into(), + gas_limit: 30_000_000, + builder_index: 42, + slot: Slot::new(slot), + value, + execution_payment, + ..Default::default() + }; + SignedExecutionPayloadBid { message, signature: BlsSignature::empty() } +} + async fn handle_get_execution_payload_bid( State(state): State>, Path((slot, parent_hash, parent_root, _pubkey)): Path<(u64, B256, B256, BlsPublicKey)>, @@ -453,23 +483,8 @@ async fn handle_get_execution_payload_bid( tokio::time::sleep(Duration::from_millis(delay_ms)).await; } - let mut block_hash = B256::ZERO; - block_hash.0[0] = 1; - - let message = ExecutionPayloadBid { - parent_block_hash: parent_hash.into(), - parent_block_root: parent_root, - block_hash: block_hash.into(), - gas_limit: 30_000_000, - builder_index: 42, - slot: Slot::new(slot), - value: state.trustless_bid_gwei, - ..Default::default() - }; - - // CB does not verify bid signatures (the beacon node does), so the mock - // serves an unsigned bid - let data = SignedExecutionPayloadBid { message, signature: BlsSignature::empty() }; + let data = + mock_execution_payload_bid(slot, parent_hash, parent_root, state.trustless_bid_gwei, 0); // Negotiate the RESPONSE encoding from the forwarded Accept, mirroring // handle_get_header: honor supported_content_types + the caller's Accept. diff --git a/tests/src/mock_ws_relay.rs b/tests/src/mock_ws_relay.rs index eeec147fb..8bb28f9a8 100644 --- a/tests/src/mock_ws_relay.rs +++ b/tests/src/mock_ws_relay.rs @@ -1,4 +1,5 @@ use std::{ + net::SocketAddr, str::FromStr, sync::{ Arc, Mutex, @@ -10,11 +11,13 @@ use std::{ use alloy::primitives::{B256, U256}; use cb_common::{ pbs::{ - BuilderApiVersion, GET_HEADER_STREAM_PATH, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, - HEADER_VERSION_KEY, + BuilderApiVersion, GET_EXECUTION_PAYLOAD_BID_STREAM_PATH, GET_HEADER_STREAM_PATH, + HEADER_REQUEST_AUTH, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, HEADER_VERSION_KEY, + RelayClient, }, types::{BlsSecretKey, Chain}, utils::utcnow_ms, + wire::CONSENSUS_VERSION_HEADER, }; use futures::SinkExt; use ssz::Encode; @@ -27,27 +30,39 @@ use tokio_tungstenite::{ accept_hdr_async, tungstenite::{ Message, - handshake::server::{Request, Response}, + handshake::server::{ErrorResponse, Request, Response}, + http::StatusCode, }, }; use tracing::debug; -use crate::{mock_relay::mock_signed_builder_bid, utils::HEADER_API_KEY}; +use crate::{ + mock_relay::{ + MockRelayState, mock_execution_payload_bid, mock_signed_builder_bid, + start_mock_relay_service_with_listener, + }, + utils::{HEADER_API_KEY, generate_mock_stream_relay, get_free_listener}, +}; const MSG_BID: u8 = 0x01; const FORK_FULU: u8 = 6; +const FORK_GLOAS: u8 = 7; /// What PBS sent in the handshake, captured for assertions. #[derive(Debug, Clone)] pub struct StreamRequest { pub slot: u64, pub parent_hash: B256, + /// Only on the ePBS bid stream + pub parent_root: Option, pub validator_pubkey: String, pub timeout_ms: Option, pub start_time_ms: Option, pub user_agent: Option, pub cb_version: Option, pub api_key: Option, + pub consensus_version: Option, + pub request_auth: Option, /// When the relay accepted the connection, unix ms pub accepted_at_ms: u64, } @@ -55,9 +70,14 @@ pub struct StreamRequest { pub struct MockWsRelayState { pub chain: Chain, pub signer: BlsSecretKey, - /// One frame pushed per value, in order. The last one is what PBS must - /// return. + /// One frame pushed per value, in order bid_values: Vec, + /// One frame per `(value, execution_payment)` in gwei, on the ePBS bid + /// stream + epbs_bids: Vec<(u64, u64)>, + /// Answer the ePBS stream handshake with 404, as a relay that streams only + /// get_header + no_epbs_stream: bool, /// Pause between updates update_interval: Duration, /// Keep the connection open after the last update, so PBS returns on its @@ -66,8 +86,8 @@ pub struct MockWsRelayState { /// Precede each bid with frames PBS can't parse, which it must skip rather /// than treat as the end of the stream unknown_frames: bool, - /// Sign this many of the last bids for another parent hash, so they fail - /// validation + /// Make this many of the last bids fail validation: signed for another + /// parent hash, or on the ePBS stream cut short so they do not decode invalid_last_bids: usize, /// Wait this long before answering the handshake, as a relay whose /// handshake outlasts PBS's budget @@ -90,6 +110,8 @@ impl MockWsRelayState { chain, signer, bid_values: vec![U256::from(10)], + epbs_bids: vec![(10, 0)], + no_epbs_stream: false, update_interval: Duration::ZERO, hold_open: false, unknown_frames: false, @@ -107,6 +129,14 @@ impl MockWsRelayState { Self { bid_values, ..self } } + pub fn with_epbs_bids(self, epbs_bids: Vec<(u64, u64)>) -> Self { + Self { epbs_bids, ..self } + } + + pub fn without_epbs_stream(self) -> Self { + Self { no_epbs_stream: true, ..self } + } + pub fn with_update_interval(self, update_interval: Duration) -> Self { Self { update_interval, ..self } } @@ -176,11 +206,16 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: let mut request = None; let mut ws = accept_hdr_async(stream, |req: &Request, res: Response| { request = parse_request(req, accepted_at_ms); + if state.no_epbs_stream && request.as_ref().is_some_and(|req| req.parent_root.is_some()) { + let mut not_found = ErrorResponse::new(None); + *not_found.status_mut() = StatusCode::NOT_FOUND; + return Err(not_found); + } Ok(res) }) .await?; - let request = request.ok_or_else(|| eyre::eyre!("malformed get_header stream request"))?; + let request = request.ok_or_else(|| eyre::eyre!("malformed stream request"))?; state.received_connections.fetch_add(1, Ordering::Relaxed); *state.last_request.lock().unwrap() = Some(request.clone()); @@ -195,11 +230,12 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: None => None, }; - for (i, value) in state.bid_values.iter().enumerate() { + let n_bids = + if request.parent_root.is_some() { state.epbs_bids.len() } else { state.bid_values.len() }; + for i in 0..n_bids { if let (Some(end_ms), Some(latency)) = (stream_end_ms, state.ends_at_timeout) { - let sent_ms = end_ms - - state.update_interval.as_millis() as u64 * - (state.bid_values.len() - 1 - i) as u64; + let sent_ms = + end_ms - state.update_interval.as_millis() as u64 * (n_bids - 1 - i) as u64; let arrives_ms = sent_ms + latency.as_millis() as u64; tokio::time::sleep(Duration::from_millis(arrives_ms.saturating_sub(utcnow_ms()))).await; } @@ -211,13 +247,8 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: } } - let is_invalid = i + state.invalid_last_bids >= state.bid_values.len(); - let parent_hash = if is_invalid { B256::repeat_byte(0xee) } else { request.parent_hash }; - let bid = - mock_signed_builder_bid(state.chain, &state.signer, request.slot, parent_hash, *value); - - let mut frame = vec![MSG_BID, FORK_FULU]; - frame.extend_from_slice(&bid.as_ssz_bytes()); + let is_invalid = i + state.invalid_last_bids >= n_bids; + let frame = bid_frame(&state, &request, i, is_invalid); ws.send(Message::Binary(frame.into())).await?; if stream_end_ms.is_none() && !state.update_interval.is_zero() { @@ -239,24 +270,76 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: Ok(()) } +/// Frame `i` of the stream `request` asked for +fn bid_frame( + state: &MockWsRelayState, + request: &StreamRequest, + i: usize, + invalid: bool, +) -> Vec { + let (fork, bid) = match request.parent_root { + Some(parent_root) => { + let (value, execution_payment) = state.epbs_bids[i]; + let bid = mock_execution_payload_bid( + request.slot, + request.parent_hash, + parent_root, + value, + execution_payment, + ); + let mut bid = bid.as_ssz_bytes(); + if invalid { + bid.truncate(10); + } + (FORK_GLOAS, bid) + } + None => { + let parent_hash = if invalid { B256::repeat_byte(0xee) } else { request.parent_hash }; + let bid = mock_signed_builder_bid( + state.chain, + &state.signer, + request.slot, + parent_hash, + state.bid_values[i], + ); + (FORK_FULU, bid.as_ssz_bytes()) + } + }; + + let mut frame = vec![MSG_BID, fork]; + frame.extend_from_slice(&bid); + frame +} + /// The request is the handshake: the fixed stream path followed by -/// `/{slot}/{parent_hash}/{pubkey}`, plus the same headers the HTTP path sends. +/// `/{slot}/{parent_hash}/{pubkey}` (with `/{parent_root}` before the pubkey on +/// the ePBS bid stream), plus the same headers the HTTP path sends. fn parse_request(req: &Request, accepted_at_ms: u64) -> Option { - let prefix = format!("{}{GET_HEADER_STREAM_PATH}/", BuilderApiVersion::V1.path()); - let mut segments = req.uri().path().strip_prefix(&prefix)?.split('/'); + let path = req.uri().path(); + let v1 = BuilderApiVersion::V1.path(); + let (rest, epbs) = + match path.strip_prefix(&format!("{v1}{GET_EXECUTION_PAYLOAD_BID_STREAM_PATH}/")) { + Some(rest) => (rest, true), + None => (path.strip_prefix(&format!("{v1}{GET_HEADER_STREAM_PATH}/"))?, false), + }; + let mut segments = rest.split('/'); let slot = segments.next()?.parse().ok()?; let parent_hash = B256::from_str(segments.next()?).ok()?; + let parent_root = if epbs { Some(B256::from_str(segments.next()?).ok()?) } else { None }; let validator_pubkey = segments.next()?.to_string(); Some(StreamRequest { slot, parent_hash, + parent_root, validator_pubkey, timeout_ms: header(req, HEADER_TIMEOUT_MS).and_then(|v| v.parse().ok()), start_time_ms: header(req, HEADER_START_TIME_UNIX_MS).and_then(|v| v.parse().ok()), user_agent: header(req, "user-agent"), cb_version: header(req, HEADER_VERSION_KEY), api_key: header(req, HEADER_API_KEY), + consensus_version: header(req, CONSENSUS_VERSION_HEADER), + request_auth: header(req, HEADER_REQUEST_AUTH), accepted_at_ms, }) } @@ -265,11 +348,8 @@ fn header(req: &Request, name: &str) -> Option { req.headers().get(name)?.to_str().ok().map(ToOwned::to_owned) } -/// Starts the stream relay with its accept queue full until `drain` fires. A -/// client's SYN is dropped meanwhile, so its connect completes on the SYN -/// retransmit, about a second after it began. On loopback that is the only way -/// to slow a TCP connect from user space: a delayed accept or a proxy hop does -/// not delay the SYN-ACK. Returns the port. +/// Fills the accept queue until `drain` fires, so a connect waits a second for +/// its SYN retransmit; a delayed accept would not slow it. Returns the port. pub async fn start_mock_ws_relay_with_full_backlog( state: Arc, drain: oneshot::Receiver<()>, @@ -299,14 +379,33 @@ pub async fn start_mock_ws_relay_with_full_backlog( Ok(addr.port()) } -/// Serves a get_header stream relay and an HTTP relay on one listener, as a -/// real relay does: a connection whose request asks for a websocket upgrade -/// goes to the stream, any other is forwarded byte for byte to the HTTP relay -/// at `http_addr`. -pub async fn start_mock_dual_relay_service( +/// One relay serving its HTTP API and its bid stream on the same port, as a +/// real relay does. Returns the two mock states and a stream-mode client for +/// the relay. +pub async fn start_mock_dual_relay( + http_state: MockRelayState, + ws_state: MockWsRelayState, +) -> eyre::Result<(Arc, Arc, RelayClient)> { + let http_listener = get_free_listener().await; + let http_addr = http_listener.local_addr()?; + let pubkey = http_state.signer.public_key(); + let http_state = Arc::new(http_state); + tokio::spawn(start_mock_relay_service_with_listener(http_state.clone(), http_listener)); + + let listener = get_free_listener().await; + let port = listener.local_addr()?.port(); + let ws_state = Arc::new(ws_state); + tokio::spawn(start_mock_dual_relay_service(ws_state.clone(), listener, http_addr)); + + Ok((http_state, ws_state, generate_mock_stream_relay(port, pubkey)?)) +} + +/// A connection whose request asks for a websocket upgrade goes to the stream, +/// any other is forwarded byte for byte to the HTTP relay at `http_addr`. +async fn start_mock_dual_relay_service( ws_state: Arc, listener: TcpListener, - http_addr: std::net::SocketAddr, + http_addr: SocketAddr, ) -> eyre::Result<()> { loop { let (stream, addr) = listener.accept().await?; diff --git a/tests/tests/pbs_get_execution_payload_bid_ws.rs b/tests/tests/pbs_get_execution_payload_bid_ws.rs new file mode 100644 index 000000000..706329173 --- /dev/null +++ b/tests/tests/pbs_get_execution_payload_bid_ws.rs @@ -0,0 +1,193 @@ +use std::{sync::Arc, time::Duration}; + +use alloy::primitives::B256; +use base64::{Engine, prelude::BASE64_STANDARD}; +use cb_common::{ + pbs::GetExecutionPayloadBidResponse, signer::random_secret, types::Chain, wire::EncodingType, +}; +use cb_tests::{ + mock_relay::MockRelayState, + mock_validator::MockValidator, + mock_ws_relay::{MockWsRelayState, start_mock_dual_relay}, + utils::{API_KEY, TEST_AUTH_DATA, opaque_auth, setup_pbs}, +}; +use eyre::Result; +use reqwest::StatusCode; +use ssz::Encode; +use tokio::time::Instant; + +const TEST_SLOT: u64 = 255; +const PARENT_ROOT: B256 = B256::repeat_byte(1); +const BUDGET_MS: u64 = 1_000; + +/// A relay bidding `http` gwei over HTTP and streaming `stream_bids`, each a +/// `(value, execution_payment)` in gwei +fn relay_states(http: u64, stream_bids: Vec<(u64, u64)>) -> (MockRelayState, MockWsRelayState) { + let signer = random_secret(); + let http_state = + MockRelayState::new(Chain::Hoodi, signer.clone()).with_trustless_bid_gwei(http); + (http_state, MockWsRelayState::new(Chain::Hoodi, signer).with_epbs_bids(stream_bids)) +} + +/// PBS in front of one stream-mode relay serving both states on one port +async fn start_race( + (http_state, ws_state): (MockRelayState, MockWsRelayState), +) -> Result<(MockValidator, Arc, Arc)> { + let (http_state, ws_state, relay) = start_mock_dual_relay(http_state, ws_state).await?; + let validator = setup_pbs(Chain::Hoodi, vec![relay], |_| {}).await?; + Ok((validator, http_state, ws_state)) +} + +async fn get_bid( + validator: &MockValidator, +) -> Result<(StatusCode, Option)> { + let res = validator + .do_get_execution_payload_bid_with_timeout( + TEST_SLOT, + B256::ZERO, + PARENT_ROOT, + None, + Some(&opaque_auth(TEST_AUTH_DATA, TEST_SLOT)), + vec![EncodingType::Json], + BUDGET_MS, + ) + .await?; + let code = res.status(); + if code != StatusCode::OK { + return Ok((code, None)); + } + Ok((code, Some(serde_json::from_slice(&res.bytes().await?)?))) +} + +fn assert_bid(res: Option, value: u64, execution_payment: u64) { + let res = res.expect("a bid"); + let bid = &res.data.message; + assert_eq!((bid.value, bid.execution_payment), (value, execution_payment)); +} + +/// A relay that streams get_header but not the ePBS bid answers the handshake +/// with 404, and the HTTP request raced alongside it supplies the bid. +#[tokio::test] +async fn test_get_execution_payload_bid_ws_no_stream_route_http_bid_serves() -> Result<()> { + let (http_state, ws_state) = relay_states(42, vec![(90, 0)]); + let (validator, http_state, ws_state) = + start_race((http_state, ws_state.without_epbs_stream())).await?; + + let (code, res) = get_bid(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert_bid(res, 42, 0); + assert_eq!(http_state.received_execution_payload_bid(), 1); + assert_eq!(ws_state.handshake_attempts(), 1); + assert_eq!(ws_state.received_connections(), 0, "the handshake never completed"); + Ok(()) +} + +/// With no bid from the stream, the builder's HTTP 401 reaches the beacon node. +#[tokio::test] +async fn test_get_execution_payload_bid_ws_no_stream_bid_http_rejection_passes() -> Result<()> { + let (http_state, ws_state) = relay_states(10, vec![(50, 0)]); + let (validator, http_state, _) = + start_race((http_state, ws_state.without_epbs_stream())).await?; + http_state.set_response_override(StatusCode::UNAUTHORIZED); + + let (code, _) = get_bid(&validator).await?; + assert_eq!(code, StatusCode::UNAUTHORIZED); + Ok(()) +} + +/// The builder's HTTP 401 does not displace the stream's bid. +#[tokio::test] +async fn test_get_execution_payload_bid_ws_http_rejection_stream_bid_serves() -> Result<()> { + let (validator, http_state, _) = start_race(relay_states(10, vec![(50, 0)])).await?; + http_state.set_response_override(StatusCode::UNAUTHORIZED); + + let (code, res) = get_bid(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert_bid(res, 50, 0); + assert_eq!(http_state.received_execution_payload_bid(), 1); + Ok(()) +} + +/// An HTTP request that runs out the window does not displace the stream's bid. +#[tokio::test] +async fn test_get_execution_payload_bid_ws_http_timeout_stream_bid_serves() -> Result<()> { + let (http_state, ws_state) = relay_states(10, vec![(50, 0)]); + let (validator, ..) = start_race((http_state.with_bid_delay_ms(BUDGET_MS), ws_state)).await?; + + let (code, res) = get_bid(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert_bid(res, 50, 0); + Ok(()) +} + +/// The stream and the HTTP request are separate candidates: the higher value + +/// execution_payment wins, the stream's bid on a tie, and the stream's +/// candidate is its latest bid, not its highest. +#[tokio::test] +async fn test_get_execution_payload_bid_ws_higher_total_wins() -> Result<()> { + for (http, stream_bids, served) in + [(60, vec![(70, 0), (50, 0)], (60, 0)), (50, vec![(10, 40)], (10, 40))] + { + let (validator, http_state, ws_state) = + start_race(relay_states(http, stream_bids.clone())).await?; + + let (code, res) = get_bid(&validator).await?; + assert_eq!(code, StatusCode::OK, "http {http}, stream {stream_bids:?}"); + assert_bid(res, served.0, served.1); + assert_eq!(http_state.received_execution_payload_bid(), 1); + assert_eq!(ws_state.received_connections(), 1); + } + Ok(()) +} + +/// A later streamed bid that fails to decode does not displace the valid bid +/// (50) before it. +#[tokio::test] +async fn test_get_execution_payload_bid_ws_skips_invalid_bid() -> Result<()> { + let (http_state, ws_state) = relay_states(10, vec![(50, 0), (90, 0)]); + let (validator, ..) = start_race((http_state, ws_state.with_invalid_last_bids(1))).await?; + + let (code, res) = get_bid(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert_bid(res, 50, 0); + Ok(()) +} + +/// The handshake carries the auth as padded standard base64 of its SSZ, the +/// fork and a timeout that leaves the buffer; the bid lands within the budget. +#[tokio::test] +async fn test_get_execution_payload_bid_ws_handshake_and_deadline() -> Result<()> { + const BUFFER_MS: u64 = 200; + let (http_state, ws_state) = relay_states(10, vec![(20, 0)]); + let (_, ws_state, relay) = + start_mock_dual_relay(http_state.with_bid_delay_ms(400), ws_state.hold_open()).await?; + let validator = setup_pbs(Chain::Hoodi, vec![relay], |pbs_config| { + pbs_config.proposer_deadline_buffer_ms = BUFFER_MS + }) + .await?; + + let start = Instant::now(); + let (code, _) = get_bid(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert!(start.elapsed() < Duration::from_millis(BUDGET_MS), "{:?}", start.elapsed()); + + let request = ws_state.last_request().expect("a handshake"); + // 119 bytes of SSZ need padding, and slot 255 puts a '/' in the base64, + // which the URL-safe alphabet would write as '_' + let expected_auth = + BASE64_STANDARD.encode(opaque_auth(TEST_AUTH_DATA, TEST_SLOT).as_ssz_bytes()); + assert!(expected_auth.ends_with('=') && expected_auth.contains('/')); + assert_eq!(request.request_auth, Some(expected_auth)); + + assert_eq!(request.slot, TEST_SLOT); + assert_eq!(request.parent_root, Some(PARENT_ROOT)); + assert_eq!(request.consensus_version.as_deref(), Some("gloas")); + let relay_timeout_ms = request.timeout_ms.expect("missing timeout header"); + assert!( + (BUDGET_MS - BUFFER_MS - 100..=BUDGET_MS - BUFFER_MS).contains(&relay_timeout_ms), + "timeout header {relay_timeout_ms}ms" + ); + assert_eq!(request.api_key.as_deref(), Some(API_KEY)); + assert!(request.user_agent.is_some_and(|ua| ua.contains("commit-boost"))); + Ok(()) +} diff --git a/tests/tests/pbs_get_header_ws.rs b/tests/tests/pbs_get_header_ws.rs index 858f4daa4..e9fcd4b6c 100644 --- a/tests/tests/pbs_get_header_ws.rs +++ b/tests/tests/pbs_get_header_ws.rs @@ -18,7 +18,7 @@ use cb_tests::{ mock_relay::{MockRelayState, start_mock_relay_service_with_listener}, mock_validator::MockValidator, mock_ws_relay::{ - MockWsRelayState, start_mock_dual_relay_service, start_mock_ws_relay_service, + MockWsRelayState, start_mock_dual_relay, start_mock_ws_relay_service, start_mock_ws_relay_with_full_backlog, }, utils::{ @@ -65,8 +65,7 @@ async fn start_pbs( let config = to_pbs_config(chain, pbs_config, relays); let state = PbsState::new(config, PathBuf::new()); - drop(listener); - tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, listener)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -289,10 +288,8 @@ async fn test_get_header_ws_returns_at_deadline() -> Result<()> { Ok(()) } -/// A relay that ends its stream at Date-Milliseconds + X-Timeout-Ms sends its -/// last bid then, and each frame takes one downlink trip to arrive. PBS asks it -/// to end half the TCP connect time before PBS's own deadline, so that bid -/// still lands. +/// PBS asks the relay to end its stream half a TCP connect before PBS's own +/// deadline, so the last bid, one downlink trip later, still lands. #[tokio::test] async fn test_get_header_ws_last_bid_lands_before_deadline() -> Result<()> { setup_test_env(); @@ -329,8 +326,9 @@ async fn test_get_header_ws_last_bid_lands_before_deadline() -> Result<()> { let relay_end_ms = request.start_time_ms.unwrap() + request.timeout_ms.unwrap(); let early_ms = (requested_at_ms + timeout_ms) as i64 - relay_end_ms as i64; let half_connect_ms = connect_ms as i64 / 2; + // PBS's delay before connecting skews both sides, hence the slack assert!( - (half_connect_ms - 100..=half_connect_ms + 20).contains(&early_ms), + (half_connect_ms - 250..=half_connect_ms + 20).contains(&early_ms), "relay asked to end {early_ms}ms before the deadline, connect took {connect_ms}ms" ); Ok(()) @@ -441,27 +439,6 @@ async fn test_get_header_ws_wins_auction_against_http() -> Result<()> { Ok(()) } -/// One relay serving both the HTTP `get_header` and the stream on the same -/// port, as a real relay does. Returns the two mock states and the stream-mode -/// relay client. -async fn start_dual_relay( - http_state: MockRelayState, - ws_state: MockWsRelayState, - pubkey: cb_common::types::BlsPublicKey, -) -> Result<(Arc, Arc, cb_common::pbs::RelayClient)> { - let http_listener = get_free_listener().await; - let http_addr = http_listener.local_addr()?; - let http_state = Arc::new(http_state); - tokio::spawn(start_mock_relay_service_with_listener(http_state.clone(), http_listener)); - - let listener = get_free_listener().await; - let port = listener.local_addr()?.port(); - let ws_state = Arc::new(ws_state); - tokio::spawn(start_mock_dual_relay_service(ws_state.clone(), listener, http_addr)); - - Ok((http_state, ws_state, generate_mock_stream_relay(port, pubkey)?)) -} - /// A handshake that outlasts the budget: the HTTP request raced alongside it /// supplies the bid. #[tokio::test] @@ -469,10 +446,9 @@ async fn test_get_header_ws_race_handshake_timeout_http_bid_serves() -> Result<( setup_test_env(); let signer = random_secret(); let chain = Chain::Hoodi; - let (http_state, ws_state, relay) = start_dual_relay( + let (http_state, ws_state, relay) = start_mock_dual_relay( MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(42)), MockWsRelayState::new(chain, signer.clone()).with_handshake_delay(Duration::from_secs(2)), - signer.public_key(), ) .await?; let validator = start_pbs(chain, vec![relay], 300).await?; @@ -492,12 +468,11 @@ async fn test_get_header_ws_race_stream_break_keeps_held_bid() -> Result<()> { setup_test_env(); let signer = random_secret(); let chain = Chain::Hoodi; - let (_, ws_state, relay) = start_dual_relay( + let (_, ws_state, relay) = start_mock_dual_relay( MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(10)), MockWsRelayState::new(chain, signer.clone()) .with_bid_values(vec![U256::from(50)]) .abort_after_bids(), - signer.public_key(), ) .await?; let validator = start_pbs(chain, vec![relay], 1_000).await?; @@ -509,20 +484,21 @@ async fn test_get_header_ws_race_stream_break_keeps_held_bid() -> Result<()> { Ok(()) } -/// The stream and the HTTP request are separate candidates: the higher bid -/// wins, and the stream's candidate is its latest bid, not its highest. +/// The higher candidate wins, the stream's being its latest bid, not its +/// highest; a stream with no bid leaves the HTTP one. #[tokio::test] async fn test_get_header_ws_race_higher_bid_wins() -> Result<()> { setup_test_env(); let signer = random_secret(); let chain = Chain::Hoodi; - for (http_bid, stream_bids, served) in [(10, [20, 50], 50), (60, [70, 50], 60)] { - let (http_state, ws_state, relay) = start_dual_relay( + for (http_bid, stream_bids, served) in + [(10, vec![20, 50], 50), (60, vec![70, 50], 60), (42, vec![], 42)] + { + let (http_state, ws_state, relay) = start_mock_dual_relay( MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(http_bid)), MockWsRelayState::new(chain, signer.clone()) - .with_bid_values(stream_bids.map(U256::from).to_vec()), - signer.public_key(), + .with_bid_values(stream_bids.into_iter().map(U256::from).collect()), ) .await?; let validator = start_pbs(chain, vec![relay], 1_000).await?; From 7d89ecae88dce3bd7746258b525c8479098e9373 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Sat, 10 Oct 2026 13:08:05 -0700 Subject: [PATCH 17/24] docs(pbs): say what the auth data checks accept decode_auth_data_url only ever gets the auth data address, the part before any `?`, so its parameter and doc now say so. dial_target's comparison of the parsed host with the address passes only a hostname already in the builder-specs default form, since parsing also accepts a port, userinfo, a path, upper case and numeric IPv4. --- crates/common/src/pbs/relay.rs | 8 ++++---- crates/pbs/src/dial.rs | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index b877d8580..13eb4c6be 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -261,10 +261,10 @@ impl RelayClient { } } -/// The builder URL in URL-form auth data. Only `http` and `https` count, since -/// opaque data such as `builder-a:prod` parses as a URL too. -pub fn decode_auth_data_url(data: &[u8]) -> Option { - let url = Url::parse(std::str::from_utf8(data).ok()?).ok()?; +/// `address`, the auth data before any `?`, as a builder URL. Only `http` and +/// `https` count, since `builder-a:prod` parses as a URL too. +pub fn decode_auth_data_url(address: &[u8]) -> Option { + let url = Url::parse(std::str::from_utf8(address).ok()?).ok()?; matches!(url.scheme(), "http" | "https").then_some(url) } diff --git a/crates/pbs/src/dial.rs b/crates/pbs/src/dial.rs index d5a75a007..31590f0df 100644 --- a/crates/pbs/src/dial.rs +++ b/crates/pbs/src/dial.rs @@ -59,6 +59,8 @@ fn dial_target(data_url: Option, address: &[u8]) -> Result Date: Sat, 10 Oct 2026 13:13:10 -0700 Subject: [PATCH 18/24] perf(pbs): reuse a dial check's lookup for its connection A dial to a hostname looked the host up twice in a row on the request path: once in dial_relay's check, and again in DialResolver when the connection opened. The resolver now takes the check's addresses while they are under 2 s old, checking them again, and looks the host up itself otherwise, so a later connection still sees a DNS change. The addresses are kept with port 0, as the resolver's own lookup returns them, so a dial to the same host on another port connects to its own. --- crates/pbs/src/dial.rs | 86 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/crates/pbs/src/dial.rs b/crates/pbs/src/dial.rs index 31590f0df..5333b866d 100644 --- a/crates/pbs/src/dial.rs +++ b/crates/pbs/src/dial.rs @@ -1,8 +1,9 @@ use std::{ + collections::HashMap, io, net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}, sync::{Arc, LazyLock, OnceLock}, - time::Duration, + time::{Duration, Instant}, }; use cb_common::{ @@ -11,6 +12,7 @@ use cb_common::{ types::BlsPublicKey, utils::bls_pubkey_from_hex, }; +use parking_lot::Mutex; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use tokio::{sync::Semaphore, task::JoinHandle}; use tracing::{info, warn}; @@ -109,9 +111,34 @@ async fn resolve_dial_target( } }; vet_dial_addrs(origin, &addrs)?; + remember_checked(url.host_str().unwrap_or_default(), &addrs); Ok(addrs) } +/// How long a dial check's lookup serves new connections to its host, so the +/// resolver does not repeat it. Short, so a later connection sees a DNS change +const CHECKED_ADDRS_TTL: Duration = Duration::from_secs(2); + +/// Each host's checked addresses and when they were resolved +type CheckedAddrs = HashMap)>; + +static CHECKED_ADDRS: LazyLock> = LazyLock::new(Default::default); + +/// Kept with port 0, as the resolver's own lookup returns them, so another +/// dial to the host on another port connects to its own. Drops expired entries +fn remember_checked(host: &str, addrs: &[SocketAddr]) { + let addrs = addrs.iter().map(|addr| SocketAddr::new(addr.ip(), 0)).collect(); + let mut checked = CHECKED_ADDRS.lock(); + checked.retain(|_, (at, _)| at.elapsed() < CHECKED_ADDRS_TTL); + checked.insert(host.to_owned(), (Instant::now(), addrs)); +} + +fn recently_checked(host: &str) -> Option> { + let checked = CHECKED_ADDRS.lock(); + let (at, addrs) = checked.get(host)?; + (at.elapsed() < CHECKED_ADDRS_TTL).then(|| addrs.clone()) +} + /// `None` when every lookup slot is taken fn spawn_lookup(host_port: String) -> Option>>> { let permit = DIAL_LOOKUPS.try_acquire().ok()?; @@ -184,16 +211,21 @@ fn dial_config(url: Url) -> RelayConfig { } } -/// Run for each new connection to a hostname (an IP address skips it): refuses -/// it while every lookup slot is taken or if any address is disallowed +/// Run for each new connection to a hostname (an IP address skips it): takes +/// the dial check's recent lookup or its own, and refuses a disallowed address struct DialResolver; impl Resolve for DialResolver { fn resolve(&self, name: Name) -> Resolving { Box::pin(async move { - let lookup = spawn_lookup(format!("{}:0", name.as_str())) - .ok_or("too many dial lookups in flight")?; - let addrs = lookup.await??; + let addrs = match recently_checked(name.as_str()) { + Some(addrs) => addrs, + None => { + spawn_lookup(format!("{}:0", name.as_str())) + .ok_or("too many dial lookups in flight")? + .await?? + } + }; vet_dial_addrs(name.as_str(), &addrs)?; Ok(Box::new(addrs.into_iter()) as Addrs) }) @@ -407,6 +439,48 @@ mod tests { Ok(()) } + // With every lookup slot held, the connection can only use the addresses + // the dial's check resolved + #[cfg(feature = "testing-flags")] + #[tokio::test] + async fn dial_connects_to_the_addresses_its_check_resolved() -> eyre::Result<()> { + let _slots = LOOKUP_SLOTS.lock().await; + let (port, _) = mock_builder().await?; + let url = Url::parse(&format!("http://localhost:{port}/bid"))?; + set_skip_dial_target_check(true); + let relay = dial_relay(Some(url.clone()), b"", Duration::from_secs(5)).await?; + let held = DIAL_LOOKUPS.try_acquire_many(32)?; + let res = relay.client.post(url).send().await; + drop(held); + set_skip_dial_target_check(false); + // dial_client_checks_the_addresses_it_resolves needs a lookup + CHECKED_ADDRS.lock().remove("localhost"); + assert_eq!(res?.status(), StatusCode::TEMPORARY_REDIRECT); + Ok(()) + } + + // A remembered address is checked again before a connection uses it + #[tokio::test] + async fn dial_resolver_checks_remembered_addresses() { + remember_checked("remembered.invalid", &["127.0.0.1:0".parse().unwrap()]); + let res = DialResolver.resolve("remembered.invalid".parse().unwrap()).await; + CHECKED_ADDRS.lock().remove("remembered.invalid"); + assert!(res.is_err()); + } + + #[test] + fn checked_addrs_drop_their_port_expire_and_are_swept() { + let addrs = ["1.1.1.1:443".parse().unwrap()]; + let expired = Instant::now().checked_sub(CHECKED_ADDRS_TTL).unwrap(); + CHECKED_ADDRS.lock().insert("expired.invalid".into(), (expired, addrs.to_vec())); + assert_eq!(recently_checked("expired.invalid"), None); + remember_checked("fresh.invalid", &addrs); + assert_eq!(recently_checked("fresh.invalid"), Some(vec!["1.1.1.1:0".parse().unwrap()])); + let mut checked = CHECKED_ADDRS.lock(); + assert!(!checked.contains_key("expired.invalid")); + checked.remove("fresh.invalid"); + } + #[test] fn ip_is_disallowed_table() { for (ip, disallowed) in [ From b9f1283dda5babb34b4a6b8cea56b9f8e3b82a25 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 12 Oct 2026 11:15:20 -0700 Subject: [PATCH 19/24] fix(pbs): route via [[relays]], forward JSON blocks, refuse own host Routing: a request whose auth data names a builder looks first among the relays serving the key, then among [[relays]] in config order, before dialing. A key in a mux whose auth data named a relay configured only in [[relays]] was dialed at https://:443 without that entry's port or headers, or, on a private address, refused with 400; it now reaches the relay through its entry. Relays listed only in a mux are not searched for other keys, since their entries carry settings meant for that mux's keys; such a relay is dialed like any builder outside the config. Signed block: builder-specs makes JSON the default, and Lighthouse forwards the signed block as JSON, but Commit-Boost answered anything but SSZ with 415, so no builder received a Lighthouse block through it (415 on 17 of 17 blocks on a devnet). The block is forwarded undecoded in the encoding it arrived in, labelled JSON when it is JSON or unlabelled, SSZ when it is SSZ; other types are 415. Own host: on a validator client whose builder URL is Commit-Boost, a key with no builder config sends Commit-Boost's hostname as auth data. When that host resolved to a public address, Commit-Boost dialed it on port 443, where it does not listen; behind a firewall that drops packets, the dial waited out the whole bid timeout. Auth data equal to the host in the request's Host header, in the form of default auth data, is refused with 400 before any lookup. Also: the signed block logs one info line with how many builders accepted it, since a block whose bid came from a dialed builder is accepted by none of the configured ones. The ePBS routes read a builder's error body through one helper, and an empty one on the bid route reads "expected 200 or 204". A dial lookup that fails to run is logged. The cb-pbs testing flag moves to dev-dependencies, so a workspace build never compiles in the switch that turns off the dial check. Each change has a test; others pin the lookup cap holding its slot, a dial never carrying a configured relay's headers, a 204 when no lookup slot is free, and the lookup time taken from the builder's timeout. The docs correct the claims on the signed block, default auth data, routing and dial counting, the lookup time in X-Timeout-Ms, 500 on an internal error, and 555 on get_header_stream. --- crates/common/src/pbs/relay.rs | 3 +- crates/pbs/src/dial.rs | 160 ++++++++++++++--- crates/pbs/src/routes/builder_preferences.rs | 8 +- .../pbs/src/routes/execution_payload_bid.rs | 21 +-- .../src/routes/submit_signed_beacon_block.rs | 32 ++-- crates/pbs/src/state.rs | 5 + crates/pbs/src/utils.rs | 170 ++++++++++++++---- docs/docs/get_started/epbs.md | 24 ++- .../get_started/running/metrics-catalog.md | 2 +- tests/Cargo.toml | 3 +- tests/src/mock_relay.rs | 21 ++- tests/src/mock_validator.rs | 13 +- tests/src/utils.rs | 40 ++++- tests/tests/pbs_get_execution_payload_bid.rs | 99 +++++----- tests/tests/pbs_get_header_ws.rs | 4 +- tests/tests/pbs_submit_builder_preferences.rs | 31 +++- tests/tests/pbs_submit_signed_beacon_block.rs | 84 +++++++-- 17 files changed, 531 insertions(+), 189 deletions(-) diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index 13eb4c6be..4dd48e40c 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -291,13 +291,14 @@ mod tests { #[test] fn decode_auth_data_url_accepts_only_http_urls() { - let cases: [(&[u8], Option<&str>); 5] = [ + let cases: [(&[u8], Option<&str>); 6] = [ (b"https://Builder-A.example.com:443/eth", Some("https://builder-a.example.com/eth")), (b"http://10.0.0.1:18550", Some("http://10.0.0.1:18550/")), // Opaque data and host:port parse as URLs with their own scheme (b"builder-a:prod", None), (b"localhost:18550", None), (b"builder-a.example.com", None), + (b"http://1.1.1.1/\xff", None), ]; for (data, expected) in cases { let decoded = decode_auth_data_url(data).map(|url| url.to_string()); diff --git a/crates/pbs/src/dial.rs b/crates/pbs/src/dial.rs index 5333b866d..62be496e5 100644 --- a/crates/pbs/src/dial.rs +++ b/crates/pbs/src/dial.rs @@ -24,9 +24,8 @@ use crate::error::PbsClientError; /// per-host metric label would be unbounded const DIAL_RELAY_ID: &str = "dial"; -/// The BLS12-381 G1 generator, standing in for the pubkey `RelayEntry` requires -/// but only PBS reads. A workaround to leave `RelayClient` unchanged until PBS -/// is deprecated, when the pubkey can go. +/// The BLS12-381 G1 generator, filling the pubkey `RelayEntry` requires and +/// only PBS reads static DIAL_PUBKEY: LazyLock = LazyLock::new(|| { bls_pubkey_from_hex( "0x97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb", @@ -54,13 +53,18 @@ pub(crate) async fn dial_relay( dial_client(url) } +/// `https:///`, whose `host_str` is the authority's hostname in the +/// form of default auth data +pub(crate) fn host_url(authority: &str) -> Option { + Url::parse(&format!("https://{authority}/")).ok() +} + /// `data_url`, else `https:///` for the builder-specs default auth /// data, a bare hostname fn dial_target(data_url: Option, address: &[u8]) -> Result { data_url .or_else(|| { - let url = - Url::parse(&format!("https://{}/", std::str::from_utf8(address).ok()?)).ok()?; + let url = host_url(std::str::from_utf8(address).ok()?)?; // Parsing also takes a port, userinfo, a path, upper case and // numeric IPv4, so only a host already in canonical form passes (url.host_str()?.as_bytes() == address).then_some(url) @@ -83,11 +87,8 @@ async fn resolve_dial_target( origin: &str, lookup_timeout: Duration, ) -> Result, PbsClientError> { - let host_port = format!( - "{}:{}", - url.host_str().unwrap_or_default(), - url.port_or_known_default().unwrap_or_default() - ); + let host = url.host_str().unwrap_or_default(); + let host_port = format!("{host}:{}", url.port_or_known_default().unwrap_or_default()); // An IP literal needs no lookup if let Ok(addr) = host_port.parse::() { vet_dial_addrs(origin, &[addr])?; @@ -103,15 +104,17 @@ async fn resolve_dial_target( warn!(%origin, %err, "dial target lookup failed, not dialing"); return Err(PbsClientError::DialTargetBlocked); } - Ok(Err(_)) => return Err(PbsClientError::Internal), - // Transient, unlike a failed lookup: the builder counts as not answering + Ok(Err(err)) => { + warn!(%origin, %err, "dial target lookup did not finish"); + return Err(PbsClientError::Internal); + } Err(_) => { warn!(%origin, ?lookup_timeout, "dial target lookup timed out, not dialing"); return Err(PbsClientError::NoBuilderResponse); } }; vet_dial_addrs(origin, &addrs)?; - remember_checked(url.host_str().unwrap_or_default(), &addrs); + remember_checked(host, &addrs); Ok(addrs) } @@ -119,7 +122,6 @@ async fn resolve_dial_target( /// resolver does not repeat it. Short, so a later connection sees a DNS change const CHECKED_ADDRS_TTL: Duration = Duration::from_secs(2); -/// Each host's checked addresses and when they were resolved type CheckedAddrs = HashMap)>; static CHECKED_ADDRS: LazyLock> = LazyLock::new(Default::default); @@ -158,8 +160,8 @@ fn vet_dial_addrs(origin: &str, addrs: &[SocketAddr]) -> Result<(), PbsClientErr Ok(()) } -/// Dial hosts are chosen by the request, so an idle connection is reused for at -/// most one slot. The pool sweeps at this interval, so it closes within two. +/// Dial hosts are chosen by the request, so an idle connection is kept for one +/// mainnet slot. The pool sweeps at this interval, so it closes within two. const DIAL_IDLE_TIMEOUT: Duration = Duration::from_secs(12); /// Lets a builder dialed again reuse its connection. A failed build is not @@ -307,13 +309,14 @@ mod tests { async fn dial_relay_targets() { const MISMATCH: &str = "auth data does not match a configured builder"; const BLOCKED: &str = "dial target does not resolve or resolves to a disallowed address"; - let cases: [(&[u8], Result<&str, &str>); 8] = [ + let cases: [(&[u8], Result<&str, &str>); 9] = [ (b"http://1.1.1.1:8551", Ok("http://1.1.1.1:8551/")), (b"1.1.1.1", Ok("https://1.1.1.1/")), - (b"1.1.1.1?ofac=1", Ok("https://1.1.1.1/")), + (b"1.1.1.1?ofac=1?v=2", Ok("https://1.1.1.1/")), (b"[2606:4700:4700::1111]", Ok("https://[2606:4700:4700::1111]/")), (b"10.0.0.1", Err(BLOCKED)), (b"1.1.1.1:8551", Err(MISMATCH)), + (b"Builder.example.com", Err(MISMATCH)), (b"ftp://1.1.1.1", Err(MISMATCH)), (&[0xde, 0xad], Err(MISMATCH)), ]; @@ -346,6 +349,117 @@ mod tests { assert!(matches!(dial(b"builder.invalid").await, Err(PbsClientError::NoBuilderResponse))); } + // A label over 63 octets fails in the stub resolver, without a query + #[tokio::test] + async fn dial_target_that_does_not_resolve_is_blocked() { + let _slots = LOOKUP_SLOTS.lock().await; + let host = format!("{}.invalid", "a".repeat(64)); + let res = dial_relay(None, host.as_bytes(), Duration::from_secs(5)).await; + assert!(matches!(res, Err(PbsClientError::DialTargetBlocked)), "{:?}", res.map(|_| ())); + } + + // One blocking thread, so the lookup queues behind another task + #[test] + fn lookup_holds_its_slot_until_it_returns() { + let _slots = LOOKUP_SLOTS.blocking_lock(); + let runtime = + tokio::runtime::Builder::new_current_thread().max_blocking_threads(1).build().unwrap(); + runtime.block_on(async { + let (release, blocked) = std::sync::mpsc::channel::<()>(); + let busy = tokio::task::spawn_blocking(move || blocked.recv()); + let lookup = spawn_lookup("1.1.1.1:443".to_owned()).unwrap(); + assert_eq!(DIAL_LOOKUPS.available_permits(), 31); + release.send(()).unwrap(); + busy.await.unwrap().unwrap(); + lookup.await.unwrap().unwrap(); + assert_eq!(DIAL_LOOKUPS.available_permits(), 32); + }); + } + + // One blocking thread, so the lookup waits behind another task, and that + // wait is taken from the time left for the builder + #[cfg(feature = "testing-flags")] + #[test] + fn dial_lookup_time_is_taken_from_the_timeout() { + let _slots = LOOKUP_SLOTS.blocking_lock(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .max_blocking_threads(1) + .build() + .unwrap(); + let left_ms = runtime.block_on(async { + tokio::task::spawn_blocking(|| std::thread::sleep(Duration::from_millis(50))); + set_skip_dial_target_check(true); + let res = crate::utils::resolve_addressed_relay( + &[], + &[], + b"localhost", + &Default::default(), + 1000, + ) + .await; + set_skip_dial_target_check(false); + res.map(|(_, left_ms)| left_ms) + }); + // dial_client_checks_the_addresses_it_resolves needs a lookup + CHECKED_ADDRS.lock().remove("localhost"); + assert!(left_ms.unwrap() <= 950); + } + + #[tokio::test] + async fn bid_dial_without_a_lookup_slot_is_no_bid() -> eyre::Result<()> { + use cb_common::{ + config::PbsModuleConfig, + pbs::{HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS}, + types::Chain, + utils::utcnow_ms, + }; + use parking_lot::RwLock; + + use crate::{DefaultBuilderApi, PbsState, routes::create_app_router}; + + let _slots = LOOKUP_SLOTS.lock().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let state = PbsState::new( + PbsModuleConfig { + chain: Chain::Hoodi, + endpoint: addr, + pbs_config: Arc::new(serde_json::from_str("{}")?), + relays: vec![], + all_relays: vec![], + signer_client: None, + registry_muxes: None, + mux_lookup: None, + }, + Default::default(), + ); + let app = create_app_router::<(), DefaultBuilderApi>(Arc::new(RwLock::new(state))); + tokio::spawn(async move { axum::serve(listener, app).await }); + + let _held = DIAL_LOOKUPS.try_acquire_many(32)?; + let zero = format!("0x{}", "0".repeat(64)); + let url = format!( + "http://{addr}/eth/v1/builder/execution_payload_bid/1/{zero}/{zero}/{}", + *DIAL_PUBKEY + ); + let auth = serde_json::json!({ + // "builder.invalid" + "message": { "data": "0x6275696c6465722e696e76616c6964", "slot": "1" }, + "signature": format!("0x{}", "0".repeat(192)), + }); + let res = reqwest::Client::new() + .post(url) + .header("Eth-Consensus-Version", "gloas") + .header(HEADER_START_TIME_UNIX_MS, utcnow_ms()) + .header(HEADER_TIMEOUT_MS, 1000) + .json(&auth) + .send() + .await?; + assert_eq!(res.status(), StatusCode::NO_CONTENT); + Ok(()) + } + #[test] fn vet_dial_addrs_checks_every_address() { let addr = |s: &str| s.parse::().unwrap(); @@ -459,13 +573,17 @@ mod tests { Ok(()) } - // A remembered address is checked again before a connection uses it + // A remembered address is checked again before a connection uses it. With + // every slot held, ignoring the entry would fail on the cap instead #[tokio::test] async fn dial_resolver_checks_remembered_addresses() { + let _slots = LOOKUP_SLOTS.lock().await; + let _held = DIAL_LOOKUPS.try_acquire_many(32).unwrap(); remember_checked("remembered.invalid", &["127.0.0.1:0".parse().unwrap()]); let res = DialResolver.resolve("remembered.invalid".parse().unwrap()).await; CHECKED_ADDRS.lock().remove("remembered.invalid"); - assert!(res.is_err()); + let err = res.err().expect("a loopback address is refused").to_string(); + assert_eq!(err, PbsClientError::DialTargetBlocked.to_string()); } #[test] @@ -514,7 +632,7 @@ mod tests { ("fec0::1", true), ("ff02::1", true), ("2001:db8::1", true), - ("64:ff9b:1::a00:1", true), + ("64:ff9b:1::101:101", true), ("2002:a00:1::1", true), ("2001::1", true), ("1.1.1.1", false), diff --git a/crates/pbs/src/routes/builder_preferences.rs b/crates/pbs/src/routes/builder_preferences.rs index cba99d0ef..cdbeb4e4d 100644 --- a/crates/pbs/src/routes/builder_preferences.rs +++ b/crates/pbs/src/routes/builder_preferences.rs @@ -6,7 +6,7 @@ use axum::{ }; use cb_common::{ pbs::{BuilderPreferencesRequest, SubmitBuilderPreferencesParams}, - wire::{decode_versioned_request_body, get_user_agent}, + wire::{EncodingType, decode_versioned_request_body, get_user_agent}, }; use reqwest::StatusCode; use ssz::Encode; @@ -18,7 +18,7 @@ use crate::{ error::PbsClientError, state::{BuilderApiState, PbsState}, utils::{ - builder_rejection, epbs_base_send_headers, log_mux_selection, post_ssz_expect_accepted, + builder_rejection, epbs_base_send_headers, log_mux_selection, post_expect_accepted, record_beacon_status, record_request_failure, resolve_addressed_relay, }, }; @@ -69,6 +69,7 @@ pub async fn submit_builder_preferences( // timeout rather than the block-production one let (relay, timeout_ms) = resolve_addressed_relay( relays, + state.default_relays(), request.auth.message.data.as_ref(), &req_headers, pbs_config.timeout_register_validator_ms, @@ -83,10 +84,11 @@ pub async fn submit_builder_preferences( let relay_id = relay.id.as_ref(); let sent = match relay.submit_builder_preferences_url(¶ms.proposer_pubkey) { Ok(url) => { - post_ssz_expect_accepted( + post_expect_accepted( &relay, url, body, + EncodingType::Ssz, send_headers, timeout_ms, SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG, diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index affdfcf10..f4b57bd72 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -16,7 +16,7 @@ use cb_common::{ wire::{ CONSENSUS_VERSION_HEADER, EncodingType, GLOAS_CONSENSUS_VERSION, OUTBOUND_ACCEPT_SSZ_FIRST, decode_versioned_request_body, get_accept_types, get_user_agent, - parse_response_encoding_and_fork, read_chunked_body_with_max, safe_read_http_response, + parse_response_encoding_and_fork, safe_read_http_response, }, }; use reqwest::{ @@ -36,7 +36,8 @@ use crate::{ state::{BuilderApiState, PbsState}, utils::{ builder_rejection, epbs_base_send_headers, format_gwei_as_eth, log_mux_selection, - record_beacon_status, record_request_failure, resolve_addressed_relay, send_to_relay, + record_beacon_status, record_request_failure, relay_response_error, + resolve_addressed_relay, send_to_relay, }, }; @@ -146,14 +147,15 @@ pub async fn get_execution_payload_bid( let (relay, max_timeout_ms) = match resolve_addressed_relay( relays, + state.default_relays(), auth.message.data.as_ref(), &req_headers, max_timeout_ms, ) .await { - // A dial with no time left, or refused for too many lookups, is a builder - // that did not answer + // A dial that runs out of time or lookup slots is a builder that did not + // answer Err(PbsClientError::NoBuilderResponse) => return Ok(None), resolved => resolved?, }; @@ -183,7 +185,7 @@ pub async fn get_execution_payload_bid( Ok(None) } Err(err) => { - error!(err = ?err, %relay_id); + error!(?err, %relay_id); builder_rejection(&err).map_or(Ok(None), Err) } } @@ -244,14 +246,7 @@ async fn send_get_execution_payload_bid( send_to_relay(request, &relay, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG).await?; let code = res.status(); if !code.is_success() { - // Read after the status check, so a body over the cap still reports the - // builder's status - let url = res.url().to_string(); - let error_msg = match read_chunked_body_with_max(res, MAX_SIZE_DEFAULT, &url).await { - Ok(body) => String::from_utf8_lossy(&body).into_owned(), - Err(err) => err.to_string(), - }; - return Err(PbsError::RelayResponse { error_msg, code: code.as_u16() }); + return Err(relay_response_error(res, "expected 200 or 204").await); } let (content_type, fork) = parse_response_encoding_and_fork(res.headers(), code.as_u16())?; diff --git a/crates/pbs/src/routes/submit_signed_beacon_block.rs b/crates/pbs/src/routes/submit_signed_beacon_block.rs index f8471bade..43344c428 100644 --- a/crates/pbs/src/routes/submit_signed_beacon_block.rs +++ b/crates/pbs/src/routes/submit_signed_beacon_block.rs @@ -1,10 +1,9 @@ use axum::{body::Bytes, extract::State, http::HeaderMap, response::IntoResponse}; use cb_common::wire::{ - BodyDeserializeError, EncodingType, content_type_encoding, get_user_agent, - require_consensus_version_header, + BodyDeserializeError, content_type_encoding, get_user_agent, require_consensus_version_header, }; use reqwest::StatusCode; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; use crate::{ PbsStateGuard, @@ -12,8 +11,8 @@ use crate::{ error::PbsClientError, state::{BuilderApiState, PbsState}, utils::{ - epbs_base_send_headers, join_detached_sends, post_ssz_expect_accepted, - record_beacon_status, record_request_failure, + epbs_base_send_headers, join_detached_sends, post_expect_accepted, record_beacon_status, + record_request_failure, }, }; @@ -34,9 +33,8 @@ pub async fn handle_submit_signed_beacon_block( } /// Implements https://ethereum.github.io/builder-specs/?urls.primaryName=dev#/Builder/submitSignedBeaconBlock -/// Forwards the block bytes, undecoded, to every configured builder, since CB -/// keeps no auction state to know which bid won. Returns 202 whatever they -/// answer: the beacon node gossips the block anyway +/// Forwards the block, undecoded, to every configured builder, as CB does not +/// know which bid won, and answers 202 whatever they answer pub async fn submit_signed_beacon_block( body: Bytes, req_headers: HeaderMap, @@ -49,11 +47,8 @@ pub async fn submit_signed_beacon_block( if body.is_empty() { return Err(BodyDeserializeError::MissingBody.into()); } - // Forwarded undecoded as SSZ, so anything else, an unlabelled (JSON) body - // included, is a 415 - if content_type_encoding(&req_headers)? != EncodingType::Ssz { - return Err(BodyDeserializeError::UnsupportedMediaType.into()); - } + // An unlabelled block is JSON, the builder-specs default + let encoding = content_type_encoding(&req_headers)?; info!(ua, "new request"); @@ -64,10 +59,11 @@ pub async fn submit_signed_beacon_block( let (relay, body, headers) = (relay.clone(), body.clone(), send_headers.clone()); async move { let url = relay.submit_signed_beacon_block_url()?; - post_ssz_expect_accepted( + post_expect_accepted( &relay, url, body, + encoding, headers, timeout_ms, SUBMIT_SIGNED_BEACON_BLOCK_ENDPOINT_TAG, @@ -82,15 +78,11 @@ pub async fn submit_signed_beacon_block( .filter(|(res, relay)| match res { Ok(()) => true, Err(err) => { - debug!(relay_id = relay.id.as_ref(), %err, "builder did not accept the block; only the winner accepts"); + debug!(relay_id = relay.id.as_ref(), %err, "builder did not accept the block"); false } }) .count(); - if accepted == 0 { - warn!(addressed = relays.len(), "no builder accepted the signed beacon block"); - } else { - info!(accepted, addressed = relays.len(), "signed beacon block accepted"); - } + info!(accepted, relays = relays.len(), "signed beacon block forwarded"); Ok(()) } diff --git a/crates/pbs/src/state.rs b/crates/pbs/src/state.rs index bd683e5f4..1ca181bff 100644 --- a/crates/pbs/src/state.rs +++ b/crates/pbs/src/state.rs @@ -51,6 +51,11 @@ where &self.config.all_relays } + /// Returns the default relays, `[[relays]]`, in config order + pub fn default_relays(&self) -> &[RelayClient] { + &self.config.relays + } + /// Returns the PBS config and relay clients for the given validator pubkey. /// If the pubkey is not found in any mux, the default configs are /// returned diff --git a/crates/pbs/src/utils.rs b/crates/pbs/src/utils.rs index 1ec49bc91..cd416f33b 100644 --- a/crates/pbs/src/utils.rs +++ b/crates/pbs/src/utils.rs @@ -15,15 +15,15 @@ use cb_common::{ }; use futures::future::join_all; use reqwest::{ - StatusCode, - header::{CONTENT_TYPE, HeaderMap, USER_AGENT}, + Response, StatusCode, + header::{CONTENT_TYPE, HOST, HeaderMap, USER_AGENT}, }; use tracing::{Instrument, debug, error, warn}; use url::Url; use crate::{ constants::{MAX_SIZE_DEFAULT, TIMEOUT_ERROR_CODE_STR}, - dial::{auth_data_address, dial_relay}, + dial::{auth_data_address, dial_relay, host_url}, error::PbsClientError, metrics::{BEACON_NODE_STATUS, RELAY_LATENCY, RELAY_STATUS_CODE}, }; @@ -104,13 +104,13 @@ pub(crate) fn log_mux_selection( } } -/// POSTs an SSZ body to a builder; 202 Accepted is the only success. A failed -/// response's body, capped at `MAX_SIZE_DEFAULT`, becomes the error message. -/// Returns the request latency. -pub(crate) async fn post_ssz_expect_accepted( +/// POSTs `body` in `encoding` to a builder and returns the latency; 202 is the +/// only success +pub(crate) async fn post_expect_accepted( relay: &RelayClient, url: Url, body: Bytes, + encoding: EncodingType, headers: HeaderMap, timeout_ms: u64, tag: &str, @@ -120,22 +120,26 @@ pub(crate) async fn post_ssz_expect_accepted( .post(url) .timeout(Duration::from_millis(timeout_ms)) .headers(headers) - .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .header(CONTENT_TYPE, encoding.content_type_header().clone()) .body(body); let (res, latency) = send_to_relay(req, relay, tag).await?; - let code = res.status(); - if code == StatusCode::ACCEPTED { + if res.status() == StatusCode::ACCEPTED { return Ok(latency); } - // Read after the status check, so a body over the cap still reports the - // builder's status + Err(relay_response_error(res, "expected 202").await) +} + +/// A failed builder response as an error; a body over the cap still reports the +/// status +pub(crate) async fn relay_response_error(res: Response, if_empty: &str) -> PbsError { + let code = res.status().as_u16(); let url = res.url().to_string(); let error_msg = match read_chunked_body_with_max(res, MAX_SIZE_DEFAULT, &url).await { Ok(body) if !body.is_empty() => String::from_utf8_lossy(&body).into_owned(), - Ok(_) => "expected 202".to_string(), + Ok(_) => if_empty.to_string(), Err(err) => err.to_string(), }; - Err(PbsError::RelayResponse { error_msg, code: code.as_u16() }) + PbsError::RelayResponse { error_msg, code } } /// A gwei amount in ETH, for logs @@ -187,32 +191,24 @@ pub fn check_gas_limit(gas_limit: u64, parent_gas_limit: u64) -> bool { true } -/// The first configured relay that the address in `auth_data` names, by -/// hostname or, for a URL, by origin; otherwise a relay that dials it. Also -/// returns what `timeout_ms` leaves after setting up that dial. +/// The relay `auth_data` names among `relays`, then `default_relays`, else a +/// dial to it, with what the dial's setup left of `timeout_ms` pub(crate) async fn resolve_addressed_relay( relays: &[RelayClient], + default_relays: &[RelayClient], auth_data: &[u8], req_headers: &HeaderMap, timeout_ms: u64, ) -> Result<(RelayClient, u64), PbsClientError> { let address = auth_data_address(auth_data); - let by_host = relays.iter().find(|relay| { - relay.config.entry.url.host_str().is_some_and(|host| host.as_bytes() == address) - }); - if let Some(relay) = by_host { - return Ok((relay.clone(), timeout_ms)); - } let data_url = decode_auth_data_url(address); - let by_origin = data_url.as_ref().and_then(|data_url| { - relays.iter().find(|relay| { - let url = &relay.config.entry.url; - url.scheme() == data_url.scheme() && - url.host() == data_url.host() && - url.port_or_known_default() == data_url.port_or_known_default() - }) - }); - if let Some(relay) = by_origin { + // A relay configured for other keys is reached with its own settings rather + // than dialed without them + if let Some(relay) = relays + .iter() + .chain(default_relays) + .find(|relay| names_relay(relay, address, data_url.as_ref())) + { return Ok((relay.clone(), timeout_ms)); } // Every Commit-Boost dial carries this header, so a request dialed back into @@ -224,6 +220,15 @@ pub(crate) async fn resolve_addressed_relay( ); return Err(PbsClientError::AuthDataMismatch); } + // A key without builder config sends this Commit-Boost's host, which a dial + // would reach on port 443, not here + if is_request_host(req_headers, address) { + warn!( + auth_data = ?String::from_utf8_lossy(auth_data), + "auth data names the host this request was sent to, not dialing" + ); + return Err(PbsClientError::AuthDataMismatch); + } let started = Instant::now(); let relay = dial_relay(data_url, address, Duration::from_millis(timeout_ms)).await?; let left_ms = timeout_ms.saturating_sub(started.elapsed().as_millis() as u64); @@ -234,6 +239,29 @@ pub(crate) async fn resolve_addressed_relay( Ok((relay, left_ms)) } +/// Whether `address` is the relay's hostname or, as a URL, its scheme, host and +/// port +fn names_relay(relay: &RelayClient, address: &[u8], data_url: Option<&Url>) -> bool { + let url = &relay.config.entry.url; + match data_url { + None => url.host_str().is_some_and(|host| host.as_bytes() == address), + Some(data_url) => { + url.scheme() == data_url.scheme() && + url.host() == data_url.host() && + url.port_or_known_default() == data_url.port_or_known_default() + } + } +} + +/// Whether `address` is the host in the request's `Host` header, in the form of +/// default auth data +fn is_request_host(req_headers: &HeaderMap, address: &[u8]) -> bool { + req_headers + .get(HOST) + .and_then(|host| host_url(host.to_str().ok()?)) + .is_some_and(|url| url.host_str().is_some_and(|host| host.as_bytes() == address)) +} + #[cfg(test)] mod tests { use cb_common::{ @@ -264,6 +292,14 @@ mod tests { RelayClient::new(config).unwrap() } + /// Headers of a request from a Commit-Boost, so a miss is an error, not a + /// dial + fn from_cb() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(HEADER_VERSION_KEY, reqwest::header::HeaderValue::from_static("test")); + headers + } + #[tokio::test] async fn resolve_relay_by_auth_data() { let relays = vec![ @@ -271,11 +307,9 @@ mod tests { test_relay("https://builder-b.example.com:8443/eth"), test_relay("http://[::1]:18550"), ]; - // From a Commit-Boost, so a miss is an error, not a dial - let mut from_cb = HeaderMap::new(); - from_cb.insert(HEADER_VERSION_KEY, reqwest::header::HeaderValue::from_static("test")); + let from_cb = from_cb(); let host = async |data: &[u8]| -> Option { - resolve_addressed_relay(&relays, data, &from_cb, 0) + resolve_addressed_relay(&relays, &[], data, &from_cb, 0) .await .ok() .map(|(relay, _)| relay.config.entry.url.host_str().unwrap().to_string()) @@ -290,7 +324,7 @@ mod tests { host(b"https://builder-a.example.com:443/eth").await.as_deref(), Some("builder-a.example.com") ); - assert!(host(b"http://builder-a.example.com").await.is_none()); + assert!(host(b"http://builder-a.example.com:443").await.is_none()); assert!(host(b"https://builder-b.example.com").await.is_none()); // Parameters after `?` are for the builder and do not affect routing assert_eq!( @@ -298,4 +332,66 @@ mod tests { Some("builder-a.example.com") ); } + + #[tokio::test] + async fn resolve_falls_back_to_default_relays_in_config_order() { + let own = vec![test_relay("https://builder-a.example.com/mux")]; + let default_relays = vec![ + test_relay("https://builder-a.example.com:8443"), + test_relay("https://builder-b.example.com:8443/eth"), + test_relay("https://builder-b.example.com:9000"), + test_relay("https://builder-b.example.com:9000/second"), + ]; + let from_cb = from_cb(); + let url = async |data: &[u8]| { + resolve_addressed_relay(&own, &default_relays, data, &from_cb, 0) + .await + .ok() + .map(|(relay, _)| relay.config.entry.url.to_string()) + }; + assert_eq!( + url(b"builder-a.example.com").await.as_deref(), + Some("https://builder-a.example.com/mux") + ); + assert_eq!( + url(b"builder-b.example.com").await.as_deref(), + Some("https://builder-b.example.com:8443/eth") + ); + assert_eq!( + url(b"https://builder-b.example.com:9000").await.as_deref(), + Some("https://builder-b.example.com:9000/") + ); + } + + #[tokio::test] + async fn resolve_refuses_the_host_the_request_was_sent_to() { + let resolve = async |host: &'static str, data: &[u8]| { + let mut headers = HeaderMap::new(); + headers.insert(HOST, reqwest::header::HeaderValue::from_static(host)); + resolve_addressed_relay(&[], &[], data, &headers, 1000) + .await + .map(|(relay, _)| relay.config.entry.url.to_string()) + }; + for (host, data) in [ + ("1.1.1.1:18550", &b"1.1.1.1"[..]), + ("1.1.1.1", b"1.1.1.1"), + ("[2606:4700:4700:0:0:0:0:1111]:18550", b"[2606:4700:4700::1111]"), + ] { + let res = resolve(host, data).await; + assert!(matches!(res, Err(PbsClientError::AuthDataMismatch)), "{host}"); + } + assert_eq!(resolve("2.2.2.2:18550", b"1.1.1.1").await.unwrap(), "https://1.1.1.1/"); + } + + // Not from a Commit-Boost, so a miss is dialed at the address before `?`, + // and refused once the dial's setup leaves no time + #[tokio::test] + async fn resolve_dials_the_address_before_the_parameters() { + let headers = HeaderMap::new(); + let dial = + |timeout_ms| resolve_addressed_relay(&[], &[], b"1.1.1.1?ofac=1", &headers, timeout_ms); + let (relay, _) = dial(1000).await.unwrap(); + assert_eq!(relay.config.entry.url.as_str(), "https://1.1.1.1/"); + assert!(matches!(dial(0).await, Err(PbsClientError::NoBuilderResponse))); + } } diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 1c14de6c8..0f04c4f96 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -18,7 +18,7 @@ Normal PBS paths are unaffected. From the Gloas fork, apart from the `status` en |---|---|---| | `POST /eth/v1/builder/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}` | A bid request for the proposal slot | Asks the [addressed](#routing-by-auth-data) builder for its bid | | `POST /eth/v1/builder/builder_preferences/{proposer_pubkey}` | The proposer's `max_execution_payment` for one builder, ahead of the slot | Forwards it to the addressed builder | -| `POST /eth/v1/builder/beacon_blocks` | The signed beacon block that commits to the winning bid | Forwards it to every builder in `[[relays]]` and `[[mux.relays]]`; only the winning builder accepts it | +| `POST /eth/v1/builder/beacon_blocks` | The signed beacon block that commits to the winning bid | Forwards it to every builder in `[[relays]]` and `[[mux.relays]]` | With ePBS the beacon node validates each bid and weighs it against your builder config (`min_bid`, `builder_boost_factor`) and its local block, so Commit-Boost adds no redundant verification on the hot path. @@ -33,7 +33,7 @@ A `get_header = "stream"` relay is asked for ePBS bids over plain HTTP. From the fork, each validator key has a builder config: a list of entries, each a `url` and an `auth_data`. For every entry the beacon node sends a bid request to the entry's `url`, carrying its `auth_data` in a `SignedBuilderRequestAuth` that the validator signs. `auth_data` tells a builder that a request was meant for it, so a request signed for one builder is rejected by another. Unless the proposer and builder agree on another value, it is the builder's hostname. -To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and its `auth_data` is the hostname of the builder it stands for. Commit-Boost reads the `auth_data` of each request, finds the relay entry with that hostname and forwards the request there. If no relay entry has it, Commit-Boost dials the builder the `auth_data` names ([builders outside your config](#builders-outside-your-config)). Builder preferences are routed the same way. The signed block goes to every builder in your config, since only the one whose bid won accepts it. +To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and its `auth_data` is the hostname of the builder it stands for. Commit-Boost reads the `auth_data` of each request, finds the relay entry with that hostname and forwards the request there. If no relay entry has it, Commit-Boost dials the builder the `auth_data` names ([builders outside your config](#builders-outside-your-config)). Builder preferences are routed the same way. The signed block goes to every builder in your config, since Commit-Boost does not track which bid won. ## Setup @@ -72,21 +72,21 @@ The call is `POST /eth/v1/validator/{pubkey}/builder_config`, authenticated with ## Routing by auth data -Commit-Boost sends a bid or preferences request to the first builder serving the proposer's key (the relays of its `[[mux]]`, otherwise `[[relays]]`) whose URL hostname equals the request's `auth_data`, byte for byte. That is the [builder specs](https://github.com/ethereum/builder-specs/blob/main/specs/gloas/validator.md#default-auth-data) default auth data: the lowercase hostname, without scheme, port or path, with an IPv6 address in brackets, such as `[::1]`. Builders on one host share it, so only the first is asked. +Commit-Boost sends a bid or preferences request to the first builder whose URL hostname equals the request's `auth_data`, byte for byte. That is the [builder specs](https://github.com/ethereum/builder-specs/blob/main/specs/gloas/validator.md#default-auth-data) default auth data: the lowercase ASCII hostname (punycode for an internationalized name), without scheme, port or path, with an IPv6 address compressed and in brackets, such as `[::1]`. Builders on one host share it, so only the first is asked. Commit-Boost looks first among the relays serving the proposer's key (those of its `[[mux]]`, otherwise `[[relays]]`), then among `[[relays]]` in config order, so a key in a mux still reaches a `[[relays]]` builder through that entry, with its headers, path and port. A builder listed only in another mux's `[[mux.relays]]` is not used for this key: it is dialed like a builder outside your config. -`auth_data` can also be an `http` or `https` URL, a form some builders agree out of band. It then matches the first builder whose `url` has the same scheme, host and port; the path and the pubkey in the relay URL do not count. Any other value agreed with a builder does not route through Commit-Boost. When no builder matches, Commit-Boost [dials the builder](#builders-outside-your-config) the `auth_data` names, and answers `400` when the `auth_data` is neither a hostname nor an `http(s)` URL. +`auth_data` can also be an `http` or `https` URL agreed with a builder. It then matches the first builder whose `url` has the same scheme, host and port; the path and the pubkey in the relay URL do not count, so builders on one origin share it. Any other value agreed with a builder does not route through Commit-Boost. When no builder matches, Commit-Boost [dials the builder](#builders-outside-your-config) the `auth_data` names, and answers `400` when the `auth_data` is neither in that hostname form nor an `http(s)` URL. Either form may end in `?` and form-encoded parameters for the builder, such as `builder-a.example.com?ofac=1`. Commit-Boost routes on the part before `?`. The parameters reach the builder inside the signed auth, so set them only for a builder that accepts them; one that does not answers `400`. ## Builders outside your config {#builders-outside-your-config} -A key's builder config may name a builder you have not added as a relay entry. Commit-Boost dials that builder anyway: a hostname at `https://` on the default port, and a URL at its scheme, host and port. It answers `400` instead when the host does not resolve, or resolves to any address that is not public unicast, such as a loopback, private, link-local or `100.64.0.0/10` address, so a builder on your own network has to be a relay entry. These requests connect directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`, follow no redirects and do not carry your relay `headers`. A builder reached this way gets the signed block over gossip, not from Commit-Boost. Neither `[[relays]]` nor a mux's relays limit which builders a key can reach this way, and anyone who can reach Commit-Boost's port can make it dial a public host, so keep that port private. Commit-Boost looks up at most 32 of these hosts at once and answers further requests as if the builder had not responded. +A key's builder config may name a builder you have not added as a relay entry. Commit-Boost dials that builder anyway: a hostname at `https://` on the default port, and a URL at its scheme, host and port. It answers `400` instead when the host does not resolve, or resolves to any address that is not public unicast, such as a loopback, private, link-local or `100.64.0.0/10` address, so a builder on your own network has to be a relay entry. These requests connect directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`, follow no redirects and do not carry your relay `headers`. A builder reached this way gets the signed block over gossip, unless it is also a relay entry in `[[relays]]` or a mux. Neither `[[relays]]` nor a mux's relays limit which builders a key can reach this way, and anyone who can reach Commit-Boost's port can make it dial a public host, so keep that port private. Commit-Boost looks up at most 32 of these hosts at once and answers further requests as if the builder had not responded, as it does when a lookup runs out of the request's time. -A key without builder config sends Commit-Boost's own hostname as its `auth_data`, so it gets no bids. Commit-Boost answers `400` when that hostname resolves to a loopback or private address, as it usually does, and otherwise dials `https://` on port 443, where Commit-Boost itself does not listen. Commit-Boost never dials for a request that came from another Commit-Boost, so chained Commit-Boosts route only through relay entries. +On a validator client whose builder URL is Commit-Boost, a key without builder config sends Commit-Boost's own hostname as its `auth_data`, so it gets no bids: unless a relay entry has that hostname, which then gets the request, Commit-Boost answers `400` without dialing. A client that sends Commit-Boost's full URL instead also gets `400`. Commit-Boost never dials for a request that came from another Commit-Boost, so chained Commit-Boosts route only through relay entries. ## Timing -The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts your `proposer_deadline_buffer_ms` from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. +The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts your `proposer_deadline_buffer_ms` from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. For a builder outside your config, the time its lookup took is subtracted too. Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon node's remaining time: enough for the bid to get back from Commit-Boost to the beacon node and for the beacon node to process it. If no time is left, Commit-Boost does not ask the builder. With `0`, Commit-Boost refuses to start or reload: the builder gets the whole deadline, so a bid it sends at the deadline reaches the beacon node late. @@ -98,8 +98,8 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Question | Series | |---|---| -| What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences, `202` for the signed block whatever the builders answer, `4xx` for a rejected request, `500` when no builder accepted the preferences | -| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"`, except one Commit-Boost [refuses to dial](#builders-outside-your-config), which gets no request | +| What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences, `202` for the signed block whatever the builders answer, `4xx` for a rejected request, `500` when no builder accepted the preferences or on an internal error | +| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"`; a dial Commit-Boost [refuses](#builders-outside-your-config), or whose lookup fails, times out or finds no free slot, sends no request and is not counted | | How fast are builders? | `cb_pbs_relay_latency`, by `relay_id` | | What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from each bid a builder serves | @@ -107,10 +107,8 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Symptom | Cause | Fix | |---|---|---| -| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a hostname nor an `http(s)` URL, or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | -| Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config). A key without builder config sends Commit-Boost's own hostname, which usually does | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | +| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | +| Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config) | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | | Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | -| The signed block gets `415` | The beacon node sends it as JSON | Configure the beacon node to send SSZ | -| Commit-Boost logs "no builder accepted the signed beacon block" | The winning bid came from a builder outside your config, which gets the block over gossip, or your builders did not accept it. The beacon node gossips the block either way | If the bid came from a configured builder, check its answer in `cb_pbs_relay_status_code_total{endpoint="submit_signed_beacon_block"}` | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/docs/get_started/running/metrics-catalog.md b/docs/docs/get_started/running/metrics-catalog.md index 4db35ea41..74aba5978 100644 --- a/docs/docs/get_started/running/metrics-catalog.md +++ b/docs/docs/get_started/running/metrics-catalog.md @@ -14,7 +14,7 @@ PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. Th | Metric name (wire) | Type | Labels | Description | |---|---|---|---| -| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (timeout, DNS or connection failure), and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. | +| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (timeout, DNS or connection failure; on `get_header_stream`, only the deadline), and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. | | `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Under `get_header_stream` the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | | `cb_pbs_relay_last_slot` | Gauge | `relay_id` | Latest slot for which a relay delivered a bid. Set to the current slot on each successful `get_header` bid and (unreleased, from v0.12.0-rc1) each ePBS bid served from that relay. | | `cb_pbs_relay_header_value` | Gauge | `relay_id` | Value in gwei of the bid a relay delivered: for `get_header`, the bid's value converted from wei (รท 1e9). (unreleased, from v0.12.0-rc1) For ePBS, the returned bid's `value`, without its execution payment. | diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 772066868..3e6dfdb4e 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -8,7 +8,7 @@ version.workspace = true alloy.workspace = true axum.workspace = true cb-common.workspace = true -cb-pbs = { workspace = true, features = ["testing-flags"] } +cb-pbs.workspace = true cb-signer.workspace = true eyre.workspace = true ethereum_ssz.workspace = true @@ -34,3 +34,4 @@ url.workspace = true [dev-dependencies] tracing-test.workspace = true cb-common = { path = "../crates/common", features = ["testing-flags"] } +cb-pbs = { path = "../crates/pbs", features = ["testing-flags"] } diff --git a/tests/src/mock_relay.rs b/tests/src/mock_relay.rs index c1ad4b813..e9785d2e8 100644 --- a/tests/src/mock_relay.rs +++ b/tests/src/mock_relay.rs @@ -41,8 +41,9 @@ use cb_common::{ }, }; use cb_pbs::{ - GET_HEADER_ENDPOINT_TAG, MAX_SIZE_SUBMIT_BLOCK_RESPONSE, REGISTER_VALIDATOR_ENDPOINT_TAG, - STATUS_ENDPOINT_TAG, SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, + GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, GET_HEADER_ENDPOINT_TAG, + MAX_SIZE_SUBMIT_BLOCK_RESPONSE, REGISTER_VALIDATOR_ENDPOINT_TAG, STATUS_ENDPOINT_TAG, + SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, }; use lh_types::{KzgProof, Slot}; use reqwest::header::{ACCEPT, CONTENT_TYPE}; @@ -103,6 +104,8 @@ pub struct MockRelayState { /// `slot` of the last signed beacon block forwarded, decoded from the SSZ /// body PBS sent, so a test can assert the block survived the hop received_block_slot: RwLock>, + /// Content-Type and body of the last signed beacon block forwarded + received_block_request: RwLock, axum::body::Bytes)>>, /// The last `BuilderPreferencesRequest` submitted, so a test can assert /// both the preferences and the auth were forwarded unchanged received_preferences: RwLock>, @@ -161,6 +164,11 @@ impl MockRelayState { pub fn received_signed_beacon_block(&self) -> u64 { self.received_signed_beacon_block.load(Ordering::Relaxed) } + /// The last forwarded block's Content-Type and body + pub fn received_block_request(&self) -> Option<(Option, axum::body::Bytes)> { + self.received_block_request.read().unwrap().clone() + } + /// `slot` of the last signed beacon block PBS forwarded pub fn received_block_slot(&self) -> Option { *self.received_block_slot.read().unwrap() @@ -248,6 +256,7 @@ impl MockRelayState { received_builder_preferences: Default::default(), received_signed_beacon_block: Default::default(), received_block_slot: RwLock::new(None), + received_block_request: RwLock::new(None), received_preferences: RwLock::new(None), received_preferences_pubkey: RwLock::new(None), bid_delay_ms: None, @@ -401,6 +410,7 @@ async fn handle_get_execution_payload_bid( body: axum::body::Bytes, ) -> Response { state.received_execution_payload_bid.fetch_add(1, Ordering::Relaxed); + state.record_api_key(GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, &headers); // Builders MUST 400 a bid request without both timing headers let header_u64 = |name| headers.get(name).and_then(|v| v.to_str().ok()).and_then(|v| v.parse::().ok()); @@ -653,15 +663,16 @@ async fn handle_submit_builder_preferences( StatusCode::ACCEPTED.into_response() } -/// Decodes the forwarded block (PBS always sends SSZ with -/// `Eth-Consensus-Version`), records its slot, and 202s -/// unless the test overrode the response. +/// Records the forwarded block's Content-Type and body, and its slot when it is +/// SSZ, and 202s unless the test overrode the response async fn handle_submit_signed_beacon_block( headers: HeaderMap, State(state): State>, body: axum::body::Bytes, ) -> Response { state.received_signed_beacon_block.fetch_add(1, Ordering::Relaxed); + let content_type = headers.get(CONTENT_TYPE).and_then(|v| v.to_str().ok()).map(String::from); + *state.received_block_request.write().unwrap() = Some((content_type, body.clone())); if let Some(fork) = get_consensus_version_header(&headers) && let Ok(block) = SignedBeaconBlock::from_ssz_bytes_by_fork(&body, fork) diff --git a/tests/src/mock_validator.rs b/tests/src/mock_validator.rs index c433d322f..d3d587dfc 100644 --- a/tests/src/mock_validator.rs +++ b/tests/src/mock_validator.rs @@ -11,11 +11,11 @@ use cb_common::{ use lh_types::ForkName; use reqwest::{ Response, - header::{ACCEPT, CONTENT_TYPE}, + header::{ACCEPT, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue}, }; use ssz::Encode; -use crate::utils::{TEST_PROPOSER_PUBKEY, generate_mock_relay}; +use crate::utils::{API_KEY, HEADER_API_KEY, TEST_PROPOSER_PUBKEY, generate_mock_relay}; /// Timeout a test beacon node advertises on bid requests; long enough that the /// deadline never bites in tests. @@ -31,8 +31,13 @@ impl MockValidator { "0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae", )?; let mut comm_boost = generate_mock_relay(port, pubkey)?; - // Like a beacon node, without X-CommitBoost-Version, which stops a dial - comm_boost.client = reqwest::Client::new(); + // Like a beacon node: no X-CommitBoost-Version, which stops a dial. The + // API key differs from the relays', so one forwarded from here shows + let headers = HeaderMap::from_iter([( + HeaderName::from_static(HEADER_API_KEY), + HeaderValue::from_static(API_KEY), + )]); + comm_boost.client = reqwest::Client::builder().default_headers(headers).build()?; Ok(Self { comm_boost }) } diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 1839b3c9c..09412d2aa 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -11,7 +11,7 @@ use cb_common::{ config::{ COMMIT_BOOST_IMAGE_DEFAULT, CommitBoostConfig, GetHeaderTransport, HeaderSource, LogsSettings, ModuleKind, ModuleSigningConfig, PbsConfig, PbsModuleConfig, RelayConfig, - ReverseProxyHeaderSetup, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, + ReverseProxyHeaderSetup, RuntimeMuxConfig, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_PORT_DEFAULT, SignerConfig, SignerType, StartSignerConfig, StaticModuleConfig, StaticPbsConfig, TlsMode, }, @@ -68,7 +68,7 @@ fn mock_relay_config(port: u16, pubkey: BlsPublicKey) -> Result { id: None, headers: Some(HashMap::from([( HEADER_API_KEY.into(), - HeaderSource::Literal(API_KEY.into()), + HeaderSource::Literal(RELAY_API_KEY.into()), )])), get_params: None, get_header: GetHeaderTransport::Http, @@ -347,6 +347,42 @@ pub async fn setup_relays_on_hosts( /// Poll /status until PBS and its relays are up. relay_check makes a 200 mean /// the whole chain is ready; the fixed 100ms sleep used elsewhere flakes under /// parallel suite load. +/// A PBS with a `[[relays]]` relay and a relay only in the `[[mux]]` of +/// `muxed_key`, both on host 0.0.0.0 +pub struct MuxedRelays { + pub validator: MockValidator, + pub muxed_key: BlsPublicKey, + pub default_state: Arc, + pub default_port: u16, + pub mux_state: Arc, + pub mux_port: u16, +} + +pub async fn setup_mux_relays(chain: Chain) -> Result { + let (default_state, default_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let (mux_state, mux_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let default_relay = generate_mock_relay(default_port, default_state.signer.public_key())?; + let mux_relay = generate_mock_relay(mux_port, mux_state.signer.public_key())?; + + let pbs_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]); + config.all_relays = vec![mux_relay.clone(), default_relay]; + let muxed_key = random_secret().public_key(); + config.mux_lookup = Some(HashMap::from([(muxed_key.clone(), RuntimeMuxConfig { + id: "test".to_string(), + config: config.pbs_config.clone(), + relays: vec![mux_relay], + })])); + let state = PbsState::new(config, PathBuf::new()); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); + let validator = MockValidator::new(pbs_port)?; + wait_for_ready(&validator).await?; + Ok(MuxedRelays { validator, muxed_key, default_state, default_port, mux_state, mux_port }) +} + pub async fn wait_for_ready(mock_validator: &MockValidator) -> Result<()> { for _ in 0..100 { if let Ok(res) = mock_validator.do_get_status().await && diff --git a/tests/tests/pbs_get_execution_payload_bid.rs b/tests/tests/pbs_get_execution_payload_bid.rs index 1d58cf52f..79d04f7e5 100644 --- a/tests/tests/pbs_get_execution_payload_bid.rs +++ b/tests/tests/pbs_get_execution_payload_bid.rs @@ -1,25 +1,21 @@ -use std::{collections::HashMap, path::PathBuf}; - use alloy::primitives::{B256, U256}; use cb_common::{ - config::RuntimeMuxConfig, pbs::{ GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, SignedExecutionPayloadBid, }, signer::random_secret, - types::Chain, + types::{BlsPublicKey, Chain}, utils::utcnow_ms, wire::{CONSENSUS_VERSION_HEADER, EncodingType}, }; -use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; +use cb_pbs::GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG; use cb_tests::{ mock_relay::MockRelayState, mock_validator::MockValidator, utils::{ - TEST_AUTH_DATA, TEST_PROPOSER_PUBKEY, generate_mock_relay, get_free_listener, - get_pbs_config, opaque_auth, setup_relay, setup_relays, setup_relays_on_hosts, - spawn_mock_relay, to_pbs_config, wait_for_ready, + RELAY_API_KEY, TEST_AUTH_DATA, TEST_PROPOSER_PUBKEY, generate_mock_relay, opaque_auth, + setup_mux_relays, setup_relay, setup_relays, setup_relays_on_hosts, spawn_mock_relay, }, }; use eyre::Result; @@ -168,60 +164,67 @@ async fn test_get_execution_payload_bid_dial() -> Result<()> { let res = get_json_bid(&mock_validator, &opaque_auth(&dial_auth_data, TEST_SLOT)).await?; assert_eq!(res.status(), StatusCode::OK); assert_eq!(dial_state.received_auth_data(), Some(dial_auth_data.clone())); + // A dial carries none of the relays' configured headers, the API key included + assert_eq!(dial_state.api_key_seen(GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG), None); // RelayClient::new's client sends the Commit-Boost version header mock_validator.comm_boost = RelayClient::new(mock_validator.comm_boost.config.as_ref().clone())?; let res = get_json_bid(&mock_validator, &opaque_auth(&dial_auth_data, TEST_SLOT)).await?; assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!( + body["message"], + "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder" + ); let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; assert_eq!(res.status(), StatusCode::OK); assert_eq!(dial_state.received_execution_payload_bid(), 1); assert_eq!(cfg_state.received_execution_payload_bid(), 1); + assert_eq!( + cfg_state.api_key_seen(GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG).as_deref(), + Some(RELAY_API_KEY) + ); Ok(()) } -/// A key listed in a `[[mux]]` is served by that mux's relays: its bid request -/// reaches the mux relay and not the default `[[relays]]` one. +/// A key in a `[[mux]]` is served by that mux's relays, and reaches a relay +/// configured only in `[[relays]]` through its entry, its API key included. A +/// relay only in a mux is not reached so by other keys: it is dialed instead #[tokio::test] async fn test_get_execution_payload_bid_mux_routes_to_mux_relays() -> Result<()> { - let chain = Chain::Hoodi; - let (default_state, default_port) = - spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; - let (mux_state, mux_port) = - spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; - let default_relay = generate_mock_relay(default_port, default_state.signer.public_key())?; - let mux_relay = generate_mock_relay(mux_port, mux_state.signer.public_key())?; - - let pbs_listener = get_free_listener().await; - let pbs_port = pbs_listener.local_addr()?.port(); - let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]); - config.all_relays = vec![mux_relay.clone(), default_relay]; - let muxed_key = random_secret().public_key(); - config.mux_lookup = Some(HashMap::from([(muxed_key.clone(), RuntimeMuxConfig { - id: "test".to_string(), - config: config.pbs_config.clone(), - relays: vec![mux_relay], - })])); - let state = PbsState::new(config, PathBuf::new()); - tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); - let mock_validator = MockValidator::new(pbs_port)?; - wait_for_ready(&mock_validator).await?; + let relays = setup_mux_relays(Chain::Hoodi).await?; + let bid = async |key: Option, auth_data: &[u8]| { + relays + .validator + .do_get_execution_payload_bid( + TEST_SLOT, + B256::ZERO, + B256::ZERO, + key, + Some(&opaque_auth(auth_data, TEST_SLOT)), + vec![EncodingType::Json], + ) + .await + }; + let muxed = Some(relays.muxed_key.clone()); - let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT); - let res = mock_validator - .do_get_execution_payload_bid( - TEST_SLOT, - B256::ZERO, - B256::ZERO, - Some(muxed_key), - Some(&auth), - vec![EncodingType::Json], - ) - .await?; - assert_eq!(res.status(), StatusCode::OK); - assert_eq!(mux_state.received_execution_payload_bid(), 1); - assert_eq!(default_state.received_execution_payload_bid(), 0); + assert_eq!(bid(muxed.clone(), TEST_AUTH_DATA).await?.status(), StatusCode::OK); + assert_eq!(relays.mux_state.received_execution_payload_bid(), 1); + assert_eq!(relays.default_state.received_execution_payload_bid(), 0); + + let to_default = format!("http://0.0.0.0:{}", relays.default_port); + assert_eq!(bid(muxed, to_default.as_bytes()).await?.status(), StatusCode::OK); + assert_eq!(relays.default_state.received_execution_payload_bid(), 1); + assert_eq!( + relays.default_state.api_key_seen(GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG).as_deref(), + Some(RELAY_API_KEY) + ); + + // Dialed, and the dial check refuses 0.0.0.0 + let to_mux = format!("http://0.0.0.0:{}", relays.mux_port); + assert_eq!(bid(None, to_mux.as_bytes()).await?.status(), StatusCode::BAD_REQUEST); + assert_eq!(relays.mux_state.received_execution_payload_bid(), 1); Ok(()) } @@ -569,8 +572,8 @@ async fn test_get_execution_payload_bid_relay_timing_headers() -> Result<()> { Ok(()) } -/// Auth data naming Commit-Boost's own URL is dialed once, and that hop, a -/// request from a Commit-Boost, is not dialed on: the builder's 400 +/// Auth data naming Commit-Boost's own URL is dialed once: the second hop, from +/// a Commit-Boost, answers 400 and the beacon node gets it #[tokio::test] async fn test_get_execution_payload_bid_dial_self_400() -> Result<()> { // As if Commit-Boost's own address were public diff --git a/tests/tests/pbs_get_header_ws.rs b/tests/tests/pbs_get_header_ws.rs index e132a70d1..abf9a7f45 100644 --- a/tests/tests/pbs_get_header_ws.rs +++ b/tests/tests/pbs_get_header_ws.rs @@ -19,7 +19,7 @@ use cb_tests::{ mock_validator::MockValidator, mock_ws_relay::{MockWsRelayState, start_mock_ws_relay_service}, utils::{ - API_KEY, generate_mock_relay, generate_mock_stream_relay, + RELAY_API_KEY, generate_mock_relay, generate_mock_stream_relay, generate_mock_stream_relay_with_timing_games, get_free_listener, get_pbs_config, setup_test_env, to_pbs_config, }, @@ -177,7 +177,7 @@ async fn test_get_header_ws_sends_api_key() -> Result<()> { assert_eq!(code, StatusCode::OK); let request = relay_state.last_request().expect("relay saw no request"); - assert_eq!(request.api_key.as_deref(), Some(API_KEY)); + assert_eq!(request.api_key.as_deref(), Some(RELAY_API_KEY)); Ok(()) } diff --git a/tests/tests/pbs_submit_builder_preferences.rs b/tests/tests/pbs_submit_builder_preferences.rs index 302fece01..53a5208d1 100644 --- a/tests/tests/pbs_submit_builder_preferences.rs +++ b/tests/tests/pbs_submit_builder_preferences.rs @@ -8,8 +8,8 @@ use cb_common::{ use cb_tests::{ mock_relay::MockRelayState, utils::{ - TEST_AUTH_DATA, generate_mock_relay, opaque_auth, setup_relay, setup_relays, - setup_relays_on_hosts, spawn_mock_relay, + TEST_AUTH_DATA, generate_mock_relay, opaque_auth, setup_mux_relays, setup_relay, + setup_relays, setup_relays_on_hosts, spawn_mock_relay, }, }; use eyre::Result; @@ -173,9 +173,8 @@ async fn test_submit_builder_preferences_past_slot_forwarded() -> Result<()> { Ok(()) } -/// Auth data that is neither a hostname nor a URL is a 400 with the builder's -/// data-mismatch message: CB has no builder to proxy to, and proposer-private -/// preferences must never broadcast to builders the proposer did not address. +/// Auth data that is neither a hostname nor a URL is a 400, and no builder is +/// asked: preferences go only to the builder the proposer addressed #[tokio::test] async fn test_submit_builder_preferences_unmatched_auth_data_400() -> Result<()> { let chain = Chain::Hoodi; @@ -289,3 +288,25 @@ async fn test_submit_builder_preferences_dial() -> Result<()> { assert_eq!(dial_state.received_builder_preferences(), 1); Ok(()) } + +/// A key in a `[[mux]]` sends preferences to a builder configured only in +/// `[[relays]]` through that entry. A builder only in the mux is not reached so +/// by other keys: it is dialed, and the dial check refuses 0.0.0.0 +#[tokio::test] +async fn test_submit_builder_preferences_mux_key_reaches_a_relays_entry() -> Result<()> { + let relays = setup_mux_relays(Chain::Hoodi).await?; + let mock_validator = &relays.validator; + let (muxed_key, default_port, mux_port) = + (relays.muxed_key.clone(), relays.default_port, relays.mux_port); + let (default_state, mux_state) = (&relays.default_state, &relays.mux_state); + let submit = async |key, port| { + let auth = opaque_auth(format!("http://0.0.0.0:{port}").as_bytes(), TEST_SLOT); + let request = preferences(auth, TEST_MAX_EXECUTION_PAYMENT); + mock_validator.do_submit_builder_preferences(key, &request, EncodingType::Ssz).await + }; + assert_eq!(submit(Some(muxed_key), default_port).await?.status(), StatusCode::ACCEPTED); + assert_eq!(submit(None, mux_port).await?.status(), StatusCode::BAD_REQUEST); + assert_eq!(default_state.received_builder_preferences(), 1); + assert_eq!(mux_state.received_builder_preferences(), 0); + Ok(()) +} diff --git a/tests/tests/pbs_submit_signed_beacon_block.rs b/tests/tests/pbs_submit_signed_beacon_block.rs index 2e05c2e3e..a9462db38 100644 --- a/tests/tests/pbs_submit_signed_beacon_block.rs +++ b/tests/tests/pbs_submit_signed_beacon_block.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use cb_common::{ pbs::SignedBeaconBlock, signer::random_secret, @@ -5,9 +7,14 @@ use cb_common::{ utils::TestRandomSeed, wire::{CONSENSUS_VERSION_HEADER, EncodingType}, }; +use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use cb_tests::{ mock_relay::MockRelayState, - utils::{generate_mock_relay, setup_relay, setup_relays}, + mock_validator::MockValidator, + utils::{ + generate_mock_relay, get_free_listener, get_pbs_config, setup_relay, setup_relays, + spawn_mock_relay, to_pbs_config, wait_for_ready, + }, }; use eyre::Result; use lh_types::{MainnetEthSpec, SignedBeaconBlockGloas, Slot}; @@ -56,6 +63,34 @@ async fn test_submit_signed_beacon_block_broadcasts_to_all_relays() -> Result<() Ok(()) } +/// A relay listed only in a `[[mux]]` gets the block too +#[tokio::test] +async fn test_submit_signed_beacon_block_reaches_mux_relays() -> Result<()> { + let chain = Chain::Hoodi; + let (default_state, default_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let (mux_state, mux_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let default_relay = generate_mock_relay(default_port, default_state.signer.public_key())?; + let mux_relay = generate_mock_relay(mux_port, mux_state.signer.public_key())?; + + let pbs_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]); + config.all_relays = vec![default_relay, mux_relay]; + let state = PbsState::new(config, PathBuf::new()); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); + let mock_validator = MockValidator::new(pbs_port)?; + wait_for_ready(&mock_validator).await?; + + let block = gloas_block(TEST_SLOT); + let res = mock_validator.do_submit_signed_beacon_block(&block, EncodingType::Ssz).await?; + assert_eq!(res.status(), StatusCode::ACCEPTED); + assert_eq!(default_state.received_signed_beacon_block(), 1); + assert_eq!(mux_state.received_signed_beacon_block(), 1); + Ok(()) +} + /// Every builder rejecting the block is still a 202: the beacon node gossips /// the block anyway, so a builder's answer is not a failure. Each is asked. #[tokio::test] @@ -79,12 +114,42 @@ async fn test_submit_signed_beacon_block_broadcast_all_reject_202() -> Result<() Ok(()) } +/// The block reaches the builder undecoded, in the encoding it arrived in: +/// JSON, the builder-specs default when unlabelled, or SSZ +#[tokio::test] +async fn test_submit_signed_beacon_block_forwards_its_encoding() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let block = gloas_block(TEST_SLOT); + let (json_body, ssz_body) = (serde_json::to_vec(&block)?, block.as_ssz_bytes()); + let url = mock_validator.comm_boost.submit_signed_beacon_block_url()?; + + // (Content-Type sent, body, Content-Type the builder gets) + let cases = [ + // Neighbours differ, so each case's own assertions can fail + (None, &json_body, "application/json"), + (Some("application/octet-stream"), &ssz_body, "application/octet-stream"), + (Some("application/json"), &json_body, "application/json"), + ]; + for (sent, body, forwarded) in cases { + let mut req = mock_validator.comm_boost.client.post(url.clone()); + if let Some(content_type) = sent { + req = req.header(CONTENT_TYPE, content_type); + } + let res = req.header(CONSENSUS_VERSION_HEADER, "gloas").body(body.clone()).send().await?; + assert_eq!(res.status(), StatusCode::ACCEPTED, "{sent:?}"); + let (content_type, received) = state.received_block_request().expect("forwarded"); + assert_eq!(content_type.as_deref(), Some(forwarded), "{sent:?}"); + assert_eq!(&received, body, "{sent:?}: forwarded byte for byte"); + } + assert_eq!(state.received_signed_beacon_block(), 3); + Ok(()) +} + /// Requests refused before anything is broadcast, each with an ErrorMessage /// body. `Eth-Consensus-Version` is required (builder-specs #165) and checked -/// first, and a missing body is named as such rather than as an undecodable -/// one. The reveal must be SSZ: a JSON reveal, and an unlabeled one -/// (builder-specs defaults it to JSON), is a 415 rather than forwarded -/// mislabeled to fail opaquely at the builder. +/// first, a missing body is named as such, and a type other than JSON or SSZ +/// is a 415. #[tokio::test] async fn test_submit_signed_beacon_block_rejected_before_broadcast() -> Result<()> { let chain = Chain::Hoodi; @@ -103,14 +168,7 @@ async fn test_submit_signed_beacon_block_rejected_before_broadcast() -> Result<( "missing consensus version", ), (None, Some("gloas"), vec![], StatusCode::BAD_REQUEST, "missing request body"), - ( - Some("application/json"), - Some("gloas"), - json_body, - StatusCode::UNSUPPORTED_MEDIA_TYPE, - "", - ), - (None, Some("gloas"), ssz_body, StatusCode::UNSUPPORTED_MEDIA_TYPE, ""), + (Some("text/plain"), Some("gloas"), ssz_body, StatusCode::UNSUPPORTED_MEDIA_TYPE, ""), ]; for (content_type, version, body, status, message) in cases { let mut req = mock_validator.comm_boost.client.post(url.clone()); From f4b798e1e4434f599dc9a37b359a2a1e895ec0b1 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Mon, 12 Oct 2026 13:21:41 -0700 Subject: [PATCH 20/24] fix(pbs): send the time left as the bid stream's X-Timeout-Ms builder-specs defines X-Timeout-Ms as the builder's time to respond from Date-Milliseconds, so allowing for the trip back is the builder's. The stream sent the time left less half its TCP connect time. A relay that measures its own round trip, for example with pings over the open stream, estimates better than one connect sample, and with both sides subtracting the stream ends early. The handshake's X-Timeout-Ms is the time left to Commit-Boost's deadline, as on HTTP and in v0.11.0, and tungstenite opens the connection. A relay that ends its stream at that deadline loses the frames it sends in the last one-way trip. --- crates/pbs/src/bid_stream.rs | 91 ++++---------------------- docs/docs/get_started/configuration.md | 1 - tests/src/mock_ws_relay.rs | 66 +------------------ tests/tests/pbs_get_header_ws.rs | 52 +-------------- 4 files changed, 15 insertions(+), 195 deletions(-) diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs index 6dd630024..322822c32 100644 --- a/crates/pbs/src/bid_stream.rs +++ b/crates/pbs/src/bid_stream.rs @@ -10,12 +10,11 @@ use std::{ collections::VecDeque, - io, sync::{Arc, OnceLock}, time::Duration, }; -use axum::http::{HeaderMap, HeaderValue, Request, Uri, header::USER_AGENT}; +use axum::http::{HeaderMap, HeaderValue, Request, header::USER_AGENT}; use cb_common::{ pbs::{ForkName, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, error::PbsError}, utils::utcnow_ms, @@ -24,14 +23,13 @@ use futures::StreamExt; use reqwest::StatusCode; use rustls::{ClientConfig, RootCertStore, crypto::aws_lc_rs}; use tokio::{ - net::{TcpStream, lookup_host}, + net::TcpStream, time::{Instant, sleep_until, timeout_at}, }; use tokio_tungstenite::{ - Connector, MaybeTlsStream, WebSocketStream, client_async_tls_with_config, + Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config, tungstenite::{ - Bytes, Error as WsError, Message, client::IntoClientRequest, error::UrlError, - protocol::WebSocketConfig, + Bytes, Error as WsError, Message, client::IntoClientRequest, protocol::WebSocketConfig, }, }; use tracing::{debug, warn}; @@ -108,7 +106,7 @@ pub(crate) async fn read_bid_stream( ) -> Result, (StatusCode, PbsError)> { let start_request = Instant::now(); let mut stream = match timeout_at(deadline, connect(request, deadline)).await { - Ok(Ok(Some(stream))) => stream, + Ok(Ok(stream)) => stream, // A relay with no bid can answer the handshake as it answers get_header Ok(Err(WsError::Http(res))) if res.status() == StatusCode::NO_CONTENT => { return Ok(Held { @@ -120,7 +118,7 @@ pub(crate) async fn read_bid_stream( }); } Ok(Err(err)) => return Err(connect_failed(&err)), - Ok(Ok(None)) | Err(_) => return Err((TIMEOUT_ERROR_STATUS, PbsError::WebSocketTimeout)), + Err(_) => return Err((TIMEOUT_ERROR_STATUS, PbsError::WebSocketTimeout)), }; let connect_latency = start_request.elapsed(); RELAY_STREAM_CONNECT_LATENCY @@ -191,19 +189,13 @@ pub(crate) async fn read_bid_stream( Ok(Held { frames, updates, connect_latency, first_frame_latency, invalid_frames }) } -/// Opens TCP first and times it, so `X-Timeout-Ms` ends the relay's stream one -/// downlink trip before `deadline`. None when no time would be left. +/// Opens the stream with the timing headers stamped now. As over HTTP, +/// `X-Timeout-Ms` is the time left to `deadline`; the trip back is the relay's. async fn connect( mut request: Request<()>, deadline: Instant, -) -> Result>>, WsError> { - let (socket, rtt) = connect_tcp(request.uri()).await?; - - let Some(timeout_ms) = - relay_timeout_ms(deadline.saturating_duration_since(Instant::now()), rtt) - else { - return Ok(None); - }; +) -> Result>, WsError> { + let timeout_ms = deadline.saturating_duration_since(Instant::now()).as_millis() as u64; let headers = request.headers_mut(); headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); @@ -213,47 +205,9 @@ async fn connect( .max_frame_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)); let connector = Connector::Rustls(tls_config().clone()); let (stream, _) = - client_async_tls_with_config(request, socket, Some(config), Some(connector)).await?; - - Ok(Some(stream)) -} - -/// The TCP handshake's duration is one round trip, free of clock skew. Name -/// resolution is left out of it. -async fn connect_tcp(uri: &Uri) -> Result<(TcpStream, Duration), WsError> { - let host = uri.host().ok_or(WsError::Url(UrlError::NoHostName))?; - let host = host.strip_prefix('[').and_then(|host| host.strip_suffix(']')).unwrap_or(host); - let port = uri - .port_u16() - .or(match uri.scheme_str() { - Some("wss") => Some(443), - Some("ws") => Some(80), - _ => None, - }) - .ok_or(WsError::Url(UrlError::UnsupportedUrlScheme))?; - - let mut last_err = None; - for addr in lookup_host((host, port)).await? { - let start = Instant::now(); - match TcpStream::connect(addr).await { - Ok(socket) => { - let rtt = start.elapsed(); - socket.set_nodelay(true)?; - return Ok((socket, rtt)); - } - Err(err) => last_err = Some(err), - } - } + connect_async_tls_with_config(request, Some(config), true, Some(connector)).await?; - Err(WsError::Io(last_err.unwrap_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "could not resolve to any address") - }))) -} - -/// The time left less half the round trip, the estimate of one downlink trip -fn relay_timeout_ms(time_left: Duration, rtt: Duration) -> Option { - let timeout_ms = time_left.saturating_sub(rtt / 2).as_millis() as u64; - (timeout_ms > 0).then_some(timeout_ms) + Ok(stream) } pub(crate) fn record_stream_failure(endpoint: &str, relay_id: &str, err: &PbsError) { @@ -361,27 +315,6 @@ mod tests { assert_eq!(fork_from_wire(forks.len() as u8), None); } - #[test] - fn test_relay_timeout_leaves_the_downlink_trip() { - let ms = Duration::from_millis; - assert_eq!(relay_timeout_ms(ms(1_000), ms(100)), Some(950)); - assert_eq!(relay_timeout_ms(ms(1_000), Duration::ZERO), Some(1_000)); - // Nothing left: no frame could land before the deadline - assert_eq!(relay_timeout_ms(ms(50), ms(100)), None); - assert_eq!(relay_timeout_ms(ms(40), ms(100)), None); - } - - // The host of an IPv6 url keeps its brackets, which name resolution rejects - #[tokio::test] - async fn test_connect_tcp_to_an_ipv6_literal() { - let Ok(listener) = tokio::net::TcpListener::bind("[::1]:0").await else { - return; // no IPv6 loopback on this host - }; - let port = listener.local_addr().unwrap().port(); - let uri: Uri = format!("ws://[::1]:{port}/").parse().unwrap(); - assert!(connect_tcp(&uri).await.is_ok()); - } - #[test] fn test_parse_frame() { assert!(matches!( diff --git a/docs/docs/get_started/configuration.md b/docs/docs/get_started/configuration.md index c5d486168..98fada1ab 100644 --- a/docs/docs/get_started/configuration.md +++ b/docs/docs/get_started/configuration.md @@ -115,7 +115,6 @@ From v0.12.0-rc1, compared with v0.11.0: - The relay's HTTP request runs alongside every stream. In v0.11.0 it runs only when the handshake fails with time left. - The stream serves its latest valid bid. In v0.11.0 only the last bid is validated. - A `204` answer to the handshake is no bid. In v0.11.0 it records `556` and retries over HTTP. -- `X-Timeout-Ms` on the handshake leaves out half the TCP connect time. In v0.11.0 it is the whole window. - A failed stream logs `stream failed`. In v0.11.0 it logs `stream failed, falling back to http get_header`, and only when it retries. - A relay also listed with `get_header = "http"`, as the v0.11.0 docs advised, gets two HTTP requests per `get_header`, and after the Gloas fork only the first of the two serves ePBS; remove the `"http"` entry. ::: diff --git a/tests/src/mock_ws_relay.rs b/tests/src/mock_ws_relay.rs index 8bb28f9a8..997931baa 100644 --- a/tests/src/mock_ws_relay.rs +++ b/tests/src/mock_ws_relay.rs @@ -21,11 +21,7 @@ use cb_common::{ }; use futures::SinkExt; use ssz::Encode; -use tokio::{ - net::{TcpListener, TcpSocket, TcpStream}, - sync::oneshot, - time::timeout, -}; +use tokio::net::{TcpListener, TcpStream}; use tokio_tungstenite::{ accept_hdr_async, tungstenite::{ @@ -95,10 +91,6 @@ pub struct MockWsRelayState { /// Drop the connection after the last bid, with no close frame, as a /// stream that breaks abort_after_bids: bool, - /// End the stream at Date-Milliseconds + X-Timeout-Ms, as a relay honouring - /// the timing headers does: the last bid goes out then, the earlier ones - /// `update_interval` apart before it, and each reaches PBS this much later - ends_at_timeout: Option, handshake_attempts: AtomicU64, received_connections: AtomicU64, last_request: Mutex>, @@ -118,7 +110,6 @@ impl MockWsRelayState { invalid_last_bids: 0, handshake_delay: Duration::ZERO, abort_after_bids: false, - ends_at_timeout: None, handshake_attempts: AtomicU64::new(0), received_connections: AtomicU64::new(0), last_request: Mutex::new(None), @@ -161,10 +152,6 @@ impl MockWsRelayState { Self { abort_after_bids: true, ..self } } - pub fn ends_at_timeout(self, downlink_latency: Duration) -> Self { - Self { ends_at_timeout: Some(downlink_latency), ..self } - } - /// Connections that reached the stream, counted before the handshake, so /// one that never completes still shows pub fn handshake_attempts(&self) -> u64 { @@ -219,27 +206,9 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: state.received_connections.fetch_add(1, Ordering::Relaxed); *state.last_request.lock().unwrap() = Some(request.clone()); - let stream_end_ms = match state.ends_at_timeout { - Some(_) => Some( - request - .start_time_ms - .zip(request.timeout_ms) - .map(|(start, timeout)| start + timeout) - .ok_or_else(|| eyre::eyre!("no timing headers"))?, - ), - None => None, - }; - let n_bids = if request.parent_root.is_some() { state.epbs_bids.len() } else { state.bid_values.len() }; for i in 0..n_bids { - if let (Some(end_ms), Some(latency)) = (stream_end_ms, state.ends_at_timeout) { - let sent_ms = - end_ms - state.update_interval.as_millis() as u64 * (n_bids - 1 - i) as u64; - let arrives_ms = sent_ms + latency.as_millis() as u64; - tokio::time::sleep(Duration::from_millis(arrives_ms.saturating_sub(utcnow_ms()))).await; - } - if state.unknown_frames { // Unknown message type, unknown fork, truncated prefix for frame in [vec![0x7f, FORK_FULU, 1], vec![MSG_BID, 0xff, 1], vec![MSG_BID]] { @@ -251,7 +220,7 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: let frame = bid_frame(&state, &request, i, is_invalid); ws.send(Message::Binary(frame.into())).await?; - if stream_end_ms.is_none() && !state.update_interval.is_zero() { + if !state.update_interval.is_zero() { tokio::time::sleep(state.update_interval).await; } } @@ -348,37 +317,6 @@ fn header(req: &Request, name: &str) -> Option { req.headers().get(name)?.to_str().ok().map(ToOwned::to_owned) } -/// Fills the accept queue until `drain` fires, so a connect waits a second for -/// its SYN retransmit; a delayed accept would not slow it. Returns the port. -pub async fn start_mock_ws_relay_with_full_backlog( - state: Arc, - drain: oneshot::Receiver<()>, -) -> eyre::Result { - let socket = TcpSocket::new_v4()?; - socket.bind(([127, 0, 0, 1], 0).into())?; - let listener = socket.listen(1)?; - let addr = listener.local_addr()?; - - // How many connections the queue admits is up to the OS: fill it until one - // does not complete - let mut fillers = Vec::new(); - while let Ok(filler) = timeout(Duration::from_millis(200), TcpStream::connect(addr)).await { - fillers.push(filler?); - eyre::ensure!(fillers.len() < 16, "accept queue never filled"); - } - - tokio::spawn(async move { - let _ = drain.await; - for _ in &fillers { - listener.accept().await?; - } - drop(fillers); - start_mock_ws_relay_service(state, listener).await - }); - - Ok(addr.port()) -} - /// One relay serving its HTTP API and its bid stream on the same port, as a /// real relay does. Returns the two mock states and a stream-mode client for /// the relay. diff --git a/tests/tests/pbs_get_header_ws.rs b/tests/tests/pbs_get_header_ws.rs index 546966011..eead45035 100644 --- a/tests/tests/pbs_get_header_ws.rs +++ b/tests/tests/pbs_get_header_ws.rs @@ -17,10 +17,7 @@ use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use cb_tests::{ mock_relay::{MockRelayState, start_mock_relay_service_with_listener}, mock_validator::MockValidator, - mock_ws_relay::{ - MockWsRelayState, start_mock_dual_relay, start_mock_ws_relay_service, - start_mock_ws_relay_with_full_backlog, - }, + mock_ws_relay::{MockWsRelayState, start_mock_dual_relay, start_mock_ws_relay_service}, utils::{ RELAY_API_KEY, generate_mock_relay, generate_mock_stream_relay, generate_mock_stream_relay_with_timing_games, get_free_listener, get_pbs_config, @@ -30,7 +27,6 @@ use cb_tests::{ use eyre::Result; use lh_types::ForkName; use reqwest::StatusCode; -use tokio::sync::oneshot; use tree_hash::TreeHash; fn request_slot() -> u64 { @@ -288,52 +284,6 @@ async fn test_get_header_ws_returns_at_deadline() -> Result<()> { Ok(()) } -/// PBS asks the relay to end its stream half a TCP connect before PBS's own -/// deadline, so the last bid, one downlink trip later, still lands. -#[tokio::test] -async fn test_get_header_ws_last_bid_lands_before_deadline() -> Result<()> { - setup_test_env(); - let signer = random_secret(); - let chain = Chain::Hoodi; - let timeout_ms = 3_000; - - let relay_state = Arc::new( - MockWsRelayState::new(chain, signer.clone()) - .with_bid_values(vec![U256::from(10), U256::from(20)]) - .with_update_interval(Duration::from_millis(400)) - .ends_at_timeout(Duration::from_millis(200)), - ); - let (drain, drained) = oneshot::channel(); - let port = start_mock_ws_relay_with_full_backlog(relay_state.clone(), drained).await?; - let relay = generate_mock_stream_relay(port, signer.public_key())?; - let validator = start_pbs(chain, vec![relay], timeout_ms).await?; - - let requested_at_ms = utcnow_ms(); - let (res, ()) = tokio::join!(get_header_json(&validator), async { - // After PBS's first SYN, before its retransmit - tokio::time::sleep(Duration::from_millis(500)).await; - let _ = drain.send(()); - }); - let (code, res) = res?; - - assert_eq!(code, StatusCode::OK); - assert_bid(&res.unwrap(), chain, &signer, U256::from(20)); - - let request = relay_state.last_request().expect("relay saw no request"); - let connect_ms = request.accepted_at_ms - requested_at_ms; - assert!(connect_ms >= 500, "the connect was not delayed: {connect_ms}ms"); - - let relay_end_ms = request.start_time_ms.unwrap() + request.timeout_ms.unwrap(); - let early_ms = (requested_at_ms + timeout_ms) as i64 - relay_end_ms as i64; - let half_connect_ms = connect_ms as i64 / 2; - // PBS's delay before connecting skews both sides, hence the slack - assert!( - (half_connect_ms - 250..=half_connect_ms + 20).contains(&early_ms), - "relay asked to end {early_ms}ms before the deadline, connect took {connect_ms}ms" - ); - Ok(()) -} - /// A stream that never delivers a bid is a 204, same as an HTTP relay with no /// header for the slot. #[tokio::test] From 1021503471e7a56f11ffc8242d0da11c56aefdf4 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 13 Oct 2026 13:14:38 -0700 Subject: [PATCH 21/24] refactor(pbs): request ePBS bids from a stream relay over the stream only A relay with get_header = "stream" gets only the bid stream for an ePBS bid request, with no HTTP request alongside. As get_header does in v0.11.0, a stream that fails to connect, because the connection fails or the relay refuses the handshake, is retried over HTTP with the time left and counted in relay_stream_fallback_total. A handshake still unanswered at the deadline, or a stream that breaks before a bid, is no bid. This drops the HTTP request raced alongside each stream, the choice of the higher of the two bids, and the `selected bid` log. --- .../pbs/src/routes/execution_payload_bid.rs | 283 +++++++++++------- .../src/routes/execution_payload_bid_ws.rs | 11 +- docs/docs/get_started/epbs.md | 16 +- .../get_started/running/metrics-catalog.md | 2 +- docs/docs/get_started/running/metrics.md | 2 +- .../tests/pbs_get_execution_payload_bid_ws.rs | 59 +--- 6 files changed, 197 insertions(+), 176 deletions(-) diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index 9cff7747c..2a7965ea7 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -7,7 +7,6 @@ use axum::{ response::{IntoResponse, Response}, }; use cb_common::{ - config::GetHeaderTransport, pbs::{ ForkName, GetExecutionPayloadBidParams, GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, @@ -25,16 +24,19 @@ use reqwest::{ header::{ACCEPT, CONTENT_TYPE}, }; use ssz::{Decode, Encode}; +use tokio::time::Instant; use tracing::{debug, error, info, warn}; +use url::Url; use super::execution_payload_bid_ws::get_execution_payload_bid_ws; use crate::{ PbsStateGuard, constants::{ - GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, MAX_SIZE_DEFAULT, MAX_SIZE_GET_HEADER_RESPONSE, + GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, + MAX_SIZE_DEFAULT, MAX_SIZE_GET_HEADER_RESPONSE, }, error::PbsClientError, - metrics::{RELAY_HEADER_VALUE, RELAY_LAST_SLOT}, + metrics::{RELAY_HEADER_VALUE, RELAY_LAST_SLOT, RELAY_STREAM_FALLBACK}, state::{BuilderApiState, PbsState}, utils::{ builder_rejection, epbs_base_send_headers, format_gwei_as_eth, log_mux_selection, @@ -176,73 +178,71 @@ pub async fn get_execution_payload_bid( ¶ms.parent_root, ¶ms.proposer_pubkey, ); - let streams = stream_url.is_some(); - let (http, stream) = match stream_url { - None => ( - send_get_execution_payload_bid(params, body, relay, send_headers, max_timeout_ms).await, - None, - ), - Some(url) => tokio::join!( - send_get_execution_payload_bid( - params, - body.clone(), - relay.clone(), - send_headers.clone(), - max_timeout_ms, - ), - get_execution_payload_bid_ws(&body, &relay, &send_headers, url, max_timeout_ms) - ), + let res = match stream_url { + Some(url) => { + stream_execution_payload_bid(params, body, relay, send_headers, url, max_timeout_ms) + .await + } + None => { + send_get_execution_payload_bid(params, body, relay, send_headers, max_timeout_ms).await + } }; - // A relay that errors or times out contributes no bid: 204, never a 502. - // The builder's own 400 and 401 reach the proposer when no bid does. - let http = match http { - Ok(bid) => bid, + // The builder's own 400 and 401 still reach the proposer. + match res { + Ok(Some(relay_bid)) => { + RELAY_LAST_SLOT.with_label_values(&[relay_id.as_str()]).set(slot as i64); + // The bid's value is already gwei, the gauge's unit, so it is set unscaled + RELAY_HEADER_VALUE + .with_label_values(&[relay_id.as_str()]) + .set(i64::try_from(relay_bid.bid.data.message.value).unwrap_or_default()); + Ok(Some(relay_bid)) + } + Ok(None) => Ok(None), Err(err) if err.is_timeout() => { error!(err = "Timed Out", %relay_id, timeout_ms = max_timeout_ms); - None + Ok(None) } Err(err) => { error!(?err, %relay_id); - match builder_rejection(&err) { - Some(rejection) if stream.is_none() => return Err(rejection), - _ => None, - } + builder_rejection(&err).map_or(Ok(None), Err) } - }; - - let selected = select_bid(http, stream); - if streams && let Some((_, transport)) = &selected { - info!(transport = transport.as_str(), "selected bid"); - } - let relay_bid = selected.map(|(relay_bid, _)| relay_bid); - if let Some(relay_bid) = &relay_bid { - RELAY_LAST_SLOT.with_label_values(&[relay_id.as_str()]).set(slot as i64); - // The bid's value is already gwei, the gauge's unit, so it is set unscaled - RELAY_HEADER_VALUE - .with_label_values(&[relay_id.as_str()]) - .set(i64::try_from(relay_bid.bid.data.message.value).unwrap_or_default()); } - Ok(relay_bid) } -/// The higher `value + execution_payment` and the leg it came from, the -/// stream's bid on a tie. Unclamped: the beacon node applies its own cap. -fn select_bid( - http: Option, - stream: Option, -) -> Option<(RelayBid, GetHeaderTransport)> { - let total = |relay_bid: &RelayBid| { - let bid = &relay_bid.bid.data.message; - bid.value.saturating_add(bid.execution_payment) - }; - match (http, stream) { - (Some(http), Some(stream)) if total(&http) > total(&stream) => { - Some((http, GetHeaderTransport::Http)) - } - (http, None) => http.map(|bid| (bid, GetHeaderTransport::Http)), - (_, stream) => stream.map(|bid| (bid, GetHeaderTransport::Stream)), +/// The bid over the relay's stream. As on get_header, only a stream that fails +/// to connect is retried over HTTP, with the time it left; any other failure is +/// no bid. +async fn stream_execution_payload_bid( + params: GetExecutionPayloadBidParams, + body: Bytes, + relay: RelayClient, + send_headers: HeaderMap, + url: Url, + timeout_ms: u64, +) -> Result, PbsError> { + let started = Instant::now(); + let err = + match get_execution_payload_bid_ws(&body, &relay, &send_headers, url, timeout_ms).await { + Err(err @ PbsError::WebSocketConnect(_)) => err, + Err(err) => { + warn!(relay_id = relay.id.as_ref(), %err, "stream failed"); + return Ok(None); + } + res => return res, + }; + + let timeout_left_ms = timeout_ms.saturating_sub(started.elapsed().as_millis() as u64); + if timeout_left_ms == 0 { + warn!(relay_id = relay.id.as_ref(), %err, "stream failed"); + return Ok(None); } + + RELAY_STREAM_FALLBACK + .with_label_values(&[GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, relay.id.as_str()]) + .inc(); + warn!(relay_id = relay.id.as_ref(), %err, timeout_left_ms, "stream failed, falling back to http"); + send_get_execution_payload_bid(params, body, relay, send_headers, timeout_left_ms).await } /// Milliseconds left until `Date-Milliseconds + X-Timeout-Ms`, but no more than @@ -353,29 +353,33 @@ pub(super) fn decode_ssz_bid(body: Bytes, fork: ForkName) -> Result SocketAddr { + async fn start_status_relay(status: StatusCode) -> (SocketAddr, Arc) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); - let relay = axum::Router::new().fallback(move || async move { status }); + let posts = Arc::new(AtomicU64::new(0)); + let counted = posts.clone(); + let relay = axum::Router::new().fallback(move |method: Method| async move { + if method == Method::POST { + counted.fetch_add(1, Ordering::Relaxed); + } + status + }); tokio::spawn(async move { axum::serve(listener, relay).await }); - addr + (addr, posts) } /// A relay whose stream sends `messages`, `gap` apart, then closes, or with /// `close` false drops the connection with no close frame. Its HTTP request /// fails. - async fn start_stream_relay(messages: Vec, gap: Duration, close: bool) -> SocketAddr { + async fn start_stream_relay( + messages: Vec, + gap: Duration, + close: bool, + ) -> (SocketAddr, Arc) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); + let posts = Arc::new(AtomicU64::new(0)); + let counted = posts.clone(); tokio::spawn(async move { while let Ok((socket, _)) = listener.accept().await { let messages = messages.clone(); + let counted = counted.clone(); tokio::spawn(async move { + count_post(&socket, &counted).await; let Ok(mut stream) = accept_async(socket).await else { return }; for message in messages { let _ = stream.send(message).await; @@ -501,20 +503,53 @@ mod tests { }); } }); - addr + (addr, posts) } - /// A relay that accepts the connection and never answers the handshake - async fn start_silent_relay() -> SocketAddr { + /// A relay that accepts the connection and never answers + async fn start_silent_relay() -> (SocketAddr, Arc) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); + let posts = Arc::new(AtomicU64::new(0)); + let counted = posts.clone(); tokio::spawn(async move { - let mut held = Vec::new(); while let Ok((socket, _)) = listener.accept().await { - held.push(socket); + let counted = counted.clone(); + tokio::spawn(async move { + count_post(&socket, &counted).await; + std::future::pending::<()>().await; + }); } }); - addr + (addr, posts) + } + + /// A relay that refuses the stream handshake after `delay` and answers the + /// HTTP request with a `(40, 2)` bid. Returns the HTTP request's + /// `X-Timeout-Ms`. + async fn start_refusing_relay(delay: Duration) -> (SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let timeout_ms = Arc::new(AtomicU64::new(0)); + let recorded = timeout_ms.clone(); + let relay = axum::Router::new().fallback(move |method: Method, headers: HeaderMap| { + let recorded = recorded.clone(); + async move { + if method != Method::POST { + tokio::time::sleep(delay).await; + return StatusCode::NOT_FOUND.into_response(); + } + let sent = headers.get(HEADER_TIMEOUT_MS).and_then(|v| v.to_str().ok()); + recorded.store(sent.and_then(|v| v.parse().ok()).unwrap(), Ordering::Relaxed); + let mut res = relay_bid(40, 2).bid.data.as_ssz_bytes().into_response(); + let headers = res.headers_mut(); + headers.insert(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()); + headers.insert(CONSENSUS_VERSION_HEADER, GLOAS_CONSENSUS_VERSION.clone()); + res + } + }); + tokio::spawn(async move { axum::serve(listener, relay).await }); + (addr, timeout_ms) } fn bid_frame(value: u64, execution_payment: u64) -> Message { @@ -588,42 +623,60 @@ mod tests { RELAY_STATUS_CODE.with_label_values(&[status.as_str(), STREAM, relay_id]).get() } - fn failed_streams(relay_id: &str) -> u64 { + fn fallbacks(relay_id: &str) -> u64 { RELAY_STREAM_FALLBACK.with_label_values(&[STREAM, relay_id]).get() } - // A handshake answered 204 is no bid. A refused handshake, one that runs - // out the window, a stream that breaks before a bid, and a frame over the - // size cap are failed streams. + // A handshake answered 204 is no bid. Only a refused handshake falls back + // to HTTP: one that runs out the window, a stream that breaks before a bid, + // and a frame over the size cap are no bid, with no HTTP request. #[tokio::test] async fn test_epbs_stream_failures_by_status() { let oversized = Message::Binary(vec![0; MAX_SIZE_GET_HEADER_RESPONSE + 1].into()); - for (relay, status, failed) in [ + for ((relay, posts), status, fell_back) in [ (start_status_relay(StatusCode::NO_CONTENT).await, StatusCode::NO_CONTENT, 0), (start_status_relay(StatusCode::NOT_FOUND).await, StatusCode::NOT_FOUND, 1), - (start_silent_relay().await, TIMEOUT_ERROR_STATUS, 1), - (start_stream_relay(vec![], ZERO, false).await, TRANSPORT_ERROR_STATUS, 1), - (start_stream_relay(vec![oversized], ZERO, true).await, TRANSPORT_ERROR_STATUS, 1), + (start_silent_relay().await, TIMEOUT_ERROR_STATUS, 0), + (start_stream_relay(vec![], ZERO, false).await, TRANSPORT_ERROR_STATUS, 0), + (start_stream_relay(vec![oversized], ZERO, true).await, TRANSPORT_ERROR_STATUS, 0), ] { let (relay_id, bid) = bid_from_stream_relay(relay, 300).await; assert!(bid.is_none(), "{status}"); assert_eq!(status_count(status, &relay_id), 1, "{status}"); - assert_eq!(failed_streams(&relay_id), failed, "{status}"); + assert_eq!(fallbacks(&relay_id), fell_back, "{status}"); + assert_eq!(posts.load(Ordering::Relaxed), fell_back, "{status}"); } } + // The HTTP request a refused handshake falls back to has the time the + // handshake left, and its bid is served + #[tokio::test] + async fn test_epbs_stream_refused_handshake_falls_back_with_the_time_left() { + let (relay, timeout_ms) = start_refusing_relay(Duration::from_millis(200)).await; + let (relay_id, bid) = bid_from_stream_relay(relay, 1_000).await; + let bid = bid.expect("the HTTP bid").bid; + assert_eq!((bid.data.message.value, bid.data.message.execution_payment), (40, 2)); + assert_eq!(status_count(StatusCode::NOT_FOUND, &relay_id), 1); + assert_eq!(fallbacks(&relay_id), 1); + + // 1000 ms less the default 50 ms buffer and the 200 ms handshake + let timeout_ms = timeout_ms.load(Ordering::Relaxed); + assert!((1..=750).contains(&timeout_ms), "{timeout_ms}"); + } + // A window whose every bid fails to decode is no bid, not a failed stream, // and counts each of those bids as an invalid frame #[tokio::test] async fn test_epbs_stream_undecodable_bids_are_no_bid() { for n_frames in [1, 3] { let undecodable = Message::Binary(vec![0x01, 7, 1, 2, 3].into()); - let relay = start_stream_relay(vec![undecodable; n_frames], ZERO, true).await; + let (relay, posts) = start_stream_relay(vec![undecodable; n_frames], ZERO, true).await; let (relay_id, bid) = bid_from_stream_relay(relay, 1_000).await; assert!(bid.is_none()); assert_eq!(status_count(StatusCode::NO_CONTENT, &relay_id), 1); - assert_eq!(failed_streams(&relay_id), 0); + assert_eq!(fallbacks(&relay_id), 0); + assert_eq!(posts.load(Ordering::Relaxed), 0); let stream = [STREAM, relay_id.as_str()]; assert_eq!( RELAY_STREAM_INVALID_FRAMES.with_label_values(&stream).get(), @@ -635,7 +688,8 @@ mod tests { // Messages other than bids do not end the stream. The served bid keeps the // frame's fork and sets the relay's gauges: its value without the execution - // payment, and the slot. The latency series times the first bid. + // payment, and the slot. The latency series times the first bid. The relay + // gets no HTTP request. #[tokio::test] async fn test_epbs_stream_bid_after_other_messages_is_served_and_recorded() { let messages = vec![ @@ -644,12 +698,13 @@ mod tests { Message::Ping(vec![1].into()), bid_frame(77, 5), ]; - let relay = start_stream_relay(messages, Duration::from_millis(150), true).await; + let (relay, posts) = start_stream_relay(messages, Duration::from_millis(150), true).await; let (relay_id, bid) = bid_from_stream_relay(relay, 2_000).await; let bid = bid.expect("the stream's bid").bid; assert_eq!(bid.version, ForkName::Gloas); let bid = &bid.data.message; assert_eq!((bid.value, bid.execution_payment), (77, 5)); + assert_eq!(posts.load(Ordering::Relaxed), 0); assert_eq!(status_count(StatusCode::OK, &relay_id), 1); assert_eq!(RELAY_HEADER_VALUE.with_label_values(&[&relay_id]).get(), 77); @@ -668,9 +723,9 @@ mod tests { async fn test_epbs_stream_series_are_labeled_by_name() { let messages = vec![Message::Binary(vec![0x01, 7, 1].into()), bid_frame(1, 0)]; let (relay_id, _) = - bid_from_stream_relay(start_stream_relay(messages, ZERO, true).await, 1_000).await; + bid_from_stream_relay(start_stream_relay(messages, ZERO, true).await.0, 1_000).await; let (failed_id, _) = - bid_from_stream_relay(start_status_relay(StatusCode::NOT_FOUND).await, 300).await; + bid_from_stream_relay(start_status_relay(StatusCode::NOT_FOUND).await.0, 300).await; let scrape = TextEncoder::new().encode_to_string(&PBS_METRICS_REGISTRY.gather()).unwrap(); for (series, relay_id) in [ diff --git a/crates/pbs/src/routes/execution_payload_bid_ws.rs b/crates/pbs/src/routes/execution_payload_bid_ws.rs index fb5987df9..646bf2721 100644 --- a/crates/pbs/src/routes/execution_payload_bid_ws.rs +++ b/crates/pbs/src/routes/execution_payload_bid_ws.rs @@ -16,29 +16,26 @@ use url::Url; use super::execution_payload_bid::{RelayBid, decode_ssz_bid}; use crate::{ - bid_stream::{Frame, Held, handshake_request, read_bid_stream, record_stream_failure}, + bid_stream::{Frame, Held, handshake_request, read_bid_stream}, constants::{GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, TRANSPORT_ERROR_STATUS}, metrics::RELAY_STATUS_CODE, }; type StreamOutcome = (StatusCode, Result, PbsError>); -/// The relay's latest bid at the deadline. A failed stream is no bid. +/// The relay's latest bid at the deadline pub(super) async fn get_execution_payload_bid_ws( auth_ssz: &[u8], relay: &RelayClient, send_headers: &HeaderMap, url: Url, timeout_ms: u64, -) -> Option { +) -> Result, PbsError> { let (status, res) = stream_execution_payload_bid(auth_ssz, relay, send_headers, url, timeout_ms).await; let endpoint = GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG; RELAY_STATUS_CODE.with_label_values(&[status.as_str(), endpoint, &relay.id]).inc(); - res.unwrap_or_else(|err| { - record_stream_failure(endpoint, &relay.id, &err); - None - }) + res } async fn stream_execution_payload_bid( diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 3cf003392..1f41165bb 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -27,7 +27,7 @@ The ePBS bid endpoint does not use these PBS options, which will be deprecated a - `skip_sigverify`, `min_bid_eth` and `extra_validation_enabled`: the beacon node checks the bid against the on-chain builder registry and applies the `min_bid` from its builder config. - `timeout_get_header_ms`, `late_in_slot_time_ms`, their `[[mux]]` overrides and the timing-games options: the beacon node's deadline bounds the request (see [Timing](#timing)). -For a builder whose relay entry sets `get_header = "stream"`, Commit-Boost also reads its bids from a [stream](#bid-streaming). +For a builder whose relay entry sets `get_header = "stream"`, Commit-Boost reads its bids from a [stream](#bid-streaming). ## How a bid request reaches a builder @@ -96,11 +96,11 @@ Builder preferences use `timeout_register_validator_ms`, and the signed block `t A builder answers an HTTP bid request once, so a better bid it makes after answering never reaches the beacon node. Over a websocket stream it can keep sending bids until just before the beacon node's deadline. -Ask the builder whether it serves the ePBS bid stream, at `ws(s):///eth/v1/builder/execution_payload_bid_stream/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}`. If it does, set `get_header = "stream"` on its relay entry. Before the fork the same setting reads the builder's `get_header` bids from its [PBS stream](./configuration.md#bid-streaming), so for a builder that serves only one of the two, Commit-Boost logs a failed stream on each request to the other and uses the HTTP bid. The handshake carries the relay entry's `headers`, so a builder that requires an [API key](./configuration.md#relay-api-keys) gets it. +Ask the builder whether it serves the ePBS bid stream, at `ws(s):///eth/v1/builder/execution_payload_bid_stream/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}`. If it does, set `get_header = "stream"` on its relay entry. Before the fork the same setting reads the builder's `get_header` bids from its [PBS stream](./configuration.md#bid-streaming), so for a builder that serves only one of the two, Commit-Boost logs a failed stream on each request to the other and asks over HTTP instead. The handshake carries the relay entry's `headers`, so a builder that requires an [API key](./configuration.md#relay-api-keys) gets it. -For each bid request to a streaming builder, Commit-Boost sends the usual HTTP request and opens the stream at the same time. It returns the HTTP bid or the latest streamed bid that decodes, whichever has the higher `value + execution_payment`, and the streamed bid on a tie. Commit-Boost does not apply the key's `max_execution_payment` to this choice; the beacon node applies it afterwards. If the stream fails, the HTTP bid still reaches the beacon node. A builder [outside your config](#builders-outside-your-config) is always asked over HTTP. +For each bid request to a streaming builder, Commit-Boost opens the stream and returns the latest streamed bid that decodes, without an HTTP request. Only a stream that fails to connect, because the connection fails or the builder refuses the handshake, is retried over HTTP, with the time left. A handshake still unanswered at the deadline, or a stream that breaks before a bid, is no bid. A builder [outside your config](#builders-outside-your-config) is always asked over HTTP. -Streaming makes bid requests slower. Commit-Boost replies to the beacon node once the builder's HTTP response is in and the stream has ended: when the builder closes it, when it fails, or at the beacon node's deadline less `proposer_deadline_buffer_ms`. An HTTP builder is answered as soon as it responds. `timeout_get_header_ms` does not shorten the wait. If the beacon node reports bid requests timing out on streaming builders, raise `proposer_deadline_buffer_ms`. +Streaming makes bid requests slower. Commit-Boost replies to the beacon node once the stream has ended: when the builder closes it, when it fails, or at the beacon node's deadline less `proposer_deadline_buffer_ms`. An HTTP builder is answered as soon as it responds. `timeout_get_header_ms` does not shorten the wait. If the beacon node reports bid requests timing out on streaming builders, raise `proposer_deadline_buffer_ms`. The stream [connects directly](./configuration.md#connection), without your proxy settings, while the HTTP request uses them. If your host reaches builders only through a proxy, keep them on `"http"`. @@ -113,8 +113,8 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences, `202` for the signed block whatever the builders answer, `4xx` for a rejected request, `500` when no builder accepted the preferences or on an internal error | | What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"`; a dial Commit-Boost [refuses](#builders-outside-your-config), or whose lookup fails, times out or finds no free slot, sends no request and is not counted | | How fast are builders? | `cb_pbs_relay_latency`, by `relay_id` | -| Is a builder's bid stream working? | `cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid_stream"}`: `200` when the stream supplied a bid, `204` when it had none, any other code a failed stream: `555` the deadline ran out before the handshake completed, `556` a connection that failed or dropped before a bid, or the builder's own refusal such as `404`. When Commit-Boost returns a streaming builder's bid, it also logs `selected bid`, with `transport` set to `stream` or `http` for where that bid came from | -| What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from the bid Commit-Boost returns for each builder; with a streaming builder, whichever of its two bids it chose | +| Is a builder's bid stream working? | `cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid_stream"}`: `200` when the stream supplied a bid, `204` when it had none, any other code a failed stream: `555` the deadline ran out before the handshake completed, `556` a connection that failed or dropped before a bid, or the builder's own refusal such as `404`. | +| What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from each bid a builder serves | ## Troubleshooting @@ -123,7 +123,7 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | | Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config) | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | -| Bid requests to a streaming builder log "stream failed" with "rejected with 404", and `cb_pbs_relay_stream_fallback_total{endpoint="get_execution_payload_bid_stream"}` rises | The builder does not serve the ePBS bid stream. Its HTTP bid still reaches the beacon node | Set `get_header = "http"` on its relay entry | +| Bid requests to a streaming builder log "stream failed" with "rejected with 404", and `cb_pbs_relay_stream_fallback_total{endpoint="get_execution_payload_bid_stream"}` rises | The builder does not serve the ePBS bid stream. Commit-Boost asks it over HTTP instead | Set `get_header = "http"` on its relay entry | | Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | -| Bid requests to a streaming builder log "stream failed" with "websocket timed out" or a connection error | Commit-Boost could not open the stream before the deadline, for example because your network allows only proxied connections, which the stream does not use. The HTTP bid still reaches the beacon node | Keep the builder on `get_header = "http"`, or allow direct connections to it | +| Bid requests to a streaming builder log "stream failed" with "websocket timed out" or a connection error | Commit-Boost could not open the stream, for example because your network allows only proxied connections, which the stream does not use. A connection error with time left is retried over HTTP; a handshake that times out is no bid | Keep the builder on `get_header = "http"`, or allow direct connections to it | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/docs/get_started/running/metrics-catalog.md b/docs/docs/get_started/running/metrics-catalog.md index 325c1850c..a771ca6a1 100644 --- a/docs/docs/get_started/running/metrics-catalog.md +++ b/docs/docs/get_started/running/metrics-catalog.md @@ -21,7 +21,7 @@ PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. Th | `cb_pbs_relay_stream_connect_latency` | Histogram | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Websocket handshake latency in seconds, for a relay with `get_header = "stream"`. Observed once per successful handshake; a handshake that fails, runs out the bid window, or is answered `204` records nothing here. Custom buckets from 5 ms to 1 s. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | | `cb_pbs_relay_stream_updates` | Histogram | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Bid updates accepted on arrival per stream: on `get_header_stream`, every frame whose message type and fork byte can be read; on `get_execution_payload_bid_stream`, every bid that decodes. Observed once per stream that connected, including streams that received nothing; a handshake that fails, times out or is answered `204` records nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | | `cb_pbs_relay_stream_invalid_frames_total` | Counter | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Websocket frames whose message type or fork byte cannot be read; on the ePBS stream, also bids that fail to decode. Incremented only for windows that saw at least one, so the series is absent while zero. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | -| `cb_pbs_relay_stream_fallback_total` | Counter | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | (unreleased, from v0.12.0-rc1; in v0.11.0, handshake failures retried over HTTP) Bid streams that failed while the relay's HTTP request ran alongside: a connection or handshake that failed or timed out, a stream that broke before a bid, or, on `get_header_stream`, a window in which every held bid was invalid. A `204` answer to the handshake is no bid and does not count. Counts every failed stream, whether or not the HTTP request returned a bid; the HTTP result is recorded under `endpoint="get_header"` (or `get_execution_payload_bid`). Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | +| `cb_pbs_relay_stream_fallback_total` | Counter | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | (unreleased, from v0.12.0-rc1; in v0.11.0, handshake failures retried over HTTP) Bid streams that failed while the relay's HTTP request ran alongside: a connection or handshake that failed or timed out, a stream that broke before a bid, or, on `get_header_stream`, a window in which every held bid was invalid. A `204` answer to the handshake is no bid and does not count. Counts every failed stream, whether or not the HTTP request returned a bid; the HTTP result is recorded under `endpoint="get_header"`. On `get_execution_payload_bid_stream`, which runs no HTTP request alongside, only a stream that failed to connect and was retried over HTTP counts, and the retry is recorded under `endpoint="get_execution_payload_bid"`. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | | `cb_pbs_beacon_node_status_code_total` | Counter | `http_status_code`, `endpoint` | HTTP status code returned to the beacon node. Tracks what status codes the PBS returns for beacon node-facing requests. Endpoint values: `get_header`, `register_validator`, `submit_blinded_block`, `status`, `reload`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Error status codes (`502` for `NoResponse`/`NoPayload`, `500` for `Internal`) are set via `PbsClientError`. The handlers also record `406` directly when the request's `Accept` header offers no supported encoding, `204` when no bid is available on `get_header`, and `202` for accepted v2 `submit_blinded_block` requests. (unreleased, from v0.12.0-rc1) The ePBS endpoints record `200` for a bid, `204` for none, `202` for accepted preferences and for every signed block it forwards, `400` for a request Commit-Boost refuses, a builder's own `400` or `401`, `406` for a bid request whose `Accept` offers no supported encoding, `415` for an unsupported `Content-Type`, and `500` when no builder accepts the preferences. | | `cb_pbs_pbs_submit_block_v2_unsupported_total` | Counter | `relay_id` | Count of v2 `submit_blinded_block` requests a relay could not serve because it returned 404 on the v2 endpoint. A non-zero value means the relay does not support `submitBlindedBlockV2` and those blocks were not submitted via that relay. The double `pbs` in the wire name comes from the registry prefix plus the metric name `pbs_submit_block_v2_unsupported_total`. | diff --git a/docs/docs/get_started/running/metrics.md b/docs/docs/get_started/running/metrics.md index 1d5efdb6e..fc96a3516 100644 --- a/docs/docs/get_started/running/metrics.md +++ b/docs/docs/get_started/running/metrics.md @@ -133,4 +133,4 @@ For a relay with `get_header = "stream"` (see [Configuration > Bid streaming](.. When the newest bid fails validation and an earlier one is served, the failure is logged, not counted. A window in which every kept bid fails counts as `200` on the status series, the same as an invalid bid over HTTP, and as one failed stream. If the beacon node drops the request before the deadline, no status is recorded. -The [ePBS bid stream](../epbs.md#bid-streaming) (unreleased, from v0.12.0-rc1) reports the same series under `endpoint="get_execution_payload_bid_stream"`, and its HTTP request under `endpoint="get_execution_payload_bid"`. It decodes each bid as it arrives and does not validate it, so `cb_pbs_relay_stream_updates` counts only bids that decode, and `cb_pbs_relay_stream_invalid_frames_total` also counts those that do not. A stream that ends with only undecodable bids is no bid (`204`), not a failed stream; if its connection drops instead, it records `556` and a failed stream. +The [ePBS bid stream](../epbs.md#bid-streaming) (unreleased, from v0.12.0-rc1) reports the same series under `endpoint="get_execution_payload_bid_stream"`, and its HTTP fallback under `endpoint="get_execution_payload_bid"`. There `cb_pbs_relay_stream_fallback_total` counts only streams that failed to connect and were retried over HTTP. It decodes each bid as it arrives and does not validate it, so `cb_pbs_relay_stream_updates` counts only bids that decode, and `cb_pbs_relay_stream_invalid_frames_total` also counts those that do not. A stream that ends with only undecodable bids is no bid (`204`); if its connection drops instead, it records `556`. diff --git a/tests/tests/pbs_get_execution_payload_bid_ws.rs b/tests/tests/pbs_get_execution_payload_bid_ws.rs index e9a06d0b5..55032c020 100644 --- a/tests/tests/pbs_get_execution_payload_bid_ws.rs +++ b/tests/tests/pbs_get_execution_payload_bid_ws.rs @@ -30,7 +30,7 @@ fn relay_states(http: u64, stream_bids: Vec<(u64, u64)>) -> (MockRelayState, Moc } /// PBS in front of one stream-mode relay serving both states on one port -async fn start_race( +async fn start_relay( (http_state, ws_state): (MockRelayState, MockWsRelayState), ) -> Result<(MockValidator, Arc, Arc)> { let (http_state, ws_state, relay) = start_mock_dual_relay(http_state, ws_state).await?; @@ -66,12 +66,12 @@ fn assert_bid(res: Option, value: u64, execution } /// A relay that streams get_header but not the ePBS bid answers the handshake -/// with 404, and the HTTP request raced alongside it supplies the bid. +/// with 404, and the HTTP request PBS falls back to supplies the bid. #[tokio::test] async fn test_get_execution_payload_bid_ws_no_stream_route_http_bid_serves() -> Result<()> { let (http_state, ws_state) = relay_states(42, vec![(90, 0)]); let (validator, http_state, ws_state) = - start_race((http_state, ws_state.without_epbs_stream())).await?; + start_relay((http_state, ws_state.without_epbs_stream())).await?; let (code, res) = get_bid(&validator).await?; assert_eq!(code, StatusCode::OK); @@ -82,12 +82,12 @@ async fn test_get_execution_payload_bid_ws_no_stream_route_http_bid_serves() -> Ok(()) } -/// With no bid from the stream, the builder's HTTP 401 reaches the beacon node. +/// The builder's 401 to the fallback HTTP request reaches the beacon node. #[tokio::test] async fn test_get_execution_payload_bid_ws_no_stream_bid_http_rejection_passes() -> Result<()> { let (http_state, ws_state) = relay_states(10, vec![(50, 0)]); let (validator, http_state, _) = - start_race((http_state, ws_state.without_epbs_stream())).await?; + start_relay((http_state, ws_state.without_epbs_stream())).await?; http_state.set_response_override(StatusCode::UNAUTHORIZED); let (code, _) = get_bid(&validator).await?; @@ -95,48 +95,18 @@ async fn test_get_execution_payload_bid_ws_no_stream_bid_http_rejection_passes() Ok(()) } -/// The builder's HTTP 401 does not displace the stream's bid. +/// A relay whose stream serves a bid gets no HTTP request, and its latest bid +/// is served, not its highest. #[tokio::test] -async fn test_get_execution_payload_bid_ws_http_rejection_stream_bid_serves() -> Result<()> { - let (validator, http_state, _) = start_race(relay_states(10, vec![(50, 0)])).await?; - http_state.set_response_override(StatusCode::UNAUTHORIZED); - - let (code, res) = get_bid(&validator).await?; - assert_eq!(code, StatusCode::OK); - assert_bid(res, 50, 0); - assert_eq!(http_state.received_execution_payload_bid(), 1); - Ok(()) -} - -/// An HTTP request that runs out the window does not displace the stream's bid. -#[tokio::test] -async fn test_get_execution_payload_bid_ws_http_timeout_stream_bid_serves() -> Result<()> { - let (http_state, ws_state) = relay_states(10, vec![(50, 0)]); - let (validator, ..) = start_race((http_state.with_bid_delay_ms(BUDGET_MS), ws_state)).await?; +async fn test_get_execution_payload_bid_ws_serves_latest_bid_without_http() -> Result<()> { + let (validator, http_state, ws_state) = + start_relay(relay_states(60, vec![(70, 0), (50, 0)])).await?; let (code, res) = get_bid(&validator).await?; assert_eq!(code, StatusCode::OK); assert_bid(res, 50, 0); - Ok(()) -} - -/// The stream and the HTTP request are separate candidates: the higher value + -/// execution_payment wins, the stream's bid on a tie, and the stream's -/// candidate is its latest bid, not its highest. -#[tokio::test] -async fn test_get_execution_payload_bid_ws_higher_total_wins() -> Result<()> { - for (http, stream_bids, served) in - [(60, vec![(70, 0), (50, 0)], (60, 0)), (50, vec![(10, 40)], (10, 40))] - { - let (validator, http_state, ws_state) = - start_race(relay_states(http, stream_bids.clone())).await?; - - let (code, res) = get_bid(&validator).await?; - assert_eq!(code, StatusCode::OK, "http {http}, stream {stream_bids:?}"); - assert_bid(res, served.0, served.1); - assert_eq!(http_state.received_execution_payload_bid(), 1); - assert_eq!(ws_state.received_connections(), 1); - } + assert_eq!(http_state.received_execution_payload_bid(), 0); + assert_eq!(ws_state.received_connections(), 1); Ok(()) } @@ -145,7 +115,7 @@ async fn test_get_execution_payload_bid_ws_higher_total_wins() -> Result<()> { #[tokio::test] async fn test_get_execution_payload_bid_ws_skips_invalid_bid() -> Result<()> { let (http_state, ws_state) = relay_states(10, vec![(50, 0), (90, 0)]); - let (validator, ..) = start_race((http_state, ws_state.with_invalid_last_bids(1))).await?; + let (validator, ..) = start_relay((http_state, ws_state.with_invalid_last_bids(1))).await?; let (code, res) = get_bid(&validator).await?; assert_eq!(code, StatusCode::OK); @@ -159,8 +129,7 @@ async fn test_get_execution_payload_bid_ws_skips_invalid_bid() -> Result<()> { async fn test_get_execution_payload_bid_ws_handshake_and_deadline() -> Result<()> { const BUFFER_MS: u64 = 200; let (http_state, ws_state) = relay_states(10, vec![(20, 0)]); - let (_, ws_state, relay) = - start_mock_dual_relay(http_state.with_bid_delay_ms(400), ws_state.hold_open()).await?; + let (_, ws_state, relay) = start_mock_dual_relay(http_state, ws_state.hold_open()).await?; let validator = setup_pbs(Chain::Hoodi, vec![relay], |pbs_config| { pbs_config.proposer_deadline_buffer_ms = BUFFER_MS }) From 05151bac5ee371e145f915e1d6ad6fb34440ff52 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 13 Oct 2026 13:20:15 -0700 Subject: [PATCH 22/24] refactor(pbs): leave the PBS get_header stream as released in v0.11.0 The PBS get_header path, its stream metrics, tests and docs are back to main's, so PBS is as released in v0.11.0: no HTTP request alongside the stream, no endpoint label on the relay_stream_* series, and the handshake fallback as in v0.11.0. The ePBS bid stream keeps the reader in bid_stream.rs, which now holds only the latest bid, and records under its own relay_epbs_stream_* series, labeled by relay_id. --- config.example.toml | 7 +- crates/common/src/config/pbs.rs | 11 +- crates/common/src/pbs/relay.rs | 115 +++-- crates/pbs/src/bid_stream.rs | 54 +-- crates/pbs/src/metrics.rs | 72 ++- crates/pbs/src/mev_boost/get_header.rs | 330 +++----------- crates/pbs/src/mev_boost/get_header_ws.rs | 419 +++++++++++++++--- .../pbs/src/routes/execution_payload_bid.rs | 50 ++- .../src/routes/execution_payload_bid_ws.rs | 17 +- docs/docs/get_started/configuration.md | 46 +- docs/docs/get_started/epbs.md | 6 +- .../get_started/running/metrics-catalog.md | 18 +- docs/docs/get_started/running/metrics.md | 22 +- tests/src/mock_ws_relay.rs | 41 +- tests/tests/pbs_get_header_ws.rs | 160 ++----- 15 files changed, 703 insertions(+), 665 deletions(-) diff --git a/config.example.toml b/config.example.toml index f4a177bff..430050a1d 100644 --- a/config.example.toml +++ b/config.example.toml @@ -100,13 +100,10 @@ headers = { X-MyCustomHeader = "MyCustomValue" } # GET parameters to add to each request URL for this relay # OPTIONAL get_params = { param1 = "value1", param2 = "value2" } -# How to fetch bids from this relay. +# How to fetch headers from this relay. # "http" -> one request per get_header, at the relay url above # "stream" -> websocket stream of bid updates, only for relays that support it. Connects to -# ws(s):///eth/v1/builder/header_stream/{slot}/{parent_hash}/{pubkey}. -# (unreleased, from v0.12.0-rc1) The relay's normal HTTP request runs alongside, -# timing games included, and the better bid wins, so a failed stream still -# leaves the HTTP bid. +# ws(s):///eth/v1/builder/header_stream/{slot}/{parent_hash}/{pubkey}. # Requires a UUID api key, set it in `headers` field above under the `X-Api-Key` # name; it is sent on the websocket handshake. Can use # https://www.uuidgenerator.net/version4 to generate one. diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index 00df7f6ee..294c5b535 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -49,15 +49,6 @@ pub enum GetHeaderTransport { Stream, } -impl GetHeaderTransport { - pub fn as_str(&self) -> &'static str { - match self { - Self::Http => "http", - Self::Stream => "stream", - } - } -} - /// A custom relay header value: a literal, or a secret read from a file or an /// environment variable when the relay client is built (at startup and on every /// reload), so an API key never has to sit in plaintext in the config file. @@ -140,7 +131,7 @@ pub struct RelayConfig { pub headers: Option>, /// Optional GET parameters to add to each request pub get_params: Option>, - /// How to fetch bids from this relay + /// How to fetch headers from this relay #[serde(default)] pub get_header: GetHeaderTransport, /// Whether to enable timing games diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index 89727dfb1..7b0d8b815 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -63,15 +63,6 @@ pub enum GetHeaderRequest { Stream(Url), } -impl GetHeaderRequest { - pub fn transport(&self) -> GetHeaderTransport { - match self { - Self::Http(_) => GetHeaderTransport::Http, - Self::Stream(_) => GetHeaderTransport::Stream, - } - } -} - fn stream_url(entry: &Url) -> eyre::Result { let scheme = match entry.scheme() { "http" | "ws" => "ws", @@ -205,17 +196,24 @@ impl RelayClient { ) } - /// The get_header stream, `None` when the relay does not stream - pub fn get_header_stream_url( + pub fn get_header_request( &self, slot: u64, parent_hash: &B256, validator_pubkey: &BlsPublicKey, - ) -> Option { - let mut url = self.stream_url.clone()?; - url.set_path(&format!("{}/{slot}/{parent_hash}/{validator_pubkey}", url.path())); - self.append_get_params(&mut url); - Some(url) + ) -> Result { + Ok(match &self.stream_url { + None => { + GetHeaderRequest::Http(self.get_header_url(slot, parent_hash, validator_pubkey)?) + } + Some(base) => { + let mut url = base.clone(); + url.set_path(&format!("{}/{slot}/{parent_hash}/{validator_pubkey}", base.path())); + + self.append_get_params(&mut url); + GetHeaderRequest::Stream(url) + } + }) } pub fn get_status_url(&self) -> Result { @@ -302,7 +300,9 @@ mod tests { use alloy::primitives::B256; - use super::{RelayClient, RelayEntry, decode_auth_data_url, value_fingerprint}; + use super::{ + GetHeaderRequest, RelayClient, RelayEntry, decode_auth_data_url, value_fingerprint, + }; use crate::{ config::{GetHeaderTransport, RelayConfig, test_env::RELAY_URL}, utils::bls_pubkey_from_hex_unchecked, @@ -411,10 +411,9 @@ mod tests { } #[test] - fn test_stream_urls() { + fn test_get_header_request() { let slot = 0; let parent_hash = B256::ZERO; - let parent_root = B256::repeat_byte(1); let validator_pubkey = bls_pubkey_from_hex_unchecked( "0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae", ); @@ -424,18 +423,17 @@ mod tests { }"#; let base_config = serde_json::from_str::(relay_config).unwrap(); - // Default transport: no stream + // Default transport: plain HTTP endpoint let relay = RelayClient::new(base_config.clone()).unwrap(); - assert!(relay.get_header_stream_url(slot, &parent_hash, &validator_pubkey).is_none()); - assert!( - relay - .get_execution_payload_bid_stream_url( - slot, - &parent_hash, - &parent_root, - &validator_pubkey - ) - .is_none() + let GetHeaderRequest::Http(url) = + relay.get_header_request(slot, &parent_hash, &validator_pubkey).unwrap() + else { + panic!("expected http request"); + }; + assert_eq!( + url, + relay.get_header_url(slot, &parent_hash, &validator_pubkey).unwrap(), + "http dispatch must match the plain url builder" ); // Streaming: the relay url over ws, at the fixed stream path, with the @@ -443,7 +441,11 @@ mod tests { let mut config = base_config.clone(); config.get_header = GetHeaderTransport::Stream; let relay = RelayClient::new(config).unwrap(); - let url = relay.get_header_stream_url(slot, &parent_hash, &validator_pubkey).unwrap(); + let GetHeaderRequest::Stream(url) = + relay.get_header_request(slot, &parent_hash, &validator_pubkey).unwrap() + else { + panic!("expected stream request"); + }; assert_eq!( url.to_string(), format!( @@ -458,33 +460,58 @@ mod tests { config.get_header = GetHeaderTransport::Stream; config.get_params = Some(HashMap::from([("token".to_string(), "abc".to_string())])); let relay = RelayClient::new(config).unwrap(); - let url = relay.get_header_stream_url(slot, &parent_hash, &validator_pubkey).unwrap(); + let GetHeaderRequest::Stream(url) = + relay.get_header_request(slot, &parent_hash, &validator_pubkey).unwrap() + else { + panic!("expected stream request"); + }; assert_eq!( url.to_string(), format!( "wss://abc.xyz:4444/eth/v1/builder/header_stream/{slot}/{parent_hash}/{validator_pubkey}?token=abc" ) ); - let url = relay - .get_execution_payload_bid_stream_url( - slot, + + // A relay url we can't stream over is rejected at construction + let mut config = base_config; + config.entry.url = "unix:/tmp/relay.sock".parse().unwrap(); + config.get_header = GetHeaderTransport::Stream; + assert!(RelayClient::new(config).is_err()); + } + + #[test] + fn test_execution_payload_bid_stream_url() { + let parent_hash = B256::ZERO; + let parent_root = B256::repeat_byte(1); + let validator_pubkey = bls_pubkey_from_hex_unchecked( + "0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae", + ); + let relay_config = r#" + { + "url": "https://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@abc.xyz:4444/relay-api" + }"#; + let mut config = serde_json::from_str::(relay_config).unwrap(); + let url = |config: &RelayConfig| { + RelayClient::new(config.clone()).unwrap().get_execution_payload_bid_stream_url( + 0, &parent_hash, &parent_root, &validator_pubkey, ) - .unwrap(); + }; + + // Only a streaming relay has one + assert!(url(&config).is_none()); + + // The get_header stream's origin, at the fixed path, with get_params + config.get_header = GetHeaderTransport::Stream; + config.get_params = Some(HashMap::from([("token".to_string(), "abc".to_string())])); assert_eq!( - url.to_string(), + url(&config).unwrap().to_string(), format!( - "wss://abc.xyz:4444/eth/v1/builder/execution_payload_bid_stream/{slot}/{parent_hash}/{parent_root}/{validator_pubkey}?token=abc" + "wss://abc.xyz:4444/eth/v1/builder/execution_payload_bid_stream/0/{parent_hash}/{parent_root}/{validator_pubkey}?token=abc" ) ); - - // A relay url we can't stream over is rejected at construction - let mut config = base_config; - config.entry.url = "unix:/tmp/relay.sock".parse().unwrap(); - config.get_header = GetHeaderTransport::Stream; - assert!(RelayClient::new(config).is_err()); } #[test] diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs index 322822c32..571eefb9a 100644 --- a/crates/pbs/src/bid_stream.rs +++ b/crates/pbs/src/bid_stream.rs @@ -1,4 +1,4 @@ -//! The websocket bid stream of a relay configured with `get_header = "stream"`. +//! The ePBS bid stream of a relay configured with `get_header = "stream"`. //! One connection per request, dropped at the deadline. The handshake carries //! the request, and the relay replies with one binary frame per bid update: //! @@ -9,7 +9,6 @@ //! ``` use std::{ - collections::VecDeque, sync::{Arc, OnceLock}, time::Duration, }; @@ -36,10 +35,13 @@ use tracing::{debug, warn}; use url::Url; use crate::{ - constants::{MAX_SIZE_GET_HEADER_RESPONSE, TIMEOUT_ERROR_STATUS, TRANSPORT_ERROR_STATUS}, + constants::{ + GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, + TIMEOUT_ERROR_STATUS, TRANSPORT_ERROR_STATUS, + }, metrics::{ - RELAY_LATENCY, RELAY_STREAM_CONNECT_LATENCY, RELAY_STREAM_FALLBACK, - RELAY_STREAM_INVALID_FRAMES, RELAY_STREAM_UPDATES, + RELAY_EPBS_STREAM_CONNECT_LATENCY, RELAY_EPBS_STREAM_INVALID_FRAMES, + RELAY_EPBS_STREAM_UPDATES, RELAY_LATENCY, }, }; @@ -48,10 +50,6 @@ const FRAME_PREFIX_LEN: usize = 2; const MSG_BID: u8 = 0x01; -/// A PBS stream holds only its newest frames, which bounds its memory and the -/// validation it leaves until the deadline -pub(crate) const MAX_HELD_FRAMES: usize = 8; - /// One bid update, its bid still SSZ-encoded pub(crate) struct Frame { pub(crate) fork: ForkName, @@ -60,9 +58,8 @@ pub(crate) struct Frame { /// What a stream held when its window closed pub(crate) struct Held { - /// The newest frames the parse closure accepted, up to the caller's cap, - /// oldest first - pub(crate) frames: VecDeque, + /// The newest frame the parse closure accepted + pub(crate) latest: Option, /// Frames the parse closure accepted pub(crate) updates: usize, pub(crate) connect_latency: Duration, @@ -94,23 +91,22 @@ pub(crate) fn handshake_request( Ok(request) } -/// Holds the newest `max_held` frames `parse` accepts until `deadline` or the -/// end of the stream. An error carries the status the failure records under. +/// Holds the newest frame `parse` accepts until `deadline` or the end of the +/// stream. An error carries the status the failure records under. pub(crate) async fn read_bid_stream( request: Request<()>, deadline: Instant, relay: &RelayClient, - endpoint: &str, - max_held: usize, mut parse: impl FnMut(Frame) -> Result, ) -> Result, (StatusCode, PbsError)> { + let endpoint = GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG; let start_request = Instant::now(); let mut stream = match timeout_at(deadline, connect(request, deadline)).await { Ok(Ok(stream)) => stream, // A relay with no bid can answer the handshake as it answers get_header Ok(Err(WsError::Http(res))) if res.status() == StatusCode::NO_CONTENT => { return Ok(Held { - frames: VecDeque::new(), + latest: None, updates: 0, connect_latency: start_request.elapsed(), first_frame_latency: None, @@ -121,15 +117,15 @@ pub(crate) async fn read_bid_stream( Err(_) => return Err((TIMEOUT_ERROR_STATUS, PbsError::WebSocketTimeout)), }; let connect_latency = start_request.elapsed(); - RELAY_STREAM_CONNECT_LATENCY - .with_label_values(&[endpoint, relay.id.as_str()]) + RELAY_EPBS_STREAM_CONNECT_LATENCY + .with_label_values(&[relay.id.as_str()]) .observe(connect_latency.as_secs_f64()); debug!(relay_id = relay.id.as_ref(), endpoint, ?connect_latency, "ws connected"); let timer = sleep_until(deadline); tokio::pin!(timer); - let mut frames = VecDeque::with_capacity(max_held); + let mut latest = None; let mut updates = 0usize; let mut first_frame_latency = None; let mut invalid_frames = 0usize; @@ -157,10 +153,7 @@ pub(crate) async fn read_bid_stream( Ok(frame) => { first_frame_latency.get_or_insert_with(|| start_request.elapsed()); updates += 1; - if frames.len() == max_held { - frames.pop_front(); - } - frames.push_back(frame); + latest = Some(frame); } Err(err) => { invalid_frames += 1; @@ -173,10 +166,10 @@ pub(crate) async fn read_bid_stream( drop(stream); - RELAY_STREAM_UPDATES.with_label_values(&[endpoint, relay.id.as_str()]).observe(updates as f64); + RELAY_EPBS_STREAM_UPDATES.with_label_values(&[relay.id.as_str()]).observe(updates as f64); if invalid_frames > 0 { - RELAY_STREAM_INVALID_FRAMES - .with_label_values(&[endpoint, relay.id.as_str()]) + RELAY_EPBS_STREAM_INVALID_FRAMES + .with_label_values(&[relay.id.as_str()]) .inc_by(invalid_frames as u64); } @@ -186,7 +179,7 @@ pub(crate) async fn read_bid_stream( return Err((TRANSPORT_ERROR_STATUS, err)); } - Ok(Held { frames, updates, connect_latency, first_frame_latency, invalid_frames }) + Ok(Held { latest, updates, connect_latency, first_frame_latency, invalid_frames }) } /// Opens the stream with the timing headers stamped now. As over HTTP, @@ -210,11 +203,6 @@ async fn connect( Ok(stream) } -pub(crate) fn record_stream_failure(endpoint: &str, relay_id: &str, err: &PbsError) { - warn!(%err, relay_id, endpoint, "stream failed"); - RELAY_STREAM_FALLBACK.with_label_values(&[endpoint, relay_id]).inc(); -} - /// A relay rejecting the handshake puts the reason in the body, which /// tungstenite's own `Display` drops. The body is only what arrived alongside /// the headers, so it can be partial or empty. diff --git a/crates/pbs/src/metrics.rs b/crates/pbs/src/metrics.rs index 9745d53e4..a16491aa1 100644 --- a/crates/pbs/src/metrics.rs +++ b/crates/pbs/src/metrics.rs @@ -52,46 +52,90 @@ lazy_static! { .unwrap(); - // THE WEBSOCKET BID STREAMS + // THE WEBSOCKET BID STREAM // Outcome and time-to-first-bid ride RELAY_STATUS_CODE / RELAY_LATENCY - // under `get_header_stream` and `get_execution_payload_bid_stream` + // under `get_header_stream` /// Websocket handshake latency by relay pub static ref RELAY_STREAM_CONNECT_LATENCY: HistogramVec = register_histogram_vec_with_registry!( "relay_stream_connect_latency", "Websocket handshake latency by relay", - &["endpoint", "relay_id"], + &["relay_id"], vec![0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 1.0], PBS_METRICS_REGISTRY ) .unwrap(); - /// Stream frames accepted on arrival per window, by relay - // On arrival get_header_stream checks only the frame prefix, while - // get_execution_payload_bid_stream decodes the bid + /// Bid updates received per stream window by relay pub static ref RELAY_STREAM_UPDATES: HistogramVec = register_histogram_vec_with_registry!( "relay_stream_updates", - "Stream frames accepted on arrival per window, by relay", - &["endpoint", "relay_id"], + "Bid updates received per stream window by relay", + &["relay_id"], vec![0.0, 1.0, 2.0, 3.0, 5.0, 10.0, 20.0, 50.0, 100.0], PBS_METRICS_REGISTRY ) .unwrap(); - /// Stream frames rejected as invalid on arrival, by relay + /// Websocket frames that could not be parsed as a bid, by relay pub static ref RELAY_STREAM_INVALID_FRAMES: IntCounterVec = register_int_counter_vec_with_registry!( "relay_stream_invalid_frames_total", - "Stream frames rejected as invalid on arrival, by relay", - &["endpoint", "relay_id"], + "Websocket frames that could not be parsed as a bid, by relay", + &["relay_id"], PBS_METRICS_REGISTRY ) .unwrap(); - /// Bid streams that failed while the relay's HTTP request ran alongside + /// Stream attempts that fell back to HTTP, by relay + // Only a handshake failure with bid window left retries; one at the + // deadline shows on the status series alone pub static ref RELAY_STREAM_FALLBACK: IntCounterVec = register_int_counter_vec_with_registry!( "relay_stream_fallback_total", - "Bid streams that failed while the relay's HTTP request ran alongside, by relay", - &["endpoint", "relay_id"], + "get_header stream attempts that fell back to HTTP, by relay", + &["relay_id"], + PBS_METRICS_REGISTRY + ) + .unwrap(); + + // THE ePBS BID STREAM + // Outcome and time-to-first-bid ride RELAY_STATUS_CODE / RELAY_LATENCY + // under `get_execution_payload_bid_stream` + + /// ePBS bid stream handshake latency by relay + pub static ref RELAY_EPBS_STREAM_CONNECT_LATENCY: HistogramVec = register_histogram_vec_with_registry!( + "relay_epbs_stream_connect_latency", + "ePBS bid stream handshake latency by relay", + &["relay_id"], + vec![0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 1.0], + PBS_METRICS_REGISTRY + ) + .unwrap(); + + /// Bids that decoded per ePBS bid stream, by relay + pub static ref RELAY_EPBS_STREAM_UPDATES: HistogramVec = register_histogram_vec_with_registry!( + "relay_epbs_stream_updates", + "Bids that decoded per ePBS bid stream, by relay", + &["relay_id"], + vec![0.0, 1.0, 2.0, 3.0, 5.0, 10.0, 20.0, 50.0, 100.0], + PBS_METRICS_REGISTRY + ) + .unwrap(); + + /// ePBS bid stream frames that could not be read or decoded, by relay + pub static ref RELAY_EPBS_STREAM_INVALID_FRAMES: IntCounterVec = register_int_counter_vec_with_registry!( + "relay_epbs_stream_invalid_frames_total", + "ePBS bid stream frames that could not be read or decoded, by relay", + &["relay_id"], + PBS_METRICS_REGISTRY + ) + .unwrap(); + + /// ePBS bid streams that fell back to HTTP, by relay + // Only a failed connect with time left retries; one at the deadline shows + // on the status series alone + pub static ref RELAY_EPBS_STREAM_FALLBACK: IntCounterVec = register_int_counter_vec_with_registry!( + "relay_epbs_stream_fallback_total", + "ePBS bid streams that fell back to HTTP, by relay", + &["relay_id"], PBS_METRICS_REGISTRY ) .unwrap(); diff --git a/crates/pbs/src/mev_boost/get_header.rs b/crates/pbs/src/mev_boost/get_header.rs index 19067d521..670e80b16 100644 --- a/crates/pbs/src/mev_boost/get_header.rs +++ b/crates/pbs/src/mev_boost/get_header.rs @@ -1,6 +1,4 @@ use std::{ - collections::HashMap, - iter::once, sync::Arc, time::{Duration, Instant}, }; @@ -12,7 +10,6 @@ use alloy::{ }; use axum::http::{HeaderMap, HeaderValue}; use cb_common::{ - config::GetHeaderTransport, constants::APPLICATION_BUILDER_DOMAIN, pbs::{ EMPTY_TX_ROOT_HASH, ExecutionPayloadHeaderRef, ForkName, ForkVersionDecode, GetHeaderInfo, @@ -28,7 +25,7 @@ use cb_common::{ parse_response_encoding_and_fork, safe_read_http_response, }, }; -use futures::{FutureExt, future::join_all}; +use futures::future::join_all; use parking_lot::RwLock; use reqwest::{ StatusCode, @@ -41,12 +38,14 @@ use url::Url; use super::get_header_ws::get_header_ws; use crate::{ - bid_stream::record_stream_failure, constants::{ - GET_HEADER_ENDPOINT_TAG, GET_HEADER_STREAM_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, - TIMEOUT_ERROR_CODE, TIMEOUT_ERROR_CODE_STR, + GET_HEADER_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, TIMEOUT_ERROR_CODE, + TIMEOUT_ERROR_CODE_STR, + }, + metrics::{ + RELAY_HEADER_VALUE, RELAY_LAST_SLOT, RELAY_LATENCY, RELAY_STATUS_CODE, + RELAY_STREAM_FALLBACK, }, - metrics::{RELAY_HEADER_VALUE, RELAY_LAST_SLOT, RELAY_LATENCY, RELAY_STATUS_CODE}, state::{BuilderApiState, PbsState}, utils::check_gas_limit, }; @@ -172,77 +171,52 @@ pub async fn get_header( parent_block, }, }); - // A stream relay races its normal HTTP request; each is a separate candidate - // below - let params = &request_info.params; let mut handles = Vec::with_capacity(relays.len()); for relay in relays.iter() { - let relay_id = relay.id.as_str(); - let http_url = match relay.get_header_url(params.slot, ¶ms.parent_hash, ¶ms.pubkey) - { - Ok(url) => url, - Err(err) => { - error!(%err, relay_id); - continue; - } - }; - let stream_url = - relay.get_header_stream_url(params.slot, ¶ms.parent_hash, ¶ms.pubkey); - - let requests = - once(GetHeaderRequest::Http(http_url)).chain(stream_url.map(GetHeaderRequest::Stream)); - for request in requests { - let transport = request.transport(); - handles.push( - get_header_from_relay( - request_info.clone(), - relay.clone(), - ms_into_slot, - max_timeout_ms, - request, - ) - .map(move |res| (relay_id, transport, res)) - .in_current_span(), - ); - } + handles.push( + get_header_from_relay( + request_info.clone(), + relay.clone(), + ms_into_slot, + max_timeout_ms, + ) + .in_current_span(), + ); } - let mut relay_bids = Vec::with_capacity(handles.len()); - let mut best_per_relay: HashMap<&str, U256> = HashMap::new(); - for (relay_id, transport, res) in join_all(handles).await { + let results = join_all(handles).await; + let mut relay_bids = Vec::with_capacity(relays.len()); + for (i, res) in results.into_iter().enumerate() { + let relay_id = relays[i].id.as_str(); + match res { Ok(Some(res)) => { - let best = best_per_relay.entry(relay_id).or_default(); - *best = (*best).max(*res.value()); - relay_bids.push((relay_id, transport, res)) - } - Ok(None) => {} - Err(err) if transport == GetHeaderTransport::Stream => { - record_stream_failure(GET_HEADER_STREAM_ENDPOINT_TAG, relay_id, &err) + RELAY_LAST_SLOT.with_label_values(&[relay_id]).set(slot); + let value_gwei = (res.data.message.value() / U256::from(1_000_000_000)) + .try_into() + .unwrap_or_default(); + RELAY_HEADER_VALUE.with_label_values(&[relay_id]).set(value_gwei); + + relay_bids.push((relay_id, res)) } + Ok(_) => {} Err(err) if err.is_timeout() => error!(err = "Timed Out", relay_id), Err(err) => error!(%err, relay_id), } } - for (relay_id, value) in best_per_relay { - RELAY_LAST_SLOT.with_label_values(&[relay_id]).set(slot); - let value_gwei = (value / U256::from(1_000_000_000)).try_into().unwrap_or_default(); - RELAY_HEADER_VALUE.with_label_values(&[relay_id]).set(value_gwei); - } - let max_bid = relay_bids.into_iter().max_by_key(|(_, _, bid)| *bid.value()); + let max_bid = relay_bids.into_iter().max_by_key(|(_, bid)| *bid.value()); - if let Some((winning_relay_id, transport, ref bid)) = max_bid { + if let Some((winning_relay_id, ref bid)) = max_bid { info!( relay_id = winning_relay_id, - transport = transport.as_str(), value_eth = format_ether(*bid.value()), block_hash = %bid.block_hash(), "auction winner" ); } - Ok(max_bid.map(|(_, _, bid)| bid)) + Ok(max_bid.map(|(_, bid)| bid)) } /// Fetch the parent block from the RPC URL for extra validation of the header. @@ -275,11 +249,35 @@ async fn get_header_from_relay( relay: RelayClient, ms_into_slot: u64, timeout_left_ms: u64, - request: GetHeaderRequest, ) -> Result, PbsError> { - match request { + let params = &request_info.params; + + match relay.get_header_request(params.slot, ¶ms.parent_hash, ¶ms.pubkey)? { GetHeaderRequest::Stream(url) => { - get_header_ws(&request_info, &relay, url, timeout_left_ms).await + let started = Instant::now(); + let err = match get_header_ws(&request_info, &relay, url, timeout_left_ms).await { + Err(PbsError::WebSocketConnect(err)) => err, + res => return res, + }; + + let elapsed_ms = started.elapsed().as_millis() as u64; + let timeout_left_ms = timeout_left_ms.saturating_sub(elapsed_ms); + if timeout_left_ms == 0 { + return Err(PbsError::WebSocketConnect(err)); + } + + RELAY_STREAM_FALLBACK.with_label_values(&[relay.id.as_str()]).inc(); + warn!(relay_id = relay.id.as_ref(), %err, timeout_left_ms, "stream failed, falling back to http get_header"); + + let url = relay.get_header_url(params.slot, ¶ms.parent_hash, ¶ms.pubkey)?; + send_timed_get_header( + request_info, + relay, + ms_into_slot + elapsed_ms, + url, + timeout_left_ms, + ) + .await } GetHeaderRequest::Http(url) => { send_timed_get_header(request_info, relay, ms_into_slot, url, timeout_left_ms).await @@ -735,24 +733,19 @@ fn extra_validation( #[cfg(test)] mod tests { - use std::{fs, net::SocketAddr, path::Path}; + use std::{fs, path::Path}; use alloy::primitives::{B256, U256}; use cb_common::{ - config::{PbsConfig, PbsModuleConfig, RelayConfig}, pbs::*, signature::sign_builder_message, ssz::get_bid_value_from_signed_builder_bid_ssz, types::{BlsPublicKeyBytes, BlsSecretKey, BlsSignature, Chain}, utils::{TestRandomSeed, timestamp_of_slot_start_sec}, }; - use futures::SinkExt; use ssz::Encode; - use tokio::net::TcpListener; - use tokio_tungstenite::{accept_async, tungstenite::Message}; use super::{validate_header_data, *}; - use crate::metrics::{RELAY_STREAM_FALLBACK, RELAY_STREAM_UPDATES}; #[test] fn test_validate_header() { @@ -888,207 +881,4 @@ mod tests { assert_eq!(*decoded.value(), bid_value); } } - - /// A relay that answers the stream handshake with `status`, and its HTTP - /// get_header with 204, so only the stream can count as failed - async fn start_status_relay(status: StatusCode) -> SocketAddr { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let relay = axum::Router::new().fallback(move |headers: HeaderMap| async move { - if headers.contains_key(reqwest::header::UPGRADE) { - status - } else { - StatusCode::NO_CONTENT - } - }); - tokio::spawn(async move { axum::serve(listener, relay).await }); - addr - } - - /// A relay whose stream sends `frames` copies of a bid for another parent, - /// then closes. Its HTTP get_header fails. - async fn start_invalid_bid_stream_relay(frames: usize) -> SocketAddr { - let json = fs::read("../../tests/data/get_header/fulu.json").unwrap(); - // Message type bid, fork fulu - let mut frame = vec![0x01, 6]; - frame.extend(decode_json_payload(&json).unwrap().data.as_ssz_bytes()); - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - while let Ok((socket, _)) = listener.accept().await { - let frame = frame.clone(); - tokio::spawn(async move { - let Ok(mut stream) = accept_async(socket).await else { return }; - for _ in 0..frames { - let _ = stream.send(Message::Binary(frame.clone().into())).await; - } - let _ = stream.close(None).await; - }); - } - }); - addr - } - - /// get_header for `slot` and `parent_hash` against one stream relay at - /// `addr`. Returns the relay's id, which labels its metrics, and the bid. - async fn get_header_from_stream_relay( - addr: SocketAddr, - slot: u64, - parent_hash: B256, - ) -> (String, Option) { - let pubkey = BlsSecretKey::random().public_key(); - let relay_id = format!("stream_relay_{}", addr.port()); - let relay = RelayClient::new(RelayConfig { - entry: RelayEntry { - id: relay_id.clone(), - pubkey: pubkey.clone(), - url: format!("http://{pubkey}@{addr}").parse().unwrap(), - }, - id: None, - headers: None, - get_params: None, - get_header: GetHeaderTransport::Stream, - enable_timing_games: false, - target_first_request_ms: None, - frequency_get_header_ms: None, - validator_registration_batch_size: None, - }) - .unwrap(); - let mut pbs_config: PbsConfig = serde_json::from_str("{}").unwrap(); - pbs_config.late_in_slot_time_ms = u64::MAX; - // The fixture bids are not signed by the relay's key - pbs_config.skip_sigverify = true; - let state = PbsState::new( - PbsModuleConfig { - chain: Chain::Holesky, - endpoint: addr, - pbs_config: Arc::new(pbs_config), - relays: vec![relay.clone()], - all_relays: vec![relay], - signer_client: None, - registry_muxes: None, - mux_lookup: None, - }, - Default::default(), - ); - let params = GetHeaderParams { slot, parent_hash, pubkey }; - - let bid = get_header(params, HeaderMap::new(), state).await.unwrap(); - (relay_id, bid) - } - - #[tokio::test] - async fn test_stream_fallback_counts_a_refused_handshake_not_a_204() { - for (status, fallbacks) in [(StatusCode::NO_CONTENT, 0), (StatusCode::NOT_FOUND, 1)] { - let addr = start_status_relay(status).await; - let (relay_id, bid) = get_header_from_stream_relay(addr, 1, B256::ZERO).await; - assert!(bid.is_none()); - assert_eq!( - RELAY_STATUS_CODE - .with_label_values(&[ - status.as_str(), - GET_HEADER_STREAM_ENDPOINT_TAG, - &relay_id - ]) - .get(), - 1, - "handshake answered {status}" - ); - assert_eq!( - RELAY_STREAM_FALLBACK - .with_label_values(&[GET_HEADER_STREAM_ENDPOINT_TAG, &relay_id]) - .get(), - fallbacks, - "handshake answered {status}" - ); - } - } - - // A window whose every bid fails validation records 200 and one failed - // stream. Its updates count every bid it accepted, not only those it held. - #[tokio::test] - async fn test_stream_fallback_counts_a_window_with_no_valid_bid() { - let addr = start_invalid_bid_stream_relay(10).await; - let (relay_id, bid) = get_header_from_stream_relay(addr, 1, B256::ZERO).await; - assert!(bid.is_none()); - - let status = RELAY_STATUS_CODE.with_label_values(&[ - StatusCode::OK.as_str(), - GET_HEADER_STREAM_ENDPOINT_TAG, - &relay_id, - ]); - assert_eq!(status.get(), 1); - let stream = [GET_HEADER_STREAM_ENDPOINT_TAG, relay_id.as_str()]; - assert_eq!(RELAY_STREAM_FALLBACK.with_label_values(&stream).get(), 1); - assert_eq!(RELAY_STREAM_UPDATES.with_label_values(&stream).get_sample_sum(), 10.0); - } - - /// The slot whose start is the fixture bid's timestamp, on Holesky - const FIXTURE_SLOT: u64 = 1_714_953; - - /// The fixture's get_header response bidding `gwei`, and its parent hash - fn fixture_bid_json(gwei: u64) -> (B256, Vec) { - let json = fs::read("../../tests/data/get_header/fulu.json").unwrap(); - let mut json: serde_json::Value = serde_json::from_slice(&json).unwrap(); - let wei = U256::from(gwei) * U256::from(1_000_000_000); - json["data"]["message"]["value"] = wei.to_string().into(); - let parent_hash = json["data"]["message"]["header"]["parent_hash"].as_str().unwrap(); - (parent_hash.parse().unwrap(), serde_json::to_vec(&json).unwrap()) - } - - /// A relay serving `http_body` as its HTTP get_header and `frame` on its - /// stream, both on one port - async fn start_dual_relay(http_body: Vec, frame: Vec) -> SocketAddr { - let http = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let http_addr = http.local_addr().unwrap(); - let relay = axum::Router::new().fallback(move || { - let body = http_body.clone(); - async move { ([(reqwest::header::CONTENT_TYPE, "application/json")], body) } - }); - tokio::spawn(async move { axum::serve(http, relay).await }); - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - while let Ok((mut socket, _)) = listener.accept().await { - let frame = frame.clone(); - tokio::spawn(async move { - let mut head = [0; 2048]; - let n = socket.peek(&mut head).await.unwrap_or_default(); - let head = String::from_utf8_lossy(&head[..n]).to_ascii_lowercase(); - if !head.contains("upgrade: websocket") { - let Ok(mut http) = tokio::net::TcpStream::connect(http_addr).await else { - return; - }; - let _ = tokio::io::copy_bidirectional(&mut socket, &mut http).await; - return; - } - let Ok(mut stream) = accept_async(socket).await else { return }; - let _ = stream.send(Message::Binary(frame.into())).await; - let _ = stream.close(None).await; - }); - } - }); - addr - } - - // A relay's gauges record the better of its HTTP and stream bids - #[tokio::test] - async fn test_relay_gauges_take_the_better_leg() { - for (http_gwei, stream_gwei) in [(60, 50), (50, 60)] { - let (parent_hash, http_bid) = fixture_bid_json(http_gwei); - let (_, stream_bid) = fixture_bid_json(stream_gwei); - // Message type bid, fork fulu - let mut frame = vec![0x01, 6]; - frame.extend(decode_json_payload(&stream_bid).unwrap().data.as_ssz_bytes()); - let addr = start_dual_relay(http_bid, frame).await; - - let (relay_id, bid) = - get_header_from_stream_relay(addr, FIXTURE_SLOT, parent_hash).await; - assert_eq!(*bid.expect("a bid").value(), U256::from(60_000_000_000u64)); - assert_eq!(RELAY_HEADER_VALUE.with_label_values(&[&relay_id]).get(), 60); - assert_eq!(RELAY_LAST_SLOT.with_label_values(&[&relay_id]).get(), FIXTURE_SLOT as i64); - } - } } diff --git a/crates/pbs/src/mev_boost/get_header_ws.rs b/crates/pbs/src/mev_boost/get_header_ws.rs index 3fd9bd20f..a91017ec8 100644 --- a/crates/pbs/src/mev_boost/get_header_ws.rs +++ b/crates/pbs/src/mev_boost/get_header_ws.rs @@ -1,28 +1,83 @@ -//! get_header over the bid stream, for relays configured with +//! Streaming get_header over a websocket, for relays configured with //! `get_header = "stream"`. The stream URL is derived from the relay's own -//! `url`. The handshake carries the same data as the HTTP request: slot / -//! parent_hash / pubkey in the path, deadline and timestamp in headers. +//! `url`. One connection per get_header call, dropped when the call returns. +//! +//! The request is the handshake itself: slot / parent_hash / pubkey in the +//! path, deadline and timestamp in headers, same data the HTTP request carries. +//! The relay replies with one binary frame per bid update: +//! +//! ```text +//! u8 message type +//! u8 fork +//! .. SSZ SignedBuilderBid +//! ``` -use std::time::Duration; +use std::{ + sync::{Arc, OnceLock}, + time::Duration, +}; use alloy::primitives::utils::format_ether; -use cb_common::pbs::{GetHeaderInfo, GetHeaderResponse, RelayClient, error::PbsError}; +use axum::http::{HeaderValue, Request, header::USER_AGENT}; +use cb_common::{ + pbs::{ + ForkName, GetHeaderInfo, GetHeaderResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, + RelayClient, error::PbsError, + }, + utils::utcnow_ms, +}; +use futures::StreamExt; use reqwest::StatusCode; -use tokio::time::Instant; +use rustls::{ClientConfig, RootCertStore, crypto::aws_lc_rs}; +use tokio::time::{Instant, sleep_until, timeout_at}; +use tokio_tungstenite::{ + Connector, connect_async_tls_with_config, + tungstenite::{ + Bytes, Error as WsError, Message, client::IntoClientRequest, protocol::WebSocketConfig, + }, +}; use tracing::{debug, info, warn}; use url::Url; -use super::get_header::{RequestInfo, decode_ssz_payload, validate_get_header_response}; +use super::get_header::{RequestInfo, validate_get_header_response}; use crate::{ - bid_stream::{Frame, Held, MAX_HELD_FRAMES, handshake_request, read_bid_stream}, - constants::{GET_HEADER_STREAM_ENDPOINT_TAG, TRANSPORT_ERROR_STATUS}, - metrics::RELAY_STATUS_CODE, + constants::{ + GET_HEADER_STREAM_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, TIMEOUT_ERROR_STATUS, + TRANSPORT_ERROR_STATUS, + }, + metrics::{ + RELAY_LATENCY, RELAY_STATUS_CODE, RELAY_STREAM_CONNECT_LATENCY, + RELAY_STREAM_INVALID_FRAMES, RELAY_STREAM_UPDATES, + }, + mev_boost::get_header::decode_ssz_payload, }; +/// Frame prefix: message type + fork. +const FRAME_PREFIX_LEN: usize = 2; + +const MSG_BID: u8 = 0x01; + +fn fork_from_wire(byte: u8) -> Option { + // TODO @nina: I don't see a point of extending a u8 for supporting older forks + // we could rotate these instead, i.e. 0 becomes Heze, etc + Some(match byte { + 0 => ForkName::Base, + 1 => ForkName::Altair, + 2 => ForkName::Bellatrix, + 3 => ForkName::Capella, + 4 => ForkName::Deneb, + 5 => ForkName::Electra, + 6 => ForkName::Fulu, + 7 => ForkName::Gloas, + 8 => ForkName::Heze, + _ => return None, + }) +} + type StreamOutcome = (StatusCode, Result, PbsError>); -/// Open a stream to the relay, hold its newest bids until the deadline, then -/// return the latest one that passes validation. +/// Open a stream to the relay, keep the latest bid until the deadline, then +/// validate and return it. pub(super) async fn get_header_ws( request_info: &RequestInfo, relay: &RelayClient, @@ -43,61 +98,126 @@ async fn stream_header( timeout_ms: u64, ) -> StreamOutcome { let deadline = Instant::now() + Duration::from_millis(timeout_ms); - let request = match handshake_request(&url, relay, &request_info.headers) { + let request = match build_handshake_request(request_info, relay, &url, timeout_ms) { Ok(request) => request, Err(err) => return (TRANSPORT_ERROR_STATUS, Err(err)), }; - let Held { frames, updates, connect_latency, first_frame_latency, invalid_frames } = - match read_bid_stream( - request, - deadline, - relay, - GET_HEADER_STREAM_ENDPOINT_TAG, - MAX_HELD_FRAMES, - Ok, - ) - .await - { - Ok(held) => held, - Err((status, err)) => return (status, Err(err)), + let config = WebSocketConfig::default() + .max_message_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)) + .max_frame_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)); + + let start_request = Instant::now(); + let connect = connect_async_tls_with_config( + request, + Some(config), + true, + Some(Connector::Rustls(tls_config().clone())), + ); + let (mut stream, _) = match timeout_at(deadline, connect).await { + Ok(Ok(connected)) => connected, + Ok(Err(err)) => return connect_failed(&err), + Err(_) => { + return (TIMEOUT_ERROR_STATUS, Err(PbsError::WebSocketTimeout)); + } + }; + let connect_latency = start_request.elapsed(); + RELAY_STREAM_CONNECT_LATENCY + .with_label_values(&[relay.id.as_str()]) + .observe(connect_latency.as_secs_f64()); + debug!(relay_id = relay.id.as_ref(), ?connect_latency, "ws connected"); + + let timer = sleep_until(deadline); + tokio::pin!(timer); + + let mut latest: Option<(ForkName, Bytes)> = None; + let mut first_bid_latency: Option = None; + let mut updates = 0usize; + let mut invalid_frames = 0usize; + let mut stream_error = None; + + loop { + let message = tokio::select! { + biased; + _ = &mut timer => break, + message = stream.next() => message, }; - // The latest valid bid stands. Newest first, so the usual case validates once - let start_validate = Instant::now(); - let mut newest_err = None; - let latest_valid = frames.iter().rev().find_map(|frame| { - let validated = decode_ssz_payload(&frame.bid, frame.fork).and_then(|response| { - validate_get_header_response(request_info, relay, &response).map(|()| response) - }); - match validated { - Ok(response) => Some((frame, response)), + let message = match message { + Some(Ok(message)) => message, + Some(Err(err)) => { + warn!(relay_id = relay.id.as_ref(), %err, "ws stream error"); + stream_error = Some(PbsError::WebSocket(format!("stream error: {err}"))); + break; + } + None => break, + }; + + let payload = match message { + Message::Binary(payload) => payload, + Message::Close(_) => break, + _ => continue, + }; + + match parse_frame(payload) { + Ok((fork, bid)) => { + updates += 1; + if first_bid_latency.is_none() { + first_bid_latency = Some(start_request.elapsed()); + } + latest = Some((fork, bid)); + } Err(err) => { - newest_err.get_or_insert(err); - None + invalid_frames += 1; + if invalid_frames == 1 { + warn!(relay_id = relay.id.as_ref(), %err, "invalid ws frame, skipping"); + } } } - }); - let validate_latency = start_validate.elapsed(); + } - let (Frame { fork, bid }, response) = match (latest_valid, newest_err) { - (Some(valid), None) => valid, - (Some(valid), Some(err)) => { - warn!(relay_id = relay.id.as_ref(), %err, "latest stream bid invalid, an earlier one stands"); - valid - } - (None, Some(err)) => return (StatusCode::OK, Err(err)), - (None, None) => { - debug!(relay_id = relay.id.as_ref(), ?connect_latency, invalid_frames, "no header"); - return (StatusCode::NO_CONTENT, Ok(None)); + drop(stream); + + RELAY_STREAM_UPDATES.with_label_values(&[relay.id.as_str()]).observe(updates as f64); + if invalid_frames > 0 { + RELAY_STREAM_INVALID_FRAMES + .with_label_values(&[relay.id.as_str()]) + .inc_by(invalid_frames as u64); + } + + let Some((fork, bid_bytes)) = latest else { + if let Some(err) = stream_error { + return (TRANSPORT_ERROR_STATUS, Err(err)); } + + debug!(relay_id = relay.id.as_ref(), ?connect_latency, invalid_frames, "no header"); + return (StatusCode::NO_CONTENT, Ok(None)); }; + if let Some(first_bid_latency) = first_bid_latency { + RELAY_LATENCY + .with_label_values(&[GET_HEADER_STREAM_ENDPOINT_TAG, &relay.id]) + .observe(first_bid_latency.as_secs_f64()); + } + + let response = match decode_ssz_payload(&bid_bytes, fork) { + Ok(response) => response, + Err(err) => return (StatusCode::OK, Err(err)), + }; + + let start_validate = Instant::now(); + let validated = validate_get_header_response(request_info, relay, &response); + let validate_latency = start_validate.elapsed(); + + if let Err(err) = validated { + return (StatusCode::OK, Err(err)); + } + info!( relay_id = relay.id.as_ref(), - header_size_bytes = bid.len(), + header_size_bytes = bid_bytes.len(), ?connect_latency, - first_bid_latency = ?first_frame_latency, + ?first_bid_latency, ?validate_latency, version = ?fork, value_eth = format_ether(*response.value()), @@ -109,3 +229,200 @@ async fn stream_header( (StatusCode::OK, Ok(Some(response))) } + +/// A relay rejecting the handshake puts the reason in the body, which +/// tungstenite's own `Display` drops. The body is only what arrived alongside +/// the headers, so it can be partial or empty. +fn connect_failed(err: &WsError) -> StreamOutcome { + let WsError::Http(res) = err else { + return (TRANSPORT_ERROR_STATUS, Err(PbsError::WebSocketConnect(err.to_string()))); + }; + + let code = res.status(); + let body = res.body().as_deref().unwrap_or_default(); + let msg = if body.is_empty() { + format!("rejected with {code}") + } else { + format!("rejected with {code}: {}", String::from_utf8_lossy(body)) + }; + + // A 2xx handshake answer is a failed connect, not a delivered bid + let code = if code.is_success() { TRANSPORT_ERROR_STATUS } else { code }; + + (code, Err(PbsError::WebSocketConnect(msg))) +} + +fn build_handshake_request( + request_info: &RequestInfo, + relay: &RelayClient, + url: &Url, + timeout_ms: u64, +) -> Result, PbsError> { + let mut request = url + .as_str() + .into_client_request() + .map_err(|err| PbsError::WebSocketConnect(format!("invalid ws url: {err}")))?; + + let headers = request.headers_mut(); + if let Some(user_agent) = request_info.headers.get(USER_AGENT) { + headers.insert(USER_AGENT, user_agent.clone()); + } + + for (key, value) in relay.stream_headers() { + headers.insert(key, value.clone()); + } + + headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); + headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); + + Ok(request) +} + +fn parse_frame(payload: Bytes) -> Result<(ForkName, Bytes), PbsError> { + let &[msg_type, fork_byte] = payload + .first_chunk::() + .ok_or_else(|| PbsError::WebSocket(format!("frame too short: {} bytes", payload.len())))?; + + if msg_type != MSG_BID { + return Err(PbsError::WebSocket(format!("unknown message type: {msg_type}"))); + } + + let fork = fork_from_wire(fork_byte) + .ok_or_else(|| PbsError::WebSocket(format!("unknown fork: {fork_byte}")))?; + + Ok((fork, payload.slice(FRAME_PREFIX_LEN..))) +} + +/// One TLS config for every stream connection. Left to tokio-tungstenite it is +/// rebuilt per connect, which reparses the root store and, worse, gives each +/// connection its own session cache: every slot then pays a full handshake +/// instead of a resumed one. The provider is named explicitly because rustls is +/// built with both `ring` and `aws-lc-rs` here, so the default builder needs a +/// process-wide install to pick one. +fn tls_config() -> &'static Arc { + static CONFIG: OnceLock> = OnceLock::new(); + CONFIG.get_or_init(|| { + let mut roots = RootCertStore::empty(); + roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + + Arc::new( + ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .expect("aws-lc-rs supports tls 1.2 and 1.3") + .with_root_certificates(roots) + .with_no_client_auth(), + ) + }) +} + +#[cfg(test)] +mod tests { + use std::{fs, path::Path}; + + use ssz::Encode; + + use super::*; + + fn bid_frame(fork_byte: u8, bid: &[u8]) -> Bytes { + let mut frame = vec![MSG_BID, fork_byte]; + frame.extend_from_slice(bid); + Bytes::from(frame) + } + + #[test] + fn test_fork_from_wire_covers_all_forks() { + let forks = ForkName::list_all(); + + for (byte, fork) in forks.iter().enumerate() { + if *fork <= ForkName::Base { + continue; + } + assert_eq!(fork_from_wire(byte as u8), Some(*fork), "fork {fork} unmapped"); + } + + assert_eq!(fork_from_wire(forks.len() as u8), None); + } + + #[test] + fn test_parse_frame() { + assert!(matches!( + parse_frame(bid_frame(6, &[1, 2, 3])), + Ok((ForkName::Fulu, bid)) if bid.as_ref() == [1, 2, 3] + )); + + // Empty bid payload is well-formed at this layer, SSZ decoding rejects it + assert!(matches!(parse_frame(bid_frame(6, &[])), Ok((ForkName::Fulu, _)))); + + for bad in [ + // Truncated prefix + Bytes::from_static(&[]), + Bytes::from_static(&[MSG_BID]), + // Unknown fork + bid_frame(0xff, &[1]), + // Unknown message type + Bytes::from_static(&[0xff, 6]), + ] { + assert!(matches!(parse_frame(bad), Err(PbsError::WebSocket(_)))); + } + } + + #[test] + fn test_connect_failed_carries_relay_reason() { + let rejected = axum::http::Response::builder() + .status(401) + .body(Some(b"api key not registered".to_vec())) + .unwrap(); + + let (status, res) = connect_failed(&WsError::Http(Box::new(rejected))); + assert_eq!(status, StatusCode::UNAUTHORIZED); + let Err(PbsError::WebSocketConnect(msg)) = res else { panic!("wrong outcome") }; + assert!(msg.contains("401"), "{msg}"); + assert!(msg.contains("api key not registered"), "{msg}"); + + // No body to add, and transport failures keep tungstenite's own message + let empty = axum::http::Response::builder().status(404).body(None).unwrap(); + let (status, _) = connect_failed(&WsError::Http(Box::new(empty))); + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, res) = connect_failed(&WsError::ConnectionClosed); + assert_eq!(status, TRANSPORT_ERROR_STATUS); + assert!(matches!(res, Err(PbsError::WebSocketConnect(_)))); + } + + // A url pointing at a plain http endpoint answers the handshake 200. That + // is the code the stream series uses for a delivered bid, so a failed + // handshake must never carry it. + #[test] + fn test_connect_failed_never_reports_a_success_code() { + for code in [200u16, 204, 299] { + let answered = axum::http::Response::builder().status(code).body(None).unwrap(); + let (status, res) = connect_failed(&WsError::Http(Box::new(answered))); + assert_eq!( + status, TRANSPORT_ERROR_STATUS, + "handshake answered {code} counted as a served stream" + ); + let Err(PbsError::WebSocketConnect(msg)) = res else { panic!("wrong outcome") }; + assert!(msg.contains(&code.to_string()), "{msg}"); + } + + // A relay's own rejection code still reaches the series unchanged + let moved = axum::http::Response::builder().status(302).body(None).unwrap(); + let (status, _) = connect_failed(&WsError::Http(Box::new(moved))); + assert_eq!(status, StatusCode::FOUND); + } + + #[test] + fn test_decode_streamed_bid() { + let json_bytes = + fs::read(Path::new("../../tests/data/get_header/fulu.json")).expect("file not found"); + let expected: GetHeaderResponse = + serde_json::from_slice(&json_bytes).expect("failed to decode JSON"); + + let frame = bid_frame(6, &expected.data.as_ssz_bytes()); + let (fork, bid_bytes) = parse_frame(frame).unwrap(); + + let decoded = decode_ssz_payload(&bid_bytes, fork).unwrap(); + assert_eq!(fork, ForkName::Fulu); + assert_eq!(decoded.data, expected.data); + } +} diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index 2a7965ea7..661f2ddb6 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -32,11 +32,10 @@ use super::execution_payload_bid_ws::get_execution_payload_bid_ws; use crate::{ PbsStateGuard, constants::{ - GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, - MAX_SIZE_DEFAULT, MAX_SIZE_GET_HEADER_RESPONSE, + GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, MAX_SIZE_DEFAULT, MAX_SIZE_GET_HEADER_RESPONSE, }, error::PbsClientError, - metrics::{RELAY_HEADER_VALUE, RELAY_LAST_SLOT, RELAY_STREAM_FALLBACK}, + metrics::{RELAY_EPBS_STREAM_FALLBACK, RELAY_HEADER_VALUE, RELAY_LAST_SLOT}, state::{BuilderApiState, PbsState}, utils::{ builder_rejection, epbs_base_send_headers, format_gwei_as_eth, log_mux_selection, @@ -238,9 +237,7 @@ async fn stream_execution_payload_bid( return Ok(None); } - RELAY_STREAM_FALLBACK - .with_label_values(&[GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, relay.id.as_str()]) - .inc(); + RELAY_EPBS_STREAM_FALLBACK.with_label_values(&[relay.id.as_str()]).inc(); warn!(relay_id = relay.id.as_ref(), %err, timeout_left_ms, "stream failed, falling back to http"); send_get_execution_payload_bid(params, body, relay, send_headers, timeout_left_ms).await } @@ -376,10 +373,14 @@ mod tests { use super::*; use crate::{ - constants::{TIMEOUT_ERROR_STATUS, TRANSPORT_ERROR_STATUS}, + constants::{ + GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, TIMEOUT_ERROR_STATUS, + TRANSPORT_ERROR_STATUS, + }, metrics::{ - PBS_METRICS_REGISTRY, RELAY_LATENCY, RELAY_STATUS_CODE, RELAY_STREAM_CONNECT_LATENCY, - RELAY_STREAM_INVALID_FRAMES, RELAY_STREAM_UPDATES, + PBS_METRICS_REGISTRY, RELAY_EPBS_STREAM_CONNECT_LATENCY, + RELAY_EPBS_STREAM_INVALID_FRAMES, RELAY_EPBS_STREAM_UPDATES, RELAY_LATENCY, + RELAY_STATUS_CODE, }, }; @@ -624,7 +625,7 @@ mod tests { } fn fallbacks(relay_id: &str) -> u64 { - RELAY_STREAM_FALLBACK.with_label_values(&[STREAM, relay_id]).get() + RELAY_EPBS_STREAM_FALLBACK.with_label_values(&[relay_id]).get() } // A handshake answered 204 is no bid. Only a refused handshake falls back @@ -677,12 +678,12 @@ mod tests { assert_eq!(status_count(StatusCode::NO_CONTENT, &relay_id), 1); assert_eq!(fallbacks(&relay_id), 0); assert_eq!(posts.load(Ordering::Relaxed), 0); - let stream = [STREAM, relay_id.as_str()]; + let relay = [relay_id.as_str()]; assert_eq!( - RELAY_STREAM_INVALID_FRAMES.with_label_values(&stream).get(), + RELAY_EPBS_STREAM_INVALID_FRAMES.with_label_values(&relay).get(), n_frames as u64 ); - assert_eq!(RELAY_STREAM_UPDATES.with_label_values(&stream).get_sample_sum(), 0.0); + assert_eq!(RELAY_EPBS_STREAM_UPDATES.with_label_values(&relay).get_sample_sum(), 0.0); } } @@ -709,16 +710,19 @@ mod tests { assert_eq!(status_count(StatusCode::OK, &relay_id), 1); assert_eq!(RELAY_HEADER_VALUE.with_label_values(&[&relay_id]).get(), 77); assert_eq!(RELAY_LAST_SLOT.with_label_values(&[&relay_id]).get(), 1); - let stream = [STREAM, relay_id.as_str()]; - assert_eq!(RELAY_STREAM_UPDATES.with_label_values(&stream).get_sample_sum(), 2.0); - assert_eq!(RELAY_STREAM_CONNECT_LATENCY.with_label_values(&stream).get_sample_count(), 1); + let relay = [relay_id.as_str()]; + assert_eq!(RELAY_EPBS_STREAM_UPDATES.with_label_values(&relay).get_sample_sum(), 2.0); + assert_eq!( + RELAY_EPBS_STREAM_CONNECT_LATENCY.with_label_values(&relay).get_sample_count(), + 1 + ); // The last bid arrives 450 ms in - let latency = RELAY_LATENCY.with_label_values(&stream); + let latency = RELAY_LATENCY.with_label_values(&[STREAM, relay_id.as_str()]); assert_eq!(latency.get_sample_count(), 1); assert!(latency.get_sample_sum() < 0.3, "{}", latency.get_sample_sum()); } - // Dashboards and the docs select the stream series by label name + // Dashboards and the docs select the stream series by name #[tokio::test] async fn test_epbs_stream_series_are_labeled_by_name() { let messages = vec![Message::Binary(vec![0x01, 7, 1].into()), bid_frame(1, 0)]; @@ -729,12 +733,12 @@ mod tests { let scrape = TextEncoder::new().encode_to_string(&PBS_METRICS_REGISTRY.gather()).unwrap(); for (series, relay_id) in [ - ("relay_stream_connect_latency_count", &relay_id), - ("relay_stream_updates_sum", &relay_id), - ("relay_stream_invalid_frames_total", &relay_id), - ("relay_stream_fallback_total", &failed_id), + ("relay_epbs_stream_connect_latency_count", &relay_id), + ("relay_epbs_stream_updates_sum", &relay_id), + ("relay_epbs_stream_invalid_frames_total", &relay_id), + ("relay_epbs_stream_fallback_total", &failed_id), ] { - let series = format!(r#"{series}{{endpoint="{STREAM}",relay_id="{relay_id}"}}"#); + let series = format!(r#"{series}{{relay_id="{relay_id}"}}"#); assert!(scrape.contains(&series), "{series}"); } } diff --git a/crates/pbs/src/routes/execution_payload_bid_ws.rs b/crates/pbs/src/routes/execution_payload_bid_ws.rs index 646bf2721..a3a4d8212 100644 --- a/crates/pbs/src/routes/execution_payload_bid_ws.rs +++ b/crates/pbs/src/routes/execution_payload_bid_ws.rs @@ -60,24 +60,15 @@ async fn stream_execution_payload_bid( headers.insert(HEADER_REQUEST_AUTH, request_auth); // Each frame is decoded on arrival, so the newest is the latest bid that - // decodes and the only one to hold + // decodes let parse = |frame: Frame| decode_ssz_bid(frame.bid, frame.fork); - let Held { mut frames, updates, connect_latency, first_frame_latency, invalid_frames } = - match read_bid_stream( - request, - deadline, - relay, - GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, - 1, - parse, - ) - .await - { + let Held { latest, updates, connect_latency, first_frame_latency, invalid_frames } = + match read_bid_stream(request, deadline, relay, parse).await { Ok(held) => held, Err((status, err)) => return (status, Err(err)), }; - let Some(bid) = frames.pop_back() else { + let Some(bid) = latest else { debug!(relay_id = relay.id.as_ref(), ?connect_latency, invalid_frames, "no bid"); return (StatusCode::NO_CONTENT, Ok(None)); }; diff --git a/docs/docs/get_started/configuration.md b/docs/docs/get_started/configuration.md index 98fada1ab..04a8963b2 100644 --- a/docs/docs/get_started/configuration.md +++ b/docs/docs/get_started/configuration.md @@ -75,7 +75,7 @@ Each `[[relays]]` entry supports, besides `id` and `url`: - `headers`: custom headers sent with every request to this relay, which is how a relay API key is supplied. See [Relay API keys](#relay-api-keys). - `get_params`: optional GET parameters to add to each request URL for this relay. -- `get_header`: how bids are fetched from this relay: `"http"`, one request per `get_header`, or `"stream"`, a websocket stream of bid updates for relays that support it. (unreleased, from v0.12.0-rc1) A streaming relay also gets the HTTP request. Default: `"http"`; details under [Bid streaming](#bid-streaming). +- `get_header`: how bids are fetched from this relay, either `"http"` (one request per `get_header`) or `"stream"` (a websocket stream of bid updates, for relays that support it). Default: `"http"`. See [Bid streaming](#bid-streaming). - `enable_timing_games`: whether to enable timing games for this relay, as tuned by `target_first_request_ms` and `frequency_get_header_ms`. If neither of those is set, this flag has no effect. Advanced users only: misconfiguration can result in e.g. fetching a lower header value or missing a slot (caveats and worked examples in the annotated config example). Default: `false`. - `target_first_request_ms`: target time in the slot, in milliseconds, at which to send the first `get_header` request. - `frequency_get_header_ms`: frequency, in milliseconds, at which to send `get_header` requests. @@ -110,16 +110,7 @@ Under Docker, `commit-boost init` mounts each `file` path into the container at ### Bid streaming -:::info Unreleased -From v0.12.0-rc1, compared with v0.11.0: -- The relay's HTTP request runs alongside every stream. In v0.11.0 it runs only when the handshake fails with time left. -- The stream serves its latest valid bid. In v0.11.0 only the last bid is validated. -- A `204` answer to the handshake is no bid. In v0.11.0 it records `556` and retries over HTTP. -- A failed stream logs `stream failed`. In v0.11.0 it logs `stream failed, falling back to http get_header`, and only when it retries. -- A relay also listed with `get_header = "http"`, as the v0.11.0 docs advised, gets two HTTP requests per `get_header`, and after the Gloas fork only the first of the two serves ePBS; remove the `"http"` entry. -::: - -`get_header = "stream"` adds a websocket stream of bid updates to the relay's per-slot HTTP request. It is set per relay, on `[[relays]]` and `[[mux.relays]]` entries alike. [`examples/configs/pbs_bid_stream.toml`](https://github.com/Commit-Boost/commit-boost-client/blob/main/examples/configs/pbs_bid_stream.toml) streams from two default relays and two mux relays. +`get_header = "stream"` replaces the per-slot HTTP request with a websocket. It is set per relay, on `[[relays]]` and `[[mux.relays]]` entries alike. [`examples/configs/pbs_bid_stream.toml`](https://github.com/Commit-Boost/commit-boost-client/blob/main/examples/configs/pbs_bid_stream.toml) streams from two default relays and two mux relays. (unreleased, from v0.12.0-rc1) After the Gloas fork the same setting also streams [ePBS bids](./epbs.md#bid-streaming). @@ -131,29 +122,44 @@ For each `get_header` call PBS dials: ws(s):///eth/v1/builder/header_stream/{slot}/{parent_hash}/{pubkey} ``` -The scheme and host come from the relay's `url`: `https` becomes `wss`, `http` becomes `ws`, any other scheme fails at startup. The pubkey is dropped from the userinfo and `get_params` are appended as usual. The stream connects directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`. +The scheme and host come from the relay's `url`: `https` becomes `wss`, `http` becomes `ws`, any other scheme fails at startup. The pubkey is dropped from the userinfo and `get_params` are appended as usual. -The handshake carries what the HTTP request would: slot, parent hash and pubkey in the path, plus `Date-Milliseconds`, `X-Timeout-Ms`, `X-CommitBoost-Version` and every configured header. `X-Timeout-Ms` is the time left in the bid window minus half the TCP connect time, so the relay's last update arrives before the window closes. +The handshake carries what the HTTP request would: slot, parent hash and pubkey in the path, plus `Date-Milliseconds`, `X-Timeout-Ms`, `X-CommitBoost-Version` and every configured header. The stream needs the same API key as the HTTP path, sent on the handshake with the rest of `headers`. See [Relay API keys](#relay-api-keys). #### Bid window -Once connected, the relay sends one binary frame per bid update. PBS keeps the last 8 bids and, when the window ends or the relay closes the stream, returns the newest one that passes validation. Validation is the same as on the HTTP path, so `skip_sigverify`, `min_bid_eth` and `extra_validation_enabled` apply unchanged. Frames whose message type or fork byte cannot be read are skipped and counted. +Once connected, the relay sends one binary frame per bid update. PBS keeps the most recent update, and validates and returns it when the window ends or the relay closes the stream. Validation is the same as on the HTTP path, so `skip_sigverify`, `min_bid_eth` and `extra_validation_enabled` apply unchanged. Frames that do not parse as a bid are skipped and counted. The window is the deadline the HTTP path already computes: `timeout_get_header_ms`, capped by the time left until `late_in_slot_time_ms`, and capped again by the CL's `X-Timeout-Ms` if it sends one. An HTTP `get_header` normally returns early, while a stream is held to the end, so on a streaming relay `timeout_get_header_ms` sets how long PBS holds the CL's request. A CL request arriving at or after `late_in_slot_time_ms` still skips relays entirely and forces local building. -The timing-game options have no effect on the stream, which already delivers every update the relay produces. They apply to the HTTP request that runs alongside it. +The timing-game options have no effect on a stream. They schedule repeated HTTP requests, and the stream already delivers every update the relay produces. They do apply to the HTTP fallback. + +#### Fallback + +If the handshake fails, PBS falls back to a plain HTTP `get_header` at the relay's normal URL with whatever time is left in the window, and counts `cb_pbs_relay_stream_fallback_total`. That covers a refused connection, a DNS or TLS failure, and a relay that rejects the upgrade with an HTTP response. -#### HTTP request +There is no fallback if the handshake is still unanswered when the window ends (recorded as `555`), or if it fails too late to leave time for an HTTP request. There is also none once the stream is open. If it breaks before any bid arrives, that relay contributes no header for the slot (recorded as `556`). If a bid already arrived, PBS validates and returns it as usual. -PBS sends the relay its normal HTTP `get_header` alongside the stream. The stream's bid and the HTTP bid are separate candidates and the better one wins, so a failed stream leaves the HTTP bid in the auction. The `auction winner` log line names the winning `transport`. +On a high-latency connection, where the handshake risks timing out, you can add the relay a second time with `get_header = "http"` and a different `id`. PBS asks both entries every slot and keeps the better bid, so the HTTP entry still returns a header when the stream does not. -A stream fails when it cannot connect, when the handshake times out or the relay refuses it, when the connection drops before any bid arrives, or when none of the bids PBS kept passes validation. Commit-Boost logs each failed stream as `stream failed` with the reason, and counts it in `cb_pbs_relay_stream_fallback_total`. A drop after a bid arrived is not a failure: PBS validates the bids it kept as usual. A `204` answer to the handshake means the relay has no bid. +```toml +[[relays]] +id = "relay-1-stream" +url = "https://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@relay-1.xyz" +get_header = "stream" +headers = { X-Api-Key = "..." } + +id = "relay-1-http" +url = "https://0xa1cec75a3f0661e99299274182938151e8433c61a19222347ea1313d839229cb4ce4e3e5aa2bdeb71c8fcf1b084963c2@relay-1.xyz" +get_header = "http" +headers = { X-Api-Key = "..." } +``` -#### Stream metrics +#### Metrics -Stream and HTTP outcomes are recorded separately. `cb_pbs_relay_status_code_total` and `cb_pbs_relay_latency` carry `endpoint="get_header_stream"` for the stream and `endpoint="get_header"` for the HTTP request, and four `cb_pbs_relay_stream_*` series cover handshake latency, updates per window, unparseable frames and failed streams; [Metrics > Bid stream](./running/metrics.md#bid-stream) says what each value means. +Stream and fallback outcomes are recorded separately. `cb_pbs_relay_status_code_total` and `cb_pbs_relay_latency` carry `endpoint="get_header_stream"` for the stream and `endpoint="get_header"` for the fallback, and four `cb_pbs_relay_stream_*` series cover handshake latency, updates per window, unparseable frames and fallbacks. See [Metrics > Bid stream](./running/metrics.md#bid-stream). ### SSZ support diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 1f41165bb..500ba41d5 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -102,11 +102,11 @@ For each bid request to a streaming builder, Commit-Boost opens the stream and r Streaming makes bid requests slower. Commit-Boost replies to the beacon node once the stream has ended: when the builder closes it, when it fails, or at the beacon node's deadline less `proposer_deadline_buffer_ms`. An HTTP builder is answered as soon as it responds. `timeout_get_header_ms` does not shorten the wait. If the beacon node reports bid requests timing out on streaming builders, raise `proposer_deadline_buffer_ms`. -The stream [connects directly](./configuration.md#connection), without your proxy settings, while the HTTP request uses them. If your host reaches builders only through a proxy, keep them on `"http"`. +The stream connects directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`, while the HTTP fallback uses them. If your host reaches builders only through a proxy, keep them on `"http"`. ## Metrics -With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoint` labels `get_execution_payload_bid`, `submit_builder_preferences` and `submit_signed_beacon_block`, and a [streaming](#bid-streaming) builder's stream uses `get_execution_payload_bid_stream`. +With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoint` labels `get_execution_payload_bid`, `submit_builder_preferences` and `submit_signed_beacon_block`, and a [streaming](#bid-streaming) builder's stream uses `get_execution_payload_bid_stream`, with its own `cb_pbs_relay_epbs_stream_*` series ([Metrics > Bid stream](./running/metrics.md#bid-stream)). | Question | Series | |---|---| @@ -123,7 +123,7 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | | Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config) | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | -| Bid requests to a streaming builder log "stream failed" with "rejected with 404", and `cb_pbs_relay_stream_fallback_total{endpoint="get_execution_payload_bid_stream"}` rises | The builder does not serve the ePBS bid stream. Commit-Boost asks it over HTTP instead | Set `get_header = "http"` on its relay entry | +| Bid requests to a streaming builder log "stream failed" with "rejected with 404", and `cb_pbs_relay_epbs_stream_fallback_total` rises | The builder does not serve the ePBS bid stream. Commit-Boost asks it over HTTP instead | Set `get_header = "http"` on its relay entry | | Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | | Bid requests to a streaming builder log "stream failed" with "websocket timed out" or a connection error | Commit-Boost could not open the stream, for example because your network allows only proxied connections, which the stream does not use. A connection error with time left is retried over HTTP; a handshake that times out is no bid | Keep the builder on `get_header = "http"`, or allow direct connections to it | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/docs/get_started/running/metrics-catalog.md b/docs/docs/get_started/running/metrics-catalog.md index a771ca6a1..ff926b743 100644 --- a/docs/docs/get_started/running/metrics-catalog.md +++ b/docs/docs/get_started/running/metrics-catalog.md @@ -14,14 +14,18 @@ PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. Th | Metric name (wire) | Type | Labels | Description | |---|---|---|---| -| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (over HTTP a timeout, DNS or connection failure; on a stream, the bid window ran out before the handshake completed), and `"556"` for a bid-stream transport failure (connect failed, the stream broke before a bid, or the handshake was answered with a `2xx` other than `204`). (unreleased, from v0.12.0-rc1) A `204` answer to the handshake records `204`. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. | -| `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. Under the two stream endpoints the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | +| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (timeout, DNS or connection failure; on `get_header_stream`, only the deadline), and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). (unreleased, from v0.12.0-rc1) On `get_execution_payload_bid_stream`, a `204` answer to the handshake records `204`. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. | +| `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. Under `get_header_stream` and `get_execution_payload_bid_stream` the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | | `cb_pbs_relay_last_slot` | Gauge | `relay_id` | Latest slot for which a relay delivered a bid. Set to the current slot on each successful `get_header` bid and (unreleased, from v0.12.0-rc1) each ePBS bid served from that relay. | -| `cb_pbs_relay_header_value` | Gauge | `relay_id` | Value in gwei of the bid a relay delivered: for `get_header`, the bid's value converted from wei (รท 1e9). (unreleased, from v0.12.0-rc1) For a streaming relay, the better of its HTTP and stream bids. For ePBS, the returned bid's `value`, without its execution payment. | -| `cb_pbs_relay_stream_connect_latency` | Histogram | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Websocket handshake latency in seconds, for a relay with `get_header = "stream"`. Observed once per successful handshake; a handshake that fails, runs out the bid window, or is answered `204` records nothing here. Custom buckets from 5 ms to 1 s. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | -| `cb_pbs_relay_stream_updates` | Histogram | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Bid updates accepted on arrival per stream: on `get_header_stream`, every frame whose message type and fork byte can be read; on `get_execution_payload_bid_stream`, every bid that decodes. Observed once per stream that connected, including streams that received nothing; a handshake that fails, times out or is answered `204` records nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | -| `cb_pbs_relay_stream_invalid_frames_total` | Counter | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | Websocket frames whose message type or fork byte cannot be read; on the ePBS stream, also bids that fail to decode. Incremented only for windows that saw at least one, so the series is absent while zero. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | -| `cb_pbs_relay_stream_fallback_total` | Counter | `endpoint` (unreleased, from v0.12.0-rc1), `relay_id` | (unreleased, from v0.12.0-rc1; in v0.11.0, handshake failures retried over HTTP) Bid streams that failed while the relay's HTTP request ran alongside: a connection or handshake that failed or timed out, a stream that broke before a bid, or, on `get_header_stream`, a window in which every held bid was invalid. A `204` answer to the handshake is no bid and does not count. Counts every failed stream, whether or not the HTTP request returned a bid; the HTTP result is recorded under `endpoint="get_header"`. On `get_execution_payload_bid_stream`, which runs no HTTP request alongside, only a stream that failed to connect and was retried over HTTP counts, and the retry is recorded under `endpoint="get_execution_payload_bid"`. Endpoint values: `get_header_stream`, `get_execution_payload_bid_stream`. | +| `cb_pbs_relay_header_value` | Gauge | `relay_id` | Value in gwei of the bid a relay delivered: for `get_header`, the bid's value converted from wei (รท 1e9). (unreleased, from v0.12.0-rc1) For ePBS, the returned bid's `value`, without its execution payment. | +| `cb_pbs_relay_stream_connect_latency` | Histogram | `relay_id` | Websocket handshake latency in seconds, for a relay with `get_header = "stream"`. Observed once per successful handshake; a handshake that fails or runs out the bid window records nothing here. Custom buckets from 5 ms to 1 s. | +| `cb_pbs_relay_stream_updates` | Histogram | `relay_id` | Bid updates received per stream window. Observed once per `get_header` served by the stream, including windows that received nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. | +| `cb_pbs_relay_stream_invalid_frames_total` | Counter | `relay_id` | Websocket frames that could not be parsed as a bid. Incremented only for windows that saw at least one, so the series is absent while zero. | +| `cb_pbs_relay_stream_fallback_total` | Counter | `relay_id` | Stream attempts that fell back to a plain HTTP `get_header`. Only a handshake failure with bid window left to retry counts here; one that consumed the window shows on the status series alone. The fallback's own result is recorded under `endpoint="get_header"`. | +| `cb_pbs_relay_epbs_stream_connect_latency` | Histogram | `relay_id` | (unreleased, from v0.12.0-rc1) ePBS bid stream handshake latency in seconds. Observed once per successful handshake; a handshake that fails, runs out the deadline, or is answered `204` records nothing here. Custom buckets from 5 ms to 1 s. | +| `cb_pbs_relay_epbs_stream_updates` | Histogram | `relay_id` | (unreleased, from v0.12.0-rc1) Bids that decoded per ePBS bid stream. Observed once per stream that connected, including streams that received nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. | +| `cb_pbs_relay_epbs_stream_invalid_frames_total` | Counter | `relay_id` | (unreleased, from v0.12.0-rc1) ePBS bid stream frames whose message type or fork byte cannot be read, or whose bid does not decode. Incremented only for streams that saw at least one, so the series is absent while zero. | +| `cb_pbs_relay_epbs_stream_fallback_total` | Counter | `relay_id` | (unreleased, from v0.12.0-rc1) ePBS bid streams that fell back to an HTTP bid request. Only a stream that failed to connect with time left counts here; one that ran out the deadline or broke after connecting shows on the status series alone. The fallback's own result is recorded under `endpoint="get_execution_payload_bid"`. | | `cb_pbs_beacon_node_status_code_total` | Counter | `http_status_code`, `endpoint` | HTTP status code returned to the beacon node. Tracks what status codes the PBS returns for beacon node-facing requests. Endpoint values: `get_header`, `register_validator`, `submit_blinded_block`, `status`, `reload`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Error status codes (`502` for `NoResponse`/`NoPayload`, `500` for `Internal`) are set via `PbsClientError`. The handlers also record `406` directly when the request's `Accept` header offers no supported encoding, `204` when no bid is available on `get_header`, and `202` for accepted v2 `submit_blinded_block` requests. (unreleased, from v0.12.0-rc1) The ePBS endpoints record `200` for a bid, `204` for none, `202` for accepted preferences and for every signed block it forwards, `400` for a request Commit-Boost refuses, a builder's own `400` or `401`, `406` for a bid request whose `Accept` offers no supported encoding, `415` for an unsupported `Content-Type`, and `500` when no builder accepts the preferences. | | `cb_pbs_pbs_submit_block_v2_unsupported_total` | Counter | `relay_id` | Count of v2 `submit_blinded_block` requests a relay could not serve because it returned 404 on the v2 endpoint. A non-zero value means the relay does not support `submitBlindedBlockV2` and those blocks were not submitted via that relay. The double `pbs` in the wire name comes from the registry prefix plus the metric name `pbs_submit_block_v2_unsupported_total`. | diff --git a/docs/docs/get_started/running/metrics.md b/docs/docs/get_started/running/metrics.md index fc96a3516..acb484577 100644 --- a/docs/docs/get_started/running/metrics.md +++ b/docs/docs/get_started/running/metrics.md @@ -112,25 +112,15 @@ Once Grafana is running, you can [import](https://grafana.com/docs/grafana/lates ## Bid stream -:::info Unreleased -From v0.12.0-rc1, compared with v0.11.0: -- The `cb_pbs_relay_stream_*` series carry an `endpoint` label. In v0.11.0 they carry `relay_id` only. -- `cb_pbs_relay_stream_fallback_total` counts every failed stream. In v0.11.0 it counts handshake failures retried over HTTP. -- A handshake answered `204` records `204`. In v0.11.0 it records `556`. -- A streaming relay's HTTP request records under `endpoint="get_header"` on every request. In v0.11.0 it records only on a fallback. -::: - -For a relay with `get_header = "stream"` (see [Configuration > Bid streaming](../configuration.md#bid-streaming)), every series below is labeled by `relay_id` and `endpoint`. The stream's outcome and time-to-first-bid share the two general relay series under `endpoint="get_header_stream"`, and the HTTP request that runs alongside it keeps `endpoint="get_header"`. +For a relay with `get_header = "stream"` (see [Configuration > Bid streaming](../configuration.md#bid-streaming)), every series below is labelled by `relay_id`. The stream's outcome and time-to-first-bid share the two general relay series under `endpoint="get_header_stream"`, and the HTTP fallback keeps `endpoint="get_header"`. | Question | Series | |---|---| -| Is the stream serving bids? | `cb_pbs_relay_status_code_total{endpoint="get_header_stream"}`: `200` a bid was delivered, `204` no bid (the handshake was answered `204`, or the stream ended or the deadline passed without one), `555` the bid window ran out during the handshake, `556` a transport error (connect failed, the stream broke before a bid, or the handshake was answered with a `2xx` other than `204` instead of the upgrade), any other code the relay's own refusal of the upgrade | +| Is the stream serving bids? | `cb_pbs_relay_status_code_total{endpoint="get_header_stream"}`: `200` a bid was delivered, `204` connected but no bid before the deadline, `555` the bid window ran out during the handshake, `556` a transport error (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade), any other code the relay's own refusal of the upgrade | | How fast does the first bid arrive? | `cb_pbs_relay_latency{endpoint="get_header_stream"}` | -| Is the stream failing? | `cb_pbs_relay_stream_fallback_total{endpoint="get_header_stream"}`: [failed streams](../configuration.md#http-request), counted whether or not the HTTP request returned a bid. For a steady rate, the status series above and the `stream failed` log line give the reason | +| Is it falling back to HTTP? | `cb_pbs_relay_stream_fallback_total`: handshake failures that had bid window left to retry over HTTP. A steady rate means the relay is refusing or dropping the upgrade. The fallback's own results are under `endpoint="get_header"` | | Is the handshake slow? | `cb_pbs_relay_stream_connect_latency` | -| Is it actually streaming? | `cb_pbs_relay_stream_updates`: bids received per request. On `get_header_stream` every frame with a readable message type and fork byte counts, before its bid is checked. A healthy relay sends several; at most one per request means the stream connects but does not stream | -| Are the frames usable? | `cb_pbs_relay_stream_invalid_frames_total`: frames dropped on arrival because their message type or fork byte cannot be read, absent while zero. A bid that fails decoding or validation is not counted here | - -When the newest bid fails validation and an earlier one is served, the failure is logged, not counted. A window in which every kept bid fails counts as `200` on the status series, the same as an invalid bid over HTTP, and as one failed stream. If the beacon node drops the request before the deadline, no status is recorded. +| Is it actually streaming? | `cb_pbs_relay_stream_updates`: bid updates received per window. A healthy relay sends several; windows that carry at most one update mean the stream connects but does not stream | +| Are the frames usable? | `cb_pbs_relay_stream_invalid_frames_total`: frames that could not be parsed as a bid, absent while zero | -The [ePBS bid stream](../epbs.md#bid-streaming) (unreleased, from v0.12.0-rc1) reports the same series under `endpoint="get_execution_payload_bid_stream"`, and its HTTP fallback under `endpoint="get_execution_payload_bid"`. There `cb_pbs_relay_stream_fallback_total` counts only streams that failed to connect and were retried over HTTP. It decodes each bid as it arrives and does not validate it, so `cb_pbs_relay_stream_updates` counts only bids that decode, and `cb_pbs_relay_stream_invalid_frames_total` also counts those that do not. A stream that ends with only undecodable bids is no bid (`204`); if its connection drops instead, it records `556`. +The [ePBS bid stream](../epbs.md#bid-streaming) (unreleased, from v0.12.0-rc1) records its outcome and time-to-first-bid on the same two general series under `endpoint="get_execution_payload_bid_stream"`, and its HTTP fallback under `endpoint="get_execution_payload_bid"`. A handshake answered `204` records `204`, no bid. Its own series, labeled by `relay_id`, are `cb_pbs_relay_epbs_stream_connect_latency`, `cb_pbs_relay_epbs_stream_updates`, `cb_pbs_relay_epbs_stream_invalid_frames_total` and `cb_pbs_relay_epbs_stream_fallback_total`, the counterparts of the four above. It decodes each bid as it arrives and does not validate it, so `cb_pbs_relay_epbs_stream_updates` counts bids that decode and `cb_pbs_relay_epbs_stream_invalid_frames_total` also counts those that do not. A stream that ends with only undecodable bids is no bid (`204`); if its connection drops instead, it records `556`. diff --git a/tests/src/mock_ws_relay.rs b/tests/src/mock_ws_relay.rs index 997931baa..c45f9aff7 100644 --- a/tests/src/mock_ws_relay.rs +++ b/tests/src/mock_ws_relay.rs @@ -16,7 +16,6 @@ use cb_common::{ RelayClient, }, types::{BlsSecretKey, Chain}, - utils::utcnow_ms, wire::CONSENSUS_VERSION_HEADER, }; use futures::SinkExt; @@ -59,14 +58,13 @@ pub struct StreamRequest { pub api_key: Option, pub consensus_version: Option, pub request_auth: Option, - /// When the relay accepted the connection, unix ms - pub accepted_at_ms: u64, } pub struct MockWsRelayState { pub chain: Chain, pub signer: BlsSecretKey, - /// One frame pushed per value, in order + /// One frame pushed per value, in order. The last one is what PBS must + /// return. bid_values: Vec, /// One frame per `(value, execution_payment)` in gwei, on the ePBS bid /// stream @@ -82,15 +80,8 @@ pub struct MockWsRelayState { /// Precede each bid with frames PBS can't parse, which it must skip rather /// than treat as the end of the stream unknown_frames: bool, - /// Make this many of the last bids fail validation: signed for another - /// parent hash, or on the ePBS stream cut short so they do not decode + /// Cut this many of the last ePBS bids short, so they do not decode invalid_last_bids: usize, - /// Wait this long before answering the handshake, as a relay whose - /// handshake outlasts PBS's budget - handshake_delay: Duration, - /// Drop the connection after the last bid, with no close frame, as a - /// stream that breaks - abort_after_bids: bool, handshake_attempts: AtomicU64, received_connections: AtomicU64, last_request: Mutex>, @@ -108,8 +99,6 @@ impl MockWsRelayState { hold_open: false, unknown_frames: false, invalid_last_bids: 0, - handshake_delay: Duration::ZERO, - abort_after_bids: false, handshake_attempts: AtomicU64::new(0), received_connections: AtomicU64::new(0), last_request: Mutex::new(None), @@ -144,14 +133,6 @@ impl MockWsRelayState { Self { invalid_last_bids, ..self } } - pub fn with_handshake_delay(self, handshake_delay: Duration) -> Self { - Self { handshake_delay, ..self } - } - - pub fn abort_after_bids(self) -> Self { - Self { abort_after_bids: true, ..self } - } - /// Connections that reached the stream, counted before the handshake, so /// one that never completes still shows pub fn handshake_attempts(&self) -> u64 { @@ -185,14 +166,10 @@ pub async fn start_mock_ws_relay_service( // The handshake callback's Err type is fixed by tungstenite #[allow(clippy::result_large_err)] async fn serve_stream(state: Arc, stream: TcpStream) -> eyre::Result<()> { - let accepted_at_ms = utcnow_ms(); state.handshake_attempts.fetch_add(1, Ordering::Relaxed); - if !state.handshake_delay.is_zero() { - tokio::time::sleep(state.handshake_delay).await; - } let mut request = None; let mut ws = accept_hdr_async(stream, |req: &Request, res: Response| { - request = parse_request(req, accepted_at_ms); + request = parse_request(req); if state.no_epbs_stream && request.as_ref().is_some_and(|req| req.parent_root.is_some()) { let mut not_found = ErrorResponse::new(None); *not_found.status_mut() = StatusCode::NOT_FOUND; @@ -225,10 +202,6 @@ async fn serve_stream(state: Arc, stream: TcpStream) -> eyre:: } } - if state.abort_after_bids { - return Ok(()); - } - if state.hold_open { // Never resolves: PBS has to cut the stream off at its own deadline std::future::pending::<()>().await; @@ -263,12 +236,11 @@ fn bid_frame( (FORK_GLOAS, bid) } None => { - let parent_hash = if invalid { B256::repeat_byte(0xee) } else { request.parent_hash }; let bid = mock_signed_builder_bid( state.chain, &state.signer, request.slot, - parent_hash, + request.parent_hash, state.bid_values[i], ); (FORK_FULU, bid.as_ssz_bytes()) @@ -283,7 +255,7 @@ fn bid_frame( /// The request is the handshake: the fixed stream path followed by /// `/{slot}/{parent_hash}/{pubkey}` (with `/{parent_root}` before the pubkey on /// the ePBS bid stream), plus the same headers the HTTP path sends. -fn parse_request(req: &Request, accepted_at_ms: u64) -> Option { +fn parse_request(req: &Request) -> Option { let path = req.uri().path(); let v1 = BuilderApiVersion::V1.path(); let (rest, epbs) = @@ -309,7 +281,6 @@ fn parse_request(req: &Request, accepted_at_ms: u64) -> Option { api_key: header(req, HEADER_API_KEY), consensus_version: header(req, CONSENSUS_VERSION_HEADER), request_auth: header(req, HEADER_REQUEST_AUTH), - accepted_at_ms, }) } diff --git a/tests/tests/pbs_get_header_ws.rs b/tests/tests/pbs_get_header_ws.rs index eead45035..abf9a7f45 100644 --- a/tests/tests/pbs_get_header_ws.rs +++ b/tests/tests/pbs_get_header_ws.rs @@ -10,14 +10,14 @@ use cb_common::{ signature::sign_builder_root, signer::random_secret, types::{BlsPublicKeyBytes, BlsSecretKey, Chain, KnownChain}, - utils::{timestamp_of_slot_start_sec, utcnow_ms}, + utils::timestamp_of_slot_start_sec, wire::EncodingType, }; use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use cb_tests::{ mock_relay::{MockRelayState, start_mock_relay_service_with_listener}, mock_validator::MockValidator, - mock_ws_relay::{MockWsRelayState, start_mock_dual_relay, start_mock_ws_relay_service}, + mock_ws_relay::{MockWsRelayState, start_mock_ws_relay_service}, utils::{ RELAY_API_KEY, generate_mock_relay, generate_mock_stream_relay, generate_mock_stream_relay_with_timing_games, get_free_listener, get_pbs_config, @@ -61,7 +61,8 @@ async fn start_pbs( let config = to_pbs_config(chain, pbs_config, relays); let state = PbsState::new(config, PathBuf::new()); - tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, listener)); + drop(listener); + tokio::spawn(PbsService::run::<(), DefaultBuilderApi>(state)); // leave some time to start servers tokio::time::sleep(Duration::from_millis(100)).await; @@ -131,39 +132,6 @@ async fn test_get_header_ws_returns_latest_bid() -> Result<()> { Ok(()) } -/// Bids that fail validation do not displace the valid one before them, while -/// it is among the newest 8 the stream holds -#[tokio::test] -async fn test_get_header_ws_returns_latest_valid_bid() -> Result<()> { - setup_test_env(); - let signer = random_secret(); - let chain = Chain::Hoodi; - - for (invalid_bids, served) in [(7, Some(50)), (8, None)] { - let mut bid_values = vec![U256::from(50)]; - bid_values.resize(invalid_bids + 1, U256::from(60)); - let (_relay_state, relay) = start_stream_relay( - MockWsRelayState::new(chain, signer.clone()) - .with_bid_values(bid_values) - .with_invalid_last_bids(invalid_bids), - signer.public_key(), - ) - .await?; - - let validator = start_pbs(chain, vec![relay], 1_000).await?; - - let (code, res) = get_header_json(&validator).await?; - match served { - Some(value) => { - assert_eq!(code, StatusCode::OK, "{invalid_bids} invalid bids"); - assert_bid(&res.unwrap(), chain, &signer, U256::from(value)); - } - None => assert_eq!(code, StatusCode::NO_CONTENT, "{invalid_bids} invalid bids"), - } - } - Ok(()) -} - /// Frames PBS can't parse are skipped, not treated as the end of the stream: /// the updates after them still count. #[tokio::test] @@ -226,7 +194,7 @@ async fn test_get_header_ws_handshake_carries_request() -> Result<()> { start_stream_relay(MockWsRelayState::new(chain, signer.clone()), pubkey).await?; let validator = start_pbs(chain, vec![relay], timeout_ms).await?; - let sent_at = utcnow_ms(); + let sent_at = cb_common::utils::utcnow_ms(); let (code, _) = get_header_json(&validator).await?; assert_eq!(code, StatusCode::OK); @@ -235,13 +203,8 @@ async fn test_get_header_ws_handshake_carries_request() -> Result<()> { assert_eq!(request.parent_hash, B256::ZERO); assert!(request.validator_pubkey.starts_with("0x")); - // No timeout header from the caller, so PBS asks for its own budget, less - // what the connect took - let relay_timeout_ms = request.timeout_ms.expect("missing timeout header"); - assert!( - (timeout_ms - 100..=timeout_ms).contains(&relay_timeout_ms), - "timeout header {relay_timeout_ms}ms" - ); + // No timeout header from the caller, so PBS passes its own budget through + assert_eq!(request.timeout_ms, Some(timeout_ms)); let start_time_ms = request.start_time_ms.expect("missing start time header"); assert!((sent_at..sent_at + timeout_ms).contains(&start_time_ms)); @@ -328,10 +291,38 @@ async fn test_get_header_ws_unreachable_relay_returns_204() -> Result<()> { Ok(()) } -/// The HTTP request raced alongside the stream is the relay's normal -/// get_header, timing games included. +/// A relay configured to stream that rejects the handshake โ€” here by not +/// serving the stream route at all โ€” is still asked over HTTP, so its bid stays +/// in the auction instead of the relay silently dropping out. #[tokio::test] -async fn test_get_header_ws_race_http_runs_timing_games() -> Result<()> { +async fn test_get_header_ws_falls_back_to_http() -> Result<()> { + setup_test_env(); + let signer = random_secret(); + let pubkey = signer.public_key(); + let chain = Chain::Hoodi; + + let listener = get_free_listener().await; + let port = listener.local_addr()?.port(); + let relay_state = + Arc::new(MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(42))); + tokio::spawn(start_mock_relay_service_with_listener(relay_state.clone(), listener)); + + let relay = generate_mock_stream_relay(port, pubkey)?; + let validator = start_pbs(chain, vec![relay], 1_000).await?; + + let (code, res) = get_header_json(&validator).await?; + assert_eq!(code, StatusCode::OK); + assert_bid(&res.unwrap(), chain, &signer, U256::from(42)); + + // Timing games are off for this relay, so the fallback is a single request + assert_eq!(relay_state.received_get_header(), 1); + Ok(()) +} + +/// The handshake can fail early in the slot, so the fallback is a normal http +/// get_header and still plays this relay's timing games. +#[tokio::test] +async fn test_get_header_ws_fallback_runs_timing_games() -> Result<()> { setup_test_env(); let signer = random_secret(); let pubkey = signer.public_key(); @@ -352,7 +343,7 @@ async fn test_get_header_ws_race_http_runs_timing_games() -> Result<()> { assert_bid(&res.unwrap(), chain, &signer, U256::from(10)); let n_requests = relay_state.received_get_header(); - assert!(n_requests > 1, "HTTP skipped the timing games loop: {n_requests} requests"); + assert!(n_requests > 1, "fallback skipped the timing games loop: {n_requests} requests"); Ok(()) } @@ -388,76 +379,3 @@ async fn test_get_header_ws_wins_auction_against_http() -> Result<()> { assert_eq!(stream_state.received_connections(), 1); Ok(()) } - -/// A handshake that outlasts the budget: the HTTP request raced alongside it -/// supplies the bid. -#[tokio::test] -async fn test_get_header_ws_race_handshake_timeout_http_bid_serves() -> Result<()> { - setup_test_env(); - let signer = random_secret(); - let chain = Chain::Hoodi; - let (http_state, ws_state, relay) = start_mock_dual_relay( - MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(42)), - MockWsRelayState::new(chain, signer.clone()).with_handshake_delay(Duration::from_secs(2)), - ) - .await?; - let validator = start_pbs(chain, vec![relay], 300).await?; - - let (code, res) = get_header_json(&validator).await?; - assert_eq!(code, StatusCode::OK); - assert_bid(&res.unwrap(), chain, &signer, U256::from(42)); - assert_eq!(http_state.received_get_header(), 1); - assert_eq!(ws_state.handshake_attempts(), 1); - assert_eq!(ws_state.received_connections(), 0, "the handshake never completed"); - Ok(()) -} - -/// A stream that breaks after a bid keeps that bid, here above the HTTP one. -#[tokio::test] -async fn test_get_header_ws_race_stream_break_keeps_held_bid() -> Result<()> { - setup_test_env(); - let signer = random_secret(); - let chain = Chain::Hoodi; - let (_, ws_state, relay) = start_mock_dual_relay( - MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(10)), - MockWsRelayState::new(chain, signer.clone()) - .with_bid_values(vec![U256::from(50)]) - .abort_after_bids(), - ) - .await?; - let validator = start_pbs(chain, vec![relay], 1_000).await?; - - let (code, res) = get_header_json(&validator).await?; - assert_eq!(code, StatusCode::OK); - assert_bid(&res.unwrap(), chain, &signer, U256::from(50)); - assert_eq!(ws_state.received_connections(), 1); - Ok(()) -} - -/// The higher candidate wins, the stream's being its latest bid, not its -/// highest; a stream with no bid leaves the HTTP one. -#[tokio::test] -async fn test_get_header_ws_race_higher_bid_wins() -> Result<()> { - setup_test_env(); - let signer = random_secret(); - let chain = Chain::Hoodi; - - for (http_bid, stream_bids, served) in - [(10, vec![20, 50], 50), (60, vec![70, 50], 60), (42, vec![], 42)] - { - let (http_state, ws_state, relay) = start_mock_dual_relay( - MockRelayState::new(chain, signer.clone()).with_bid_value(U256::from(http_bid)), - MockWsRelayState::new(chain, signer.clone()) - .with_bid_values(stream_bids.into_iter().map(U256::from).collect()), - ) - .await?; - let validator = start_pbs(chain, vec![relay], 1_000).await?; - - let (code, res) = get_header_json(&validator).await?; - assert_eq!(code, StatusCode::OK); - assert_bid(&res.unwrap(), chain, &signer, U256::from(served)); - assert_eq!(http_state.received_get_header(), 1); - assert_eq!(ws_state.received_connections(), 1); - } - Ok(()) -} From 94a524c4e88453a9788822196478d6e2f8880b67 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 13 Oct 2026 14:04:20 -0700 Subject: [PATCH 23/24] refactor(pbs): fold the ePBS bid stream reader into its route, and review fixes The bid stream reader was generic for a PBS caller that no longer uses it, so it moves into execution_payload_bid_ws.rs in the shape of main's get_header_ws.rs, and bid_stream.rs goes. - A handshake answered 204 is a failed connect, as on get_header in v0.11.0: it is retried over HTTP when time is left. - A frame whose fork byte is not Gloas is an invalid frame, so it cannot replace an earlier bid. - The fallback logs once per path, with the time it hands the HTTP request. - decode_ssz_bid returns the bid alone, so main's HTTP decoding is unchanged around it. - Tests pin the fallback's deadline, a bid served from a stream that then breaks, and the metrics each path records; the two HTTP fallback integration tests become one. - Docs: a builder listed twice serves ePBS from its first entry only, the 555 and 556 codes on the bid stream, and wording. --- crates/common/src/pbs/relay.rs | 3 +- crates/pbs/src/bid_stream.rs | 373 ----------------- crates/pbs/src/lib.rs | 1 - .../pbs/src/routes/execution_payload_bid.rs | 183 ++++++--- .../src/routes/execution_payload_bid_ws.rs | 384 ++++++++++++++++-- docs/docs/get_started/configuration.md | 2 +- docs/docs/get_started/epbs.md | 10 +- .../get_started/running/metrics-catalog.md | 4 +- docs/docs/get_started/running/metrics.md | 2 +- .../tests/pbs_get_execution_payload_bid_ws.rs | 62 ++- 10 files changed, 513 insertions(+), 511 deletions(-) delete mode 100644 crates/pbs/src/bid_stream.rs diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index 7b0d8b815..f6878af83 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -87,8 +87,7 @@ pub struct RelayClient { pub id: Arc, /// HTTP client to send requests pub client: reqwest::Client, - /// Base url of the get_header stream, whose origin the ePBS bid stream - /// shares. `Some` only when the relay streams. + /// Base url of the get_header stream, `Some` only when the relay streams. stream_url: Option, /// Baseline headers for the bid stream handshakes. stream_headers: Arc, diff --git a/crates/pbs/src/bid_stream.rs b/crates/pbs/src/bid_stream.rs deleted file mode 100644 index 571eefb9a..000000000 --- a/crates/pbs/src/bid_stream.rs +++ /dev/null @@ -1,373 +0,0 @@ -//! The ePBS bid stream of a relay configured with `get_header = "stream"`. -//! One connection per request, dropped at the deadline. The handshake carries -//! the request, and the relay replies with one binary frame per bid update: -//! -//! ```text -//! u8 message type -//! u8 fork -//! .. SSZ bid -//! ``` - -use std::{ - sync::{Arc, OnceLock}, - time::Duration, -}; - -use axum::http::{HeaderMap, HeaderValue, Request, header::USER_AGENT}; -use cb_common::{ - pbs::{ForkName, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, error::PbsError}, - utils::utcnow_ms, -}; -use futures::StreamExt; -use reqwest::StatusCode; -use rustls::{ClientConfig, RootCertStore, crypto::aws_lc_rs}; -use tokio::{ - net::TcpStream, - time::{Instant, sleep_until, timeout_at}, -}; -use tokio_tungstenite::{ - Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config, - tungstenite::{ - Bytes, Error as WsError, Message, client::IntoClientRequest, protocol::WebSocketConfig, - }, -}; -use tracing::{debug, warn}; -use url::Url; - -use crate::{ - constants::{ - GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, - TIMEOUT_ERROR_STATUS, TRANSPORT_ERROR_STATUS, - }, - metrics::{ - RELAY_EPBS_STREAM_CONNECT_LATENCY, RELAY_EPBS_STREAM_INVALID_FRAMES, - RELAY_EPBS_STREAM_UPDATES, RELAY_LATENCY, - }, -}; - -/// Frame prefix: message type + fork. -const FRAME_PREFIX_LEN: usize = 2; - -const MSG_BID: u8 = 0x01; - -/// One bid update, its bid still SSZ-encoded -pub(crate) struct Frame { - pub(crate) fork: ForkName, - pub(crate) bid: Bytes, -} - -/// What a stream held when its window closed -pub(crate) struct Held { - /// The newest frame the parse closure accepted - pub(crate) latest: Option, - /// Frames the parse closure accepted - pub(crate) updates: usize, - pub(crate) connect_latency: Duration, - pub(crate) first_frame_latency: Option, - pub(crate) invalid_frames: usize, -} - -/// The handshake for `url`, with the caller's user agent and the relay's -/// headers -pub(crate) fn handshake_request( - url: &Url, - relay: &RelayClient, - send_headers: &HeaderMap, -) -> Result, PbsError> { - let mut request = url - .as_str() - .into_client_request() - .map_err(|err| PbsError::WebSocketConnect(format!("invalid ws url: {err}")))?; - - let headers = request.headers_mut(); - if let Some(user_agent) = send_headers.get(USER_AGENT) { - headers.insert(USER_AGENT, user_agent.clone()); - } - - for (key, value) in relay.stream_headers() { - headers.insert(key, value.clone()); - } - - Ok(request) -} - -/// Holds the newest frame `parse` accepts until `deadline` or the end of the -/// stream. An error carries the status the failure records under. -pub(crate) async fn read_bid_stream( - request: Request<()>, - deadline: Instant, - relay: &RelayClient, - mut parse: impl FnMut(Frame) -> Result, -) -> Result, (StatusCode, PbsError)> { - let endpoint = GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG; - let start_request = Instant::now(); - let mut stream = match timeout_at(deadline, connect(request, deadline)).await { - Ok(Ok(stream)) => stream, - // A relay with no bid can answer the handshake as it answers get_header - Ok(Err(WsError::Http(res))) if res.status() == StatusCode::NO_CONTENT => { - return Ok(Held { - latest: None, - updates: 0, - connect_latency: start_request.elapsed(), - first_frame_latency: None, - invalid_frames: 0, - }); - } - Ok(Err(err)) => return Err(connect_failed(&err)), - Err(_) => return Err((TIMEOUT_ERROR_STATUS, PbsError::WebSocketTimeout)), - }; - let connect_latency = start_request.elapsed(); - RELAY_EPBS_STREAM_CONNECT_LATENCY - .with_label_values(&[relay.id.as_str()]) - .observe(connect_latency.as_secs_f64()); - debug!(relay_id = relay.id.as_ref(), endpoint, ?connect_latency, "ws connected"); - - let timer = sleep_until(deadline); - tokio::pin!(timer); - - let mut latest = None; - let mut updates = 0usize; - let mut first_frame_latency = None; - let mut invalid_frames = 0usize; - let mut stream_error = None; - - loop { - let message = tokio::select! { - biased; - _ = &mut timer => break, - message = stream.next() => message, - }; - - let payload = match message { - Some(Ok(Message::Binary(payload))) => payload, - Some(Ok(Message::Close(_))) | None => break, - Some(Ok(_)) => continue, - Some(Err(err)) => { - debug!(relay_id = relay.id.as_ref(), %err, "ws stream error"); - stream_error = Some(PbsError::WebSocket(format!("stream error: {err}"))); - break; - } - }; - - match parse_frame(payload).and_then(&mut parse) { - Ok(frame) => { - first_frame_latency.get_or_insert_with(|| start_request.elapsed()); - updates += 1; - latest = Some(frame); - } - Err(err) => { - invalid_frames += 1; - if invalid_frames == 1 { - warn!(relay_id = relay.id.as_ref(), %err, "invalid ws frame, skipping"); - } - } - } - } - - drop(stream); - - RELAY_EPBS_STREAM_UPDATES.with_label_values(&[relay.id.as_str()]).observe(updates as f64); - if invalid_frames > 0 { - RELAY_EPBS_STREAM_INVALID_FRAMES - .with_label_values(&[relay.id.as_str()]) - .inc_by(invalid_frames as u64); - } - - if let Some(latency) = first_frame_latency { - RELAY_LATENCY.with_label_values(&[endpoint, &relay.id]).observe(latency.as_secs_f64()); - } else if let Some(err) = stream_error { - return Err((TRANSPORT_ERROR_STATUS, err)); - } - - Ok(Held { latest, updates, connect_latency, first_frame_latency, invalid_frames }) -} - -/// Opens the stream with the timing headers stamped now. As over HTTP, -/// `X-Timeout-Ms` is the time left to `deadline`; the trip back is the relay's. -async fn connect( - mut request: Request<()>, - deadline: Instant, -) -> Result>, WsError> { - let timeout_ms = deadline.saturating_duration_since(Instant::now()).as_millis() as u64; - let headers = request.headers_mut(); - headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); - headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); - - let config = WebSocketConfig::default() - .max_message_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)) - .max_frame_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)); - let connector = Connector::Rustls(tls_config().clone()); - let (stream, _) = - connect_async_tls_with_config(request, Some(config), true, Some(connector)).await?; - - Ok(stream) -} - -/// A relay rejecting the handshake puts the reason in the body, which -/// tungstenite's own `Display` drops. The body is only what arrived alongside -/// the headers, so it can be partial or empty. -fn connect_failed(err: &WsError) -> (StatusCode, PbsError) { - let WsError::Http(res) = err else { - return (TRANSPORT_ERROR_STATUS, PbsError::WebSocketConnect(err.to_string())); - }; - - let code = res.status(); - let body = res.body().as_deref().unwrap_or_default(); - let msg = if body.is_empty() { - format!("rejected with {code}") - } else { - format!("rejected with {code}: {}", String::from_utf8_lossy(body)) - }; - - // A 2xx handshake answer is a failed connect, not a delivered bid - let code = if code.is_success() { TRANSPORT_ERROR_STATUS } else { code }; - - (code, PbsError::WebSocketConnect(msg)) -} - -fn fork_from_wire(byte: u8) -> Option { - // TODO @nina: I don't see a point of extending a u8 for supporting older forks - // we could rotate these instead, i.e. 0 becomes Heze, etc - Some(match byte { - 0 => ForkName::Base, - 1 => ForkName::Altair, - 2 => ForkName::Bellatrix, - 3 => ForkName::Capella, - 4 => ForkName::Deneb, - 5 => ForkName::Electra, - 6 => ForkName::Fulu, - 7 => ForkName::Gloas, - 8 => ForkName::Heze, - _ => return None, - }) -} - -fn parse_frame(payload: Bytes) -> Result { - let &[msg_type, fork_byte] = payload - .first_chunk::() - .ok_or_else(|| PbsError::WebSocket(format!("frame too short: {} bytes", payload.len())))?; - - if msg_type != MSG_BID { - return Err(PbsError::WebSocket(format!("unknown message type: {msg_type}"))); - } - - let fork = fork_from_wire(fork_byte) - .ok_or_else(|| PbsError::WebSocket(format!("unknown fork: {fork_byte}")))?; - - Ok(Frame { fork, bid: payload.slice(FRAME_PREFIX_LEN..) }) -} - -/// One TLS config for every stream connection. Left to tokio-tungstenite it is -/// rebuilt per connect, which reparses the root store and, worse, gives each -/// connection its own session cache: every slot then pays a full handshake -/// instead of a resumed one. The provider is named explicitly because rustls is -/// built with both `ring` and `aws-lc-rs` here, so the default builder needs a -/// process-wide install to pick one. -fn tls_config() -> &'static Arc { - static CONFIG: OnceLock> = OnceLock::new(); - CONFIG.get_or_init(|| { - let mut roots = RootCertStore::empty(); - roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - - Arc::new( - ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) - .with_safe_default_protocol_versions() - .expect("aws-lc-rs supports tls 1.2 and 1.3") - .with_root_certificates(roots) - .with_no_client_auth(), - ) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn bid_frame(fork_byte: u8, bid: &[u8]) -> Bytes { - let mut frame = vec![MSG_BID, fork_byte]; - frame.extend_from_slice(bid); - Bytes::from(frame) - } - - #[test] - fn test_fork_from_wire_covers_all_forks() { - let forks = ForkName::list_all(); - - for (byte, fork) in forks.iter().enumerate() { - if *fork <= ForkName::Base { - continue; - } - assert_eq!(fork_from_wire(byte as u8), Some(*fork), "fork {fork} unmapped"); - } - - assert_eq!(fork_from_wire(forks.len() as u8), None); - } - - #[test] - fn test_parse_frame() { - assert!(matches!( - parse_frame(bid_frame(6, &[1, 2, 3])), - Ok(Frame { fork: ForkName::Fulu, bid }) if bid.as_ref() == [1, 2, 3] - )); - - // Empty bid payload is well-formed at this layer, SSZ decoding rejects it - assert!(matches!(parse_frame(bid_frame(6, &[])), Ok(Frame { fork: ForkName::Fulu, .. }))); - - for bad in [ - // Truncated prefix - Bytes::from_static(&[]), - Bytes::from_static(&[MSG_BID]), - // Unknown fork - bid_frame(0xff, &[1]), - // Unknown message type - Bytes::from_static(&[0xff, 6]), - ] { - assert!(matches!(parse_frame(bad), Err(PbsError::WebSocket(_)))); - } - } - - #[test] - fn test_connect_failed_carries_relay_reason() { - let rejected = axum::http::Response::builder() - .status(401) - .body(Some(b"api key not registered".to_vec())) - .unwrap(); - - let (status, err) = connect_failed(&WsError::Http(Box::new(rejected))); - assert_eq!(status, StatusCode::UNAUTHORIZED); - let PbsError::WebSocketConnect(msg) = err else { panic!("wrong outcome") }; - assert!(msg.contains("401"), "{msg}"); - assert!(msg.contains("api key not registered"), "{msg}"); - - // No body to add, and transport failures keep tungstenite's own message - let empty = axum::http::Response::builder().status(404).body(None).unwrap(); - let (status, _) = connect_failed(&WsError::Http(Box::new(empty))); - assert_eq!(status, StatusCode::NOT_FOUND); - - let (status, err) = connect_failed(&WsError::ConnectionClosed); - assert_eq!(status, TRANSPORT_ERROR_STATUS); - assert!(matches!(err, PbsError::WebSocketConnect(_))); - } - - // A url pointing at a plain http endpoint answers the handshake 200. That - // is the code the stream series uses for a delivered bid, so a failed - // handshake must never carry it. - #[test] - fn test_connect_failed_never_reports_a_success_code() { - for code in [200u16, 299] { - let answered = axum::http::Response::builder().status(code).body(None).unwrap(); - let (status, err) = connect_failed(&WsError::Http(Box::new(answered))); - assert_eq!( - status, TRANSPORT_ERROR_STATUS, - "handshake answered {code} counted as a served stream" - ); - let PbsError::WebSocketConnect(msg) = err else { panic!("wrong outcome") }; - assert!(msg.contains(&code.to_string()), "{msg}"); - } - - // A relay's own rejection code still reaches the series unchanged - let moved = axum::http::Response::builder().status(302).body(None).unwrap(); - let (status, _) = connect_failed(&WsError::Http(Box::new(moved))); - assert_eq!(status, StatusCode::FOUND); - } -} diff --git a/crates/pbs/src/lib.rs b/crates/pbs/src/lib.rs index 19acfc2f0..9c04224a8 100644 --- a/crates/pbs/src/lib.rs +++ b/crates/pbs/src/lib.rs @@ -1,5 +1,4 @@ mod api; -mod bid_stream; mod constants; mod dial; mod error; diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index 661f2ddb6..cdd09b618 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -84,9 +84,9 @@ pub async fn handle_get_execution_payload_bid( /// A relay's bid with the body and encoding it arrived in, so a beacon node /// that asks for the same encoding gets the relay's bytes unchanged pub(crate) struct RelayBid { - bid: GetExecutionPayloadBidResponse, - body: Bytes, - encoding: EncodingType, + pub(super) bid: GetExecutionPayloadBidResponse, + pub(super) body: Bytes, + pub(super) encoding: EncodingType, } fn encode_bid_response( @@ -209,9 +209,9 @@ pub async fn get_execution_payload_bid( } } -/// The bid over the relay's stream. As on get_header, only a stream that fails -/// to connect is retried over HTTP, with the time it left; any other failure is -/// no bid. +/// The bid over the relay's stream. As on get_header, a stream that fails to +/// connect, a handshake answered with anything but the upgrade included, is +/// retried over HTTP with the time left; any other failure is no bid. async fn stream_execution_payload_bid( params: GetExecutionPayloadBidParams, body: Bytes, @@ -221,25 +221,25 @@ async fn stream_execution_payload_bid( timeout_ms: u64, ) -> Result, PbsError> { let started = Instant::now(); - let err = - match get_execution_payload_bid_ws(&body, &relay, &send_headers, url, timeout_ms).await { - Err(err @ PbsError::WebSocketConnect(_)) => err, - Err(err) => { - warn!(relay_id = relay.id.as_ref(), %err, "stream failed"); - return Ok(None); - } - res => return res, - }; - + let res = get_execution_payload_bid_ws(&body, &relay, &send_headers, url, timeout_ms).await; let timeout_left_ms = timeout_ms.saturating_sub(started.elapsed().as_millis() as u64); - if timeout_left_ms == 0 { - warn!(relay_id = relay.id.as_ref(), %err, "stream failed"); - return Ok(None); + match res { + Err(err @ PbsError::WebSocketConnect(_)) if timeout_left_ms > 0 => { + RELAY_EPBS_STREAM_FALLBACK.with_label_values(&[relay.id.as_str()]).inc(); + warn!( + relay_id = relay.id.as_ref(), + %err, + timeout_left_ms, + "stream failed, falling back to http" + ); + send_get_execution_payload_bid(params, body, relay, send_headers, timeout_left_ms).await + } + Err(err) => { + warn!(relay_id = relay.id.as_ref(), %err, "stream failed"); + Ok(None) + } + res => res, } - - RELAY_EPBS_STREAM_FALLBACK.with_label_values(&[relay.id.as_str()]).inc(); - warn!(relay_id = relay.id.as_ref(), %err, timeout_left_ms, "stream failed, falling back to http"); - send_get_execution_payload_bid(params, body, relay, send_headers, timeout_left_ms).await } /// Milliseconds left until `Date-Milliseconds + X-Timeout-Ms`, but no more than @@ -313,17 +313,15 @@ async fn send_get_execution_payload_bid( return Ok(None); } - let relay_bid = match content_type { + let bid = match content_type { EncodingType::Json => { - let bid = - serde_json::from_slice(&response_bytes).map_err(|err| PbsError::JsonDecode { - err, - raw: String::from_utf8_lossy( - &response_bytes[..response_bytes.len().min(MAX_SIZE_DEFAULT)], - ) - .into_owned(), - })?; - RelayBid { bid, body: response_bytes.into(), encoding: EncodingType::Json } + serde_json::from_slice(&response_bytes).map_err(|err| PbsError::JsonDecode { + err, + raw: String::from_utf8_lossy( + &response_bytes[..response_bytes.len().min(MAX_SIZE_DEFAULT)], + ) + .into_owned(), + })? } EncodingType::Ssz => { // SSZ requires the fork from Eth-Consensus-Version; its absence is a @@ -333,19 +331,21 @@ async fn send_get_execution_payload_bid( .to_string(), code: code.as_u16(), })?; - decode_ssz_bid(response_bytes.into(), fork)? + decode_ssz_bid(&response_bytes, fork)? } }; - Ok(Some(relay_bid)) + Ok(Some(RelayBid { bid, body: response_bytes.into(), encoding: content_type })) } -pub(super) fn decode_ssz_bid(body: Bytes, fork: ForkName) -> Result { - let data = SignedExecutionPayloadBid::from_ssz_bytes(&body).map_err(|err| { +pub(super) fn decode_ssz_bid( + bytes: &[u8], + fork: ForkName, +) -> Result { + let data = SignedExecutionPayloadBid::from_ssz_bytes(bytes).map_err(|err| { PbsError::SSZDecode { err: format!("error decoding relay payload: {err:?}"), fork } })?; - let bid = GetExecutionPayloadBidResponse { version: fork, data, metadata: Default::default() }; - Ok(RelayBid { bid, body, encoding: EncodingType::Ssz }) + Ok(GetExecutionPayloadBidResponse { version: fork, data, metadata: Default::default() }) } #[cfg(test)] @@ -353,7 +353,7 @@ mod tests { use std::{ net::SocketAddr, sync::{ - Arc, + Arc, Mutex, atomic::{AtomicU64, Ordering}, }, }; @@ -526,22 +526,25 @@ mod tests { } /// A relay that refuses the stream handshake after `delay` and answers the - /// HTTP request with a `(40, 2)` bid. Returns the HTTP request's - /// `X-Timeout-Ms`. - async fn start_refusing_relay(delay: Duration) -> (SocketAddr, Arc) { + /// HTTP request with a `(40, 2)` bid. Returns each request's arrival and + /// `X-Timeout-Ms`, the handshake's first. + async fn start_refusing_relay( + delay: Duration, + ) -> (SocketAddr, Arc>>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); - let timeout_ms = Arc::new(AtomicU64::new(0)); - let recorded = timeout_ms.clone(); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = requests.clone(); let relay = axum::Router::new().fallback(move |method: Method, headers: HeaderMap| { let recorded = recorded.clone(); async move { + let sent = headers.get(HEADER_TIMEOUT_MS).and_then(|v| v.to_str().ok()); + let timeout_ms = sent.and_then(|v| v.parse().ok()).unwrap(); + recorded.lock().unwrap().push((Instant::now(), timeout_ms)); if method != Method::POST { tokio::time::sleep(delay).await; return StatusCode::NOT_FOUND.into_response(); } - let sent = headers.get(HEADER_TIMEOUT_MS).and_then(|v| v.to_str().ok()); - recorded.store(sent.and_then(|v| v.parse().ok()).unwrap(), Ordering::Relaxed); let mut res = relay_bid(40, 2).bid.data.as_ssz_bytes().into_response(); let headers = res.headers_mut(); headers.insert(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()); @@ -550,7 +553,7 @@ mod tests { } }); tokio::spawn(async move { axum::serve(listener, relay).await }); - (addr, timeout_ms) + (addr, requests) } fn bid_frame(value: u64, execution_payment: u64) -> Message { @@ -628,24 +631,37 @@ mod tests { RELAY_EPBS_STREAM_FALLBACK.with_label_values(&[relay_id]).get() } - // A handshake answered 204 is no bid. Only a refused handshake falls back - // to HTTP: one that runs out the window, a stream that breaks before a bid, - // and a frame over the size cap are no bid, with no HTTP request. + // Only a refused handshake falls back to HTTP, a 2xx in place of the + // upgrade included: one that runs out the window, a stream that breaks + // before a bid that decodes, and a frame over the size cap are no bid, with + // no HTTP request. Only a completed handshake observes the handshake + // latency and the updates per stream. #[tokio::test] async fn test_epbs_stream_failures_by_status() { let oversized = Message::Binary(vec![0; MAX_SIZE_GET_HEADER_RESPONSE + 1].into()); - for ((relay, posts), status, fell_back) in [ - (start_status_relay(StatusCode::NO_CONTENT).await, StatusCode::NO_CONTENT, 0), - (start_status_relay(StatusCode::NOT_FOUND).await, StatusCode::NOT_FOUND, 1), - (start_silent_relay().await, TIMEOUT_ERROR_STATUS, 0), - (start_stream_relay(vec![], ZERO, false).await, TRANSPORT_ERROR_STATUS, 0), - (start_stream_relay(vec![oversized], ZERO, true).await, TRANSPORT_ERROR_STATUS, 0), + let bad_bid = Message::Binary(vec![0x01, 7, 1, 2, 3].into()); + for ((relay, posts), status, fell_back, connected) in [ + (start_status_relay(StatusCode::NO_CONTENT).await, TRANSPORT_ERROR_STATUS, 1, 0), + (start_status_relay(StatusCode::NOT_FOUND).await, StatusCode::NOT_FOUND, 1, 0), + (start_status_relay(StatusCode::OK).await, TRANSPORT_ERROR_STATUS, 1, 0), + (start_silent_relay().await, TIMEOUT_ERROR_STATUS, 0, 0), + (start_stream_relay(vec![], ZERO, false).await, TRANSPORT_ERROR_STATUS, 0, 1), + (start_stream_relay(vec![bad_bid], ZERO, false).await, TRANSPORT_ERROR_STATUS, 0, 1), + (start_stream_relay(vec![oversized], ZERO, true).await, TRANSPORT_ERROR_STATUS, 0, 1), ] { - let (relay_id, bid) = bid_from_stream_relay(relay, 300).await; + let (relay_id, bid) = + tokio::time::timeout(Duration::from_secs(5), bid_from_stream_relay(relay, 300)) + .await + .expect("the deadline ends every stream"); assert!(bid.is_none(), "{status}"); assert_eq!(status_count(status, &relay_id), 1, "{status}"); assert_eq!(fallbacks(&relay_id), fell_back, "{status}"); assert_eq!(posts.load(Ordering::Relaxed), fell_back, "{status}"); + let relay = [relay_id.as_str()]; + let handshakes = RELAY_EPBS_STREAM_CONNECT_LATENCY.with_label_values(&relay); + assert_eq!(handshakes.get_sample_count(), connected, "{status}"); + let windows = RELAY_EPBS_STREAM_UPDATES.with_label_values(&relay); + assert_eq!(windows.get_sample_count(), connected, "{status}"); } } @@ -653,7 +669,7 @@ mod tests { // handshake left, and its bid is served #[tokio::test] async fn test_epbs_stream_refused_handshake_falls_back_with_the_time_left() { - let (relay, timeout_ms) = start_refusing_relay(Duration::from_millis(200)).await; + let (relay, requests) = start_refusing_relay(Duration::from_millis(200)).await; let (relay_id, bid) = bid_from_stream_relay(relay, 1_000).await; let bid = bid.expect("the HTTP bid").bid; assert_eq!((bid.data.message.value, bid.data.message.execution_payment), (40, 2)); @@ -661,8 +677,42 @@ mod tests { assert_eq!(fallbacks(&relay_id), 1); // 1000 ms less the default 50 ms buffer and the 200 ms handshake - let timeout_ms = timeout_ms.load(Ordering::Relaxed); - assert!((1..=750).contains(&timeout_ms), "{timeout_ms}"); + let [handshake, fallback] = requests.lock().unwrap()[..] else { panic!("two requests") }; + assert!((1..=750).contains(&fallback.1), "{}", fallback.1); + + // Both requests name the same deadline: the fallback's time left is + // the handshake's, less the time the handshake took + let deadline = |(at, timeout_ms): (Instant, u64)| at + Duration::from_millis(timeout_ms); + let (handshake, fallback) = (deadline(handshake), deadline(fallback)); + let drift = fallback.max(handshake) - fallback.min(handshake); + assert!(drift < Duration::from_millis(100), "{drift:?}"); + } + + // A stream that breaks after a bid arrived serves that bid, with no HTTP + // request + #[tokio::test] + async fn test_epbs_stream_break_after_a_bid_serves_it() { + let (relay, posts) = start_stream_relay(vec![bid_frame(9, 1)], ZERO, false).await; + let (relay_id, bid) = bid_from_stream_relay(relay, 1_000).await; + let bid = bid.expect("the bid before the break").bid; + assert_eq!((bid.data.message.value, bid.data.message.execution_payment), (9, 1)); + assert_eq!(status_count(StatusCode::OK, &relay_id), 1); + assert_eq!((fallbacks(&relay_id), posts.load(Ordering::Relaxed)), (0, 0)); + } + + // A bid for a fork other than Gloas is an invalid frame, and leaves the + // bid before it served + #[tokio::test] + async fn test_epbs_stream_wrong_fork_bid_is_invalid() { + let mut fulu = bid_frame(90, 0).into_data().to_vec(); + fulu[1] = 6; + let messages = vec![bid_frame(9, 1), Message::Binary(fulu.into())]; + let (relay, _) = start_stream_relay(messages, ZERO, true).await; + let (relay_id, bid) = bid_from_stream_relay(relay, 1_000).await; + let bid = bid.expect("the gloas bid").bid; + assert_eq!((bid.version, bid.data.message.value), (ForkName::Gloas, 9)); + let invalid = RELAY_EPBS_STREAM_INVALID_FRAMES.with_label_values(&[relay_id.as_str()]); + assert_eq!(invalid.get(), 1); } // A window whose every bid fails to decode is no bid, not a failed stream, @@ -684,6 +734,9 @@ mod tests { n_frames as u64 ); assert_eq!(RELAY_EPBS_STREAM_UPDATES.with_label_values(&relay).get_sample_sum(), 0.0); + // No bid, so no time to first bid + let latency = RELAY_LATENCY.with_label_values(&[STREAM, relay_id.as_str()]); + assert_eq!(latency.get_sample_count(), 0); } } @@ -700,7 +753,10 @@ mod tests { bid_frame(77, 5), ]; let (relay, posts) = start_stream_relay(messages, Duration::from_millis(150), true).await; + let start = Instant::now(); let (relay_id, bid) = bid_from_stream_relay(relay, 2_000).await; + // The relay closes 600 ms in, long before the 1950 ms deadline + assert!(start.elapsed() < Duration::from_millis(1_500), "{:?}", start.elapsed()); let bid = bid.expect("the stream's bid").bid; assert_eq!(bid.version, ForkName::Gloas); let bid = &bid.data.message; @@ -720,6 +776,11 @@ mod tests { let latency = RELAY_LATENCY.with_label_values(&[STREAM, relay_id.as_str()]); assert_eq!(latency.get_sample_count(), 1); assert!(latency.get_sample_sum() < 0.3, "{}", latency.get_sample_sum()); + + // No frame was invalid, so that series is absent rather than zero + let scrape = TextEncoder::new().encode_to_string(&PBS_METRICS_REGISTRY.gather()).unwrap(); + let invalid = format!(r#"relay_epbs_stream_invalid_frames_total{{relay_id="{relay_id}"}}"#); + assert!(!scrape.contains(&invalid), "{invalid}"); } // Dashboards and the docs select the stream series by name diff --git a/crates/pbs/src/routes/execution_payload_bid_ws.rs b/crates/pbs/src/routes/execution_payload_bid_ws.rs index a3a4d8212..937f2409a 100644 --- a/crates/pbs/src/routes/execution_payload_bid_ws.rs +++ b/crates/pbs/src/routes/execution_payload_bid_ws.rs @@ -1,29 +1,81 @@ -//! The ePBS bid over the bid stream of a relay with `get_header = "stream"`. -//! Each frame carries an SSZ `SignedExecutionPayloadBid`. +//! The ePBS bid over a websocket, for relays configured with +//! `get_header = "stream"`. The stream URL is derived from the relay's own +//! `url`. One connection per bid request, dropped when the request returns. +//! +//! The request is the handshake itself: slot / parent_hash / parent_root / +//! pubkey in the path, deadline, timestamp and request auth in headers, same +//! data the HTTP request carries. The relay replies with one binary frame per +//! bid update: +//! +//! ```text +//! u8 message type +//! u8 fork +//! .. SSZ SignedExecutionPayloadBid +//! ``` -use std::time::Duration; +use std::{ + sync::{Arc, OnceLock}, + time::Duration, +}; -use axum::http::{HeaderMap, HeaderValue}; +use axum::http::{HeaderMap, HeaderValue, Request, header::USER_AGENT}; use base64::{Engine, prelude::BASE64_STANDARD}; use cb_common::{ - pbs::{HEADER_REQUEST_AUTH, RelayClient, error::PbsError}, - wire::CONSENSUS_VERSION_HEADER, + pbs::{ + ForkName, HEADER_REQUEST_AUTH, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, + error::PbsError, + }, + utils::utcnow_ms, + wire::{CONSENSUS_VERSION_HEADER, EncodingType, GLOAS_CONSENSUS_VERSION}, }; +use futures::StreamExt; use reqwest::StatusCode; -use tokio::time::Instant; -use tracing::debug; +use rustls::{ClientConfig, RootCertStore, crypto::aws_lc_rs}; +use tokio::time::{Instant, sleep_until, timeout_at}; +use tokio_tungstenite::{ + Connector, connect_async_tls_with_config, + tungstenite::{ + Bytes, Error as WsError, Message, client::IntoClientRequest, protocol::WebSocketConfig, + }, +}; +use tracing::{debug, warn}; use url::Url; use super::execution_payload_bid::{RelayBid, decode_ssz_bid}; use crate::{ - bid_stream::{Frame, Held, handshake_request, read_bid_stream}, - constants::{GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, TRANSPORT_ERROR_STATUS}, - metrics::RELAY_STATUS_CODE, + constants::{ + GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE, + TIMEOUT_ERROR_STATUS, TRANSPORT_ERROR_STATUS, + }, + metrics::{ + RELAY_EPBS_STREAM_CONNECT_LATENCY, RELAY_EPBS_STREAM_INVALID_FRAMES, + RELAY_EPBS_STREAM_UPDATES, RELAY_LATENCY, RELAY_STATUS_CODE, + }, }; +/// Frame prefix: message type + fork. +const FRAME_PREFIX_LEN: usize = 2; + +const MSG_BID: u8 = 0x01; + +fn fork_from_wire(byte: u8) -> Option { + Some(match byte { + 0 => ForkName::Base, + 1 => ForkName::Altair, + 2 => ForkName::Bellatrix, + 3 => ForkName::Capella, + 4 => ForkName::Deneb, + 5 => ForkName::Electra, + 6 => ForkName::Fulu, + 7 => ForkName::Gloas, + 8 => ForkName::Heze, + _ => return None, + }) +} + type StreamOutcome = (StatusCode, Result, PbsError>); -/// The relay's latest bid at the deadline +/// Open a stream to the relay and return its latest bid at the deadline pub(super) async fn get_execution_payload_bid_ws( auth_ssz: &[u8], relay: &RelayClient, @@ -31,14 +83,13 @@ pub(super) async fn get_execution_payload_bid_ws( url: Url, timeout_ms: u64, ) -> Result, PbsError> { - let (status, res) = - stream_execution_payload_bid(auth_ssz, relay, send_headers, url, timeout_ms).await; + let (status, res) = stream_bid(auth_ssz, relay, send_headers, url, timeout_ms).await; let endpoint = GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG; RELAY_STATUS_CODE.with_label_values(&[status.as_str(), endpoint, &relay.id]).inc(); res } -async fn stream_execution_payload_bid( +async fn stream_bid( auth_ssz: &[u8], relay: &RelayClient, send_headers: &HeaderMap, @@ -46,37 +97,112 @@ async fn stream_execution_payload_bid( timeout_ms: u64, ) -> StreamOutcome { let deadline = Instant::now() + Duration::from_millis(timeout_ms); - let mut request = match handshake_request(&url, relay, send_headers) { + let request = match build_handshake_request(auth_ssz, relay, send_headers, &url, timeout_ms) { Ok(request) => request, Err(err) => return (TRANSPORT_ERROR_STATUS, Err(err)), }; - let headers = request.headers_mut(); - if let Some(fork) = send_headers.get(CONSENSUS_VERSION_HEADER) { - headers.insert(CONSENSUS_VERSION_HEADER, fork.clone()); - } - let mut request_auth = HeaderValue::try_from(BASE64_STANDARD.encode(auth_ssz)) - .expect("base64 is always a valid header value"); - request_auth.set_sensitive(true); - headers.insert(HEADER_REQUEST_AUTH, request_auth); - // Each frame is decoded on arrival, so the newest is the latest bid that - // decodes - let parse = |frame: Frame| decode_ssz_bid(frame.bid, frame.fork); - let Held { latest, updates, connect_latency, first_frame_latency, invalid_frames } = - match read_bid_stream(request, deadline, relay, parse).await { - Ok(held) => held, - Err((status, err)) => return (status, Err(err)), + let config = WebSocketConfig::default() + .max_message_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)) + .max_frame_size(Some(MAX_SIZE_GET_HEADER_RESPONSE)); + + let start_request = Instant::now(); + let connect = connect_async_tls_with_config( + request, + Some(config), + true, + Some(Connector::Rustls(tls_config().clone())), + ); + let (mut stream, _) = match timeout_at(deadline, connect).await { + Ok(Ok(connected)) => connected, + Ok(Err(err)) => return connect_failed(&err), + Err(_) => { + return (TIMEOUT_ERROR_STATUS, Err(PbsError::WebSocketTimeout)); + } + }; + let connect_latency = start_request.elapsed(); + RELAY_EPBS_STREAM_CONNECT_LATENCY + .with_label_values(&[relay.id.as_str()]) + .observe(connect_latency.as_secs_f64()); + debug!(relay_id = relay.id.as_ref(), ?connect_latency, "ws connected"); + + let timer = sleep_until(deadline); + tokio::pin!(timer); + + let mut latest: Option = None; + let mut first_bid_latency: Option = None; + let mut updates = 0usize; + let mut invalid_frames = 0usize; + let mut stream_error = None; + + loop { + let message = tokio::select! { + biased; + _ = &mut timer => break, + message = stream.next() => message, + }; + + let message = match message { + Some(Ok(message)) => message, + Some(Err(err)) => { + warn!(relay_id = relay.id.as_ref(), %err, "ws stream error"); + stream_error = Some(PbsError::WebSocket(format!("stream error: {err}"))); + break; + } + None => break, + }; + + let payload = match message { + Message::Binary(payload) => payload, + Message::Close(_) => break, + _ => continue, }; + match parse_frame(payload).and_then(decode_bid) { + Ok(bid) => { + updates += 1; + if first_bid_latency.is_none() { + first_bid_latency = Some(start_request.elapsed()); + } + latest = Some(bid); + } + Err(err) => { + invalid_frames += 1; + if invalid_frames == 1 { + warn!(relay_id = relay.id.as_ref(), %err, "invalid ws frame, skipping"); + } + } + } + } + + drop(stream); + + RELAY_EPBS_STREAM_UPDATES.with_label_values(&[relay.id.as_str()]).observe(updates as f64); + if invalid_frames > 0 { + RELAY_EPBS_STREAM_INVALID_FRAMES + .with_label_values(&[relay.id.as_str()]) + .inc_by(invalid_frames as u64); + } + let Some(bid) = latest else { + if let Some(err) = stream_error { + return (TRANSPORT_ERROR_STATUS, Err(err)); + } + debug!(relay_id = relay.id.as_ref(), ?connect_latency, invalid_frames, "no bid"); return (StatusCode::NO_CONTENT, Ok(None)); }; + if let Some(first_bid_latency) = first_bid_latency { + RELAY_LATENCY + .with_label_values(&[GET_EXECUTION_PAYLOAD_BID_STREAM_ENDPOINT_TAG, &relay.id]) + .observe(first_bid_latency.as_secs_f64()); + } + debug!( relay_id = relay.id.as_ref(), ?connect_latency, - first_bid_latency = ?first_frame_latency, + ?first_bid_latency, updates, invalid_frames, "received new bid from ws stream" @@ -84,3 +210,197 @@ async fn stream_execution_payload_bid( (StatusCode::OK, Ok(Some(bid))) } + +/// A frame's bid, which must be for Gloas +fn decode_bid((fork, body): (ForkName, Bytes)) -> Result { + if fork != ForkName::Gloas { + return Err(PbsError::WebSocket(format!("bid for {fork}, expected gloas"))); + } + let bid = decode_ssz_bid(&body, fork)?; + Ok(RelayBid { bid, body, encoding: EncodingType::Ssz }) +} + +/// A relay rejecting the handshake puts the reason in the body, which +/// tungstenite's own `Display` drops. The body is only what arrived alongside +/// the headers, so it can be partial or empty. +fn connect_failed(err: &WsError) -> StreamOutcome { + let WsError::Http(res) = err else { + return (TRANSPORT_ERROR_STATUS, Err(PbsError::WebSocketConnect(err.to_string()))); + }; + + let code = res.status(); + let body = res.body().as_deref().unwrap_or_default(); + let msg = if body.is_empty() { + format!("rejected with {code}") + } else { + format!("rejected with {code}: {}", String::from_utf8_lossy(body)) + }; + + // A 2xx handshake answer is a failed connect, not a delivered bid + let code = if code.is_success() { TRANSPORT_ERROR_STATUS } else { code }; + + (code, Err(PbsError::WebSocketConnect(msg))) +} + +fn build_handshake_request( + auth_ssz: &[u8], + relay: &RelayClient, + send_headers: &HeaderMap, + url: &Url, + timeout_ms: u64, +) -> Result, PbsError> { + let mut request = url + .as_str() + .into_client_request() + .map_err(|err| PbsError::WebSocketConnect(format!("invalid ws url: {err}")))?; + + let headers = request.headers_mut(); + if let Some(user_agent) = send_headers.get(USER_AGENT) { + headers.insert(USER_AGENT, user_agent.clone()); + } + + for (key, value) in relay.stream_headers() { + headers.insert(key, value.clone()); + } + + headers.insert(CONSENSUS_VERSION_HEADER, GLOAS_CONSENSUS_VERSION.clone()); + let mut request_auth = HeaderValue::try_from(BASE64_STANDARD.encode(auth_ssz)) + .expect("base64 is always a valid header value"); + request_auth.set_sensitive(true); + headers.insert(HEADER_REQUEST_AUTH, request_auth); + + headers.insert(HEADER_START_TIME_UNIX_MS, HeaderValue::from(utcnow_ms())); + headers.insert(HEADER_TIMEOUT_MS, HeaderValue::from(timeout_ms)); + + Ok(request) +} + +fn parse_frame(payload: Bytes) -> Result<(ForkName, Bytes), PbsError> { + let &[msg_type, fork_byte] = payload + .first_chunk::() + .ok_or_else(|| PbsError::WebSocket(format!("frame too short: {} bytes", payload.len())))?; + + if msg_type != MSG_BID { + return Err(PbsError::WebSocket(format!("unknown message type: {msg_type}"))); + } + + let fork = fork_from_wire(fork_byte) + .ok_or_else(|| PbsError::WebSocket(format!("unknown fork: {fork_byte}")))?; + + Ok((fork, payload.slice(FRAME_PREFIX_LEN..))) +} + +/// One TLS config for every ePBS stream connection. Left to tokio-tungstenite +/// it is rebuilt per connect, which reparses the root store and, worse, gives +/// each connection its own session cache: every slot then pays a full handshake +/// instead of a resumed one. The provider is named explicitly because rustls is +/// built with both `ring` and `aws-lc-rs` here, so the default builder needs a +/// process-wide install to pick one. +fn tls_config() -> &'static Arc { + static CONFIG: OnceLock> = OnceLock::new(); + CONFIG.get_or_init(|| { + let mut roots = RootCertStore::empty(); + roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + + Arc::new( + ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .expect("aws-lc-rs supports tls 1.2 and 1.3") + .with_root_certificates(roots) + .with_no_client_auth(), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bid_frame(fork_byte: u8, bid: &[u8]) -> Bytes { + let mut frame = vec![MSG_BID, fork_byte]; + frame.extend_from_slice(bid); + Bytes::from(frame) + } + + #[test] + fn test_fork_from_wire_covers_all_forks() { + let forks = ForkName::list_all(); + + for (byte, fork) in forks.iter().enumerate() { + if *fork <= ForkName::Base { + continue; + } + assert_eq!(fork_from_wire(byte as u8), Some(*fork), "fork {fork} unmapped"); + } + + assert_eq!(fork_from_wire(forks.len() as u8), None); + } + + #[test] + fn test_parse_frame() { + assert!(matches!( + parse_frame(bid_frame(6, &[1, 2, 3])), + Ok((ForkName::Fulu, bid)) if bid.as_ref() == [1, 2, 3] + )); + + // Empty bid payload is well-formed at this layer, SSZ decoding rejects it + assert!(matches!(parse_frame(bid_frame(6, &[])), Ok((ForkName::Fulu, _)))); + + for bad in [ + // Truncated prefix + Bytes::from_static(&[]), + Bytes::from_static(&[MSG_BID]), + // Unknown fork + bid_frame(0xff, &[1]), + // Unknown message type + Bytes::from_static(&[0xff, 6]), + ] { + assert!(matches!(parse_frame(bad), Err(PbsError::WebSocket(_)))); + } + } + + #[test] + fn test_connect_failed_carries_relay_reason() { + let rejected = axum::http::Response::builder() + .status(401) + .body(Some(b"api key not registered".to_vec())) + .unwrap(); + + let (status, res) = connect_failed(&WsError::Http(Box::new(rejected))); + assert_eq!(status, StatusCode::UNAUTHORIZED); + let Err(PbsError::WebSocketConnect(msg)) = res else { panic!("wrong outcome") }; + assert!(msg.contains("401"), "{msg}"); + assert!(msg.contains("api key not registered"), "{msg}"); + + // No body to add, and transport failures keep tungstenite's own message + let empty = axum::http::Response::builder().status(404).body(None).unwrap(); + let (status, _) = connect_failed(&WsError::Http(Box::new(empty))); + assert_eq!(status, StatusCode::NOT_FOUND); + + let (status, res) = connect_failed(&WsError::ConnectionClosed); + assert_eq!(status, TRANSPORT_ERROR_STATUS); + assert!(matches!(res, Err(PbsError::WebSocketConnect(_)))); + } + + // A url pointing at a plain http endpoint answers the handshake 200. That + // is the code the stream series uses for a delivered bid, so a failed + // handshake must never carry it. + #[test] + fn test_connect_failed_never_reports_a_success_code() { + for code in [200u16, 204, 299] { + let answered = axum::http::Response::builder().status(code).body(None).unwrap(); + let (status, res) = connect_failed(&WsError::Http(Box::new(answered))); + assert_eq!( + status, TRANSPORT_ERROR_STATUS, + "handshake answered {code} counted as a served stream" + ); + let Err(PbsError::WebSocketConnect(msg)) = res else { panic!("wrong outcome") }; + assert!(msg.contains(&code.to_string()), "{msg}"); + } + + // A relay's own rejection code still reaches the series unchanged + let moved = axum::http::Response::builder().status(302).body(None).unwrap(); + let (status, _) = connect_failed(&WsError::Http(Box::new(moved))); + assert_eq!(status, StatusCode::FOUND); + } +} diff --git a/docs/docs/get_started/configuration.md b/docs/docs/get_started/configuration.md index 04a8963b2..55c6314eb 100644 --- a/docs/docs/get_started/configuration.md +++ b/docs/docs/get_started/configuration.md @@ -142,7 +142,7 @@ If the handshake fails, PBS falls back to a plain HTTP `get_header` at the relay There is no fallback if the handshake is still unanswered when the window ends (recorded as `555`), or if it fails too late to leave time for an HTTP request. There is also none once the stream is open. If it breaks before any bid arrives, that relay contributes no header for the slot (recorded as `556`). If a bid already arrived, PBS validates and returns it as usual. -On a high-latency connection, where the handshake risks timing out, you can add the relay a second time with `get_header = "http"` and a different `id`. PBS asks both entries every slot and keeps the better bid, so the HTTP entry still returns a header when the stream does not. +On a high-latency connection, where the handshake risks timing out, you can add the relay a second time with `get_header = "http"` and a different `id`. PBS asks both entries every slot and keeps the better bid, so the HTTP entry still returns a header when the stream does not. (unreleased, from v0.12.0-rc1) After the Gloas fork only the first of the two entries serves [ePBS bids](./epbs.md#bid-streaming). ```toml [[relays]] diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 500ba41d5..92ca6ac51 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -98,9 +98,9 @@ A builder answers an HTTP bid request once, so a better bid it makes after answe Ask the builder whether it serves the ePBS bid stream, at `ws(s):///eth/v1/builder/execution_payload_bid_stream/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}`. If it does, set `get_header = "stream"` on its relay entry. Before the fork the same setting reads the builder's `get_header` bids from its [PBS stream](./configuration.md#bid-streaming), so for a builder that serves only one of the two, Commit-Boost logs a failed stream on each request to the other and asks over HTTP instead. The handshake carries the relay entry's `headers`, so a builder that requires an [API key](./configuration.md#relay-api-keys) gets it. -For each bid request to a streaming builder, Commit-Boost opens the stream and returns the latest streamed bid that decodes, without an HTTP request. Only a stream that fails to connect, because the connection fails or the builder refuses the handshake, is retried over HTTP, with the time left. A handshake still unanswered at the deadline, or a stream that breaks before a bid, is no bid. A builder [outside your config](#builders-outside-your-config) is always asked over HTTP. +For each bid request to a streaming builder, Commit-Boost opens the stream and returns the latest streamed bid that decodes. Only a stream that fails to connect, because of a connection error or a handshake answered with anything but the upgrade, `204` included, is retried over HTTP with the time left. A handshake still unanswered at the deadline, or a stream that breaks before a bid, is no bid. A builder [outside your config](#builders-outside-your-config) is always asked over HTTP. If a builder has two relay entries, one on `"stream"` and one on `"http"`, only the first that matches the key's auth data serves ePBS bids. -Streaming makes bid requests slower. Commit-Boost replies to the beacon node once the stream has ended: when the builder closes it, when it fails, or at the beacon node's deadline less `proposer_deadline_buffer_ms`. An HTTP builder is answered as soon as it responds. `timeout_get_header_ms` does not shorten the wait. If the beacon node reports bid requests timing out on streaming builders, raise `proposer_deadline_buffer_ms`. +Streaming makes bid requests slower. Commit-Boost replies to the beacon node once the stream has ended: when the builder closes it, when it fails, or at the beacon node's deadline less `proposer_deadline_buffer_ms`. After a retry over HTTP, it replies once that request answers. A bid request to an HTTP builder is answered as soon as the builder responds. `timeout_get_header_ms` does not shorten the wait. If the beacon node reports bid requests timing out on streaming builders, raise `proposer_deadline_buffer_ms`. The stream connects directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`, while the HTTP fallback uses them. If your host reaches builders only through a proxy, keep them on `"http"`. @@ -111,9 +111,9 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Question | Series | |---|---| | What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences, `202` for the signed block whatever the builders answer, `4xx` for a rejected request, `500` when no builder accepted the preferences or on an internal error | -| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"`; a dial Commit-Boost [refuses](#builders-outside-your-config), or whose lookup fails, times out or finds no free slot, sends no request and is not counted | +| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure; on the bid stream, only the deadline). Requests to builders outside your config count under `relay_id="dial"`; a dial Commit-Boost [refuses](#builders-outside-your-config), or whose lookup fails, times out or finds no free slot, sends no request and is not counted | | How fast are builders? | `cb_pbs_relay_latency`, by `relay_id` | -| Is a builder's bid stream working? | `cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid_stream"}`: `200` when the stream supplied a bid, `204` when it had none, any other code a failed stream: `555` the deadline ran out before the handshake completed, `556` a connection that failed or dropped before a bid, or the builder's own refusal such as `404`. | +| Is a builder's bid stream working? | `cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid_stream"}`: `200` when the stream supplied a bid, `204` when it had none, `555` when the deadline ran out before the handshake completed, `556` when the connection failed, the handshake was answered with a success code instead of the upgrade, or the stream dropped before a bid, or the builder's own refusal code such as `404` | | What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from each bid a builder serves | ## Troubleshooting @@ -125,5 +125,5 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | | Bid requests to a streaming builder log "stream failed" with "rejected with 404", and `cb_pbs_relay_epbs_stream_fallback_total` rises | The builder does not serve the ePBS bid stream. Commit-Boost asks it over HTTP instead | Set `get_header = "http"` on its relay entry | | Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | -| Bid requests to a streaming builder log "stream failed" with "websocket timed out" or a connection error | Commit-Boost could not open the stream, for example because your network allows only proxied connections, which the stream does not use. A connection error with time left is retried over HTTP; a handshake that times out is no bid | Keep the builder on `get_header = "http"`, or allow direct connections to it | +| Bid requests to a streaming builder log "stream failed" with "websocket timed out" or a connection error | Commit-Boost could not open the stream, for example because your network allows only proxied connections, which the stream does not use. A connection error with time left is retried over HTTP; a handshake that times out is no bid | Set `get_header = "http"` on its relay entry, or allow direct connections to it | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/docs/get_started/running/metrics-catalog.md b/docs/docs/get_started/running/metrics-catalog.md index ff926b743..c096d5c38 100644 --- a/docs/docs/get_started/running/metrics-catalog.md +++ b/docs/docs/get_started/running/metrics-catalog.md @@ -14,7 +14,7 @@ PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. Th | Metric name (wire) | Type | Labels | Description | |---|---|---|---| -| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (timeout, DNS or connection failure; on `get_header_stream`, only the deadline), and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). (unreleased, from v0.12.0-rc1) On `get_execution_payload_bid_stream`, a `204` answer to the handshake records `204`. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. | +| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (timeout, DNS or connection failure; on `get_header_stream`, only the deadline), and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). (unreleased, from v0.12.0-rc1) On `get_execution_payload_bid_stream`, as on `get_header_stream`, `555` is only the deadline. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. | | `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `get_execution_payload_bid_stream`, `submit_builder_preferences`, `submit_signed_beacon_block`. Under `get_header_stream` and `get_execution_payload_bid_stream` the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | | `cb_pbs_relay_last_slot` | Gauge | `relay_id` | Latest slot for which a relay delivered a bid. Set to the current slot on each successful `get_header` bid and (unreleased, from v0.12.0-rc1) each ePBS bid served from that relay. | | `cb_pbs_relay_header_value` | Gauge | `relay_id` | Value in gwei of the bid a relay delivered: for `get_header`, the bid's value converted from wei (รท 1e9). (unreleased, from v0.12.0-rc1) For ePBS, the returned bid's `value`, without its execution payment. | @@ -22,7 +22,7 @@ PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. Th | `cb_pbs_relay_stream_updates` | Histogram | `relay_id` | Bid updates received per stream window. Observed once per `get_header` served by the stream, including windows that received nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. | | `cb_pbs_relay_stream_invalid_frames_total` | Counter | `relay_id` | Websocket frames that could not be parsed as a bid. Incremented only for windows that saw at least one, so the series is absent while zero. | | `cb_pbs_relay_stream_fallback_total` | Counter | `relay_id` | Stream attempts that fell back to a plain HTTP `get_header`. Only a handshake failure with bid window left to retry counts here; one that consumed the window shows on the status series alone. The fallback's own result is recorded under `endpoint="get_header"`. | -| `cb_pbs_relay_epbs_stream_connect_latency` | Histogram | `relay_id` | (unreleased, from v0.12.0-rc1) ePBS bid stream handshake latency in seconds. Observed once per successful handshake; a handshake that fails, runs out the deadline, or is answered `204` records nothing here. Custom buckets from 5 ms to 1 s. | +| `cb_pbs_relay_epbs_stream_connect_latency` | Histogram | `relay_id` | (unreleased, from v0.12.0-rc1) ePBS bid stream handshake latency in seconds. Observed once per successful handshake; a handshake that fails or runs out the deadline records nothing here. Custom buckets from 5 ms to 1 s. | | `cb_pbs_relay_epbs_stream_updates` | Histogram | `relay_id` | (unreleased, from v0.12.0-rc1) Bids that decoded per ePBS bid stream. Observed once per stream that connected, including streams that received nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. | | `cb_pbs_relay_epbs_stream_invalid_frames_total` | Counter | `relay_id` | (unreleased, from v0.12.0-rc1) ePBS bid stream frames whose message type or fork byte cannot be read, or whose bid does not decode. Incremented only for streams that saw at least one, so the series is absent while zero. | | `cb_pbs_relay_epbs_stream_fallback_total` | Counter | `relay_id` | (unreleased, from v0.12.0-rc1) ePBS bid streams that fell back to an HTTP bid request. Only a stream that failed to connect with time left counts here; one that ran out the deadline or broke after connecting shows on the status series alone. The fallback's own result is recorded under `endpoint="get_execution_payload_bid"`. | diff --git a/docs/docs/get_started/running/metrics.md b/docs/docs/get_started/running/metrics.md index acb484577..6cb6e45fe 100644 --- a/docs/docs/get_started/running/metrics.md +++ b/docs/docs/get_started/running/metrics.md @@ -123,4 +123,4 @@ For a relay with `get_header = "stream"` (see [Configuration > Bid streaming](.. | Is it actually streaming? | `cb_pbs_relay_stream_updates`: bid updates received per window. A healthy relay sends several; windows that carry at most one update mean the stream connects but does not stream | | Are the frames usable? | `cb_pbs_relay_stream_invalid_frames_total`: frames that could not be parsed as a bid, absent while zero | -The [ePBS bid stream](../epbs.md#bid-streaming) (unreleased, from v0.12.0-rc1) records its outcome and time-to-first-bid on the same two general series under `endpoint="get_execution_payload_bid_stream"`, and its HTTP fallback under `endpoint="get_execution_payload_bid"`. A handshake answered `204` records `204`, no bid. Its own series, labeled by `relay_id`, are `cb_pbs_relay_epbs_stream_connect_latency`, `cb_pbs_relay_epbs_stream_updates`, `cb_pbs_relay_epbs_stream_invalid_frames_total` and `cb_pbs_relay_epbs_stream_fallback_total`, the counterparts of the four above. It decodes each bid as it arrives and does not validate it, so `cb_pbs_relay_epbs_stream_updates` counts bids that decode and `cb_pbs_relay_epbs_stream_invalid_frames_total` also counts those that do not. A stream that ends with only undecodable bids is no bid (`204`); if its connection drops instead, it records `556`. +The [ePBS bid stream](../epbs.md#bid-streaming) (unreleased, from v0.12.0-rc1) records its outcome and time-to-first-bid on the same two general series under `endpoint="get_execution_payload_bid_stream"`, and its HTTP fallback under `endpoint="get_execution_payload_bid"`. Its own series, labelled by `relay_id`, are `cb_pbs_relay_epbs_stream_connect_latency`, `cb_pbs_relay_epbs_stream_updates`, `cb_pbs_relay_epbs_stream_invalid_frames_total` and `cb_pbs_relay_epbs_stream_fallback_total`, the counterparts of the four above. It decodes each bid as it arrives and does not validate it, so `cb_pbs_relay_epbs_stream_updates` counts bids that decode and `cb_pbs_relay_epbs_stream_invalid_frames_total` also counts those that do not. A stream that ends with only undecodable bids is no bid (`204`); if its connection drops instead, it records `556`. diff --git a/tests/tests/pbs_get_execution_payload_bid_ws.rs b/tests/tests/pbs_get_execution_payload_bid_ws.rs index 55032c020..dcaa2ecd6 100644 --- a/tests/tests/pbs_get_execution_payload_bid_ws.rs +++ b/tests/tests/pbs_get_execution_payload_bid_ws.rs @@ -20,12 +20,10 @@ const TEST_SLOT: u64 = 255; const PARENT_ROOT: B256 = B256::repeat_byte(1); const BUDGET_MS: u64 = 1_000; -/// A relay bidding `http` gwei over HTTP and streaming `stream_bids`, each a -/// `(value, execution_payment)` in gwei -fn relay_states(http: u64, stream_bids: Vec<(u64, u64)>) -> (MockRelayState, MockWsRelayState) { +/// A relay streaming `stream_bids`, each a `(value, execution_payment)` in gwei +fn relay_states(stream_bids: Vec<(u64, u64)>) -> (MockRelayState, MockWsRelayState) { let signer = random_secret(); - let http_state = - MockRelayState::new(Chain::Hoodi, signer.clone()).with_trustless_bid_gwei(http); + let http_state = MockRelayState::new(Chain::Hoodi, signer.clone()); (http_state, MockWsRelayState::new(Chain::Hoodi, signer).with_epbs_bids(stream_bids)) } @@ -66,32 +64,27 @@ fn assert_bid(res: Option, value: u64, execution } /// A relay that streams get_header but not the ePBS bid answers the handshake -/// with 404, and the HTTP request PBS falls back to supplies the bid. +/// with 404, and PBS falls back to HTTP: its bid is served, and the builder's +/// 401 reaches the beacon node. #[tokio::test] -async fn test_get_execution_payload_bid_ws_no_stream_route_http_bid_serves() -> Result<()> { - let (http_state, ws_state) = relay_states(42, vec![(90, 0)]); - let (validator, http_state, ws_state) = - start_relay((http_state, ws_state.without_epbs_stream())).await?; - - let (code, res) = get_bid(&validator).await?; - assert_eq!(code, StatusCode::OK); - assert_bid(res, 42, 0); - assert_eq!(http_state.received_execution_payload_bid(), 1); - assert_eq!(ws_state.handshake_attempts(), 1); - assert_eq!(ws_state.received_connections(), 0, "the handshake never completed"); - Ok(()) -} - -/// The builder's 401 to the fallback HTTP request reaches the beacon node. -#[tokio::test] -async fn test_get_execution_payload_bid_ws_no_stream_bid_http_rejection_passes() -> Result<()> { - let (http_state, ws_state) = relay_states(10, vec![(50, 0)]); - let (validator, http_state, _) = - start_relay((http_state, ws_state.without_epbs_stream())).await?; - http_state.set_response_override(StatusCode::UNAUTHORIZED); - - let (code, _) = get_bid(&validator).await?; - assert_eq!(code, StatusCode::UNAUTHORIZED); +async fn test_get_execution_payload_bid_ws_no_stream_route_falls_back_to_http() -> Result<()> { + for rejection in [None, Some(StatusCode::UNAUTHORIZED)] { + let (http_state, ws_state) = relay_states(vec![]); + let (validator, http_state, ws_state) = + start_relay((http_state.with_trustless_bid_gwei(42), ws_state.without_epbs_stream())) + .await?; + if let Some(status) = rejection { + http_state.set_response_override(status); + } + + let (code, res) = get_bid(&validator).await?; + assert_eq!(code, rejection.unwrap_or(StatusCode::OK)); + if rejection.is_none() { + assert_bid(res, 42, 0); + } + assert_eq!(http_state.received_execution_payload_bid(), 1); + assert_eq!(ws_state.handshake_attempts(), 1); + } Ok(()) } @@ -100,7 +93,7 @@ async fn test_get_execution_payload_bid_ws_no_stream_bid_http_rejection_passes() #[tokio::test] async fn test_get_execution_payload_bid_ws_serves_latest_bid_without_http() -> Result<()> { let (validator, http_state, ws_state) = - start_relay(relay_states(60, vec![(70, 0), (50, 0)])).await?; + start_relay(relay_states(vec![(70, 0), (50, 0)])).await?; let (code, res) = get_bid(&validator).await?; assert_eq!(code, StatusCode::OK); @@ -114,7 +107,7 @@ async fn test_get_execution_payload_bid_ws_serves_latest_bid_without_http() -> R /// (50) before it. #[tokio::test] async fn test_get_execution_payload_bid_ws_skips_invalid_bid() -> Result<()> { - let (http_state, ws_state) = relay_states(10, vec![(50, 0), (90, 0)]); + let (http_state, ws_state) = relay_states(vec![(50, 0), (90, 0)]); let (validator, ..) = start_relay((http_state, ws_state.with_invalid_last_bids(1))).await?; let (code, res) = get_bid(&validator).await?; @@ -128,7 +121,7 @@ async fn test_get_execution_payload_bid_ws_skips_invalid_bid() -> Result<()> { #[tokio::test] async fn test_get_execution_payload_bid_ws_handshake_and_deadline() -> Result<()> { const BUFFER_MS: u64 = 200; - let (http_state, ws_state) = relay_states(10, vec![(20, 0)]); + let (http_state, ws_state) = relay_states(vec![(20, 0)]); let (_, ws_state, relay) = start_mock_dual_relay(http_state, ws_state.hold_open()).await?; let validator = setup_pbs(Chain::Hoodi, vec![relay], |pbs_config| { pbs_config.proposer_deadline_buffer_ms = BUFFER_MS @@ -136,6 +129,7 @@ async fn test_get_execution_payload_bid_ws_handshake_and_deadline() -> Result<() .await?; let start = Instant::now(); + let sent_at = cb_common::utils::utcnow_ms(); let (code, _) = get_bid(&validator).await?; assert_eq!(code, StatusCode::OK); assert!(start.elapsed() < Duration::from_millis(BUDGET_MS), "{:?}", start.elapsed()); @@ -156,6 +150,8 @@ async fn test_get_execution_payload_bid_ws_handshake_and_deadline() -> Result<() (BUDGET_MS - BUFFER_MS - 100..=BUDGET_MS - BUFFER_MS).contains(&relay_timeout_ms), "timeout header {relay_timeout_ms}ms" ); + let start_time_ms = request.start_time_ms.expect("missing start time header"); + assert!((sent_at..sent_at + BUDGET_MS).contains(&start_time_ms), "{start_time_ms}"); assert_eq!(request.api_key.as_deref(), Some(RELAY_API_KEY)); assert!(request.user_agent.is_some_and(|ua| ua.contains("commit-boost"))); Ok(()) From ac9c58718662007c87964777131dd95cd3837299 Mon Sep 17 00:00:00 2001 From: Jason Vranek Date: Tue, 13 Oct 2026 14:10:59 -0700 Subject: [PATCH 24/24] test(pbs): derive the refused-handshake bound from the buffer default The bound assumed the 50 ms default buffer, so any other default fails it. --- crates/pbs/src/routes/execution_payload_bid.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index cdd09b618..444ec5b85 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -362,7 +362,7 @@ mod tests { use axum::http::Method; use cb_common::{ config::{GetHeaderTransport, PbsModuleConfig, RelayConfig}, - pbs::{BuilderRequestAuth, ExecutionPayloadBid, RelayEntry}, + pbs::{BuilderRequestAuth, ExecutionPayloadBid, PROPOSER_DEADLINE_BUFFER_MS, RelayEntry}, types::{BlsSecretKey, BlsSignature, Chain}, }; use futures::SinkExt; @@ -676,9 +676,9 @@ mod tests { assert_eq!(status_count(StatusCode::NOT_FOUND, &relay_id), 1); assert_eq!(fallbacks(&relay_id), 1); - // 1000 ms less the default 50 ms buffer and the 200 ms handshake + // 1000 ms less the default buffer and the 200 ms handshake let [handshake, fallback] = requests.lock().unwrap()[..] else { panic!("two requests") }; - assert!((1..=750).contains(&fallback.1), "{}", fallback.1); + assert!((1..=800 - PROPOSER_DEADLINE_BUFFER_MS).contains(&fallback.1), "{}", fallback.1); // Both requests name the same deadline: the fallback's time left is // the handshake's, less the time the handshake took