From 2a3424479fb5ec4464c4f8f85a4ba385512bc8b9 Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:17:41 +0000 Subject: [PATCH 1/7] Initial commit with task details Adding .gitkeep for PR creation (default mode). This file will be removed when the task is complete. Issue: https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/issues/16 --- .gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 0000000..013780a --- /dev/null +++ b/.gitkeep @@ -0,0 +1 @@ +# .gitkeep file auto-generated at 2026-10-06T23:17:41.707Z for PR creation at branch issue-16-e3181c5664cc for issue https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/issues/16 \ No newline at end of file From 27e8a9ba00af99f8bcc024bca677968c0f7424dc Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:35:55 +0000 Subject: [PATCH 2/7] Fix Rust benchmark workloads and verify backend equivalence --- experiments/reproduce-split-index.sh | 8 + rust/Cargo.toml | 2 +- rust/benches/bench.rs | 317 +++++++++++++++--- rust/doublets-patched/PATCHES.md | 14 +- .../doublets/src/mem/split/store.rs | 7 +- rust/spacetime-module/Cargo.toml | 2 +- rust/src/benched/doublets_benched.rs | 8 +- rust/src/benched/mod.rs | 2 +- rust/src/benched/spacetimedb_benched.rs | 2 +- rust/src/doublets_impl.rs | 17 +- rust/src/lib.rs | 1 + rust/src/workload.rs | 17 + rust/tests/same_behavior.rs | 153 +++++++++ 13 files changed, 491 insertions(+), 59 deletions(-) create mode 100755 experiments/reproduce-split-index.sh create mode 100644 rust/src/workload.rs create mode 100644 rust/tests/same_behavior.rs diff --git a/experiments/reproduce-split-index.sh b/experiments/reproduce-split-index.sh new file mode 100755 index 0000000..f9be8ef --- /dev/null +++ b/experiments/reproduce-split-index.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Bounded reproduction: at most 130 live links, 8 GiB address space limit (including the linker). +set -euo pipefail +ulimit -v 8388608 +root=$(cd "$(dirname "$0")/.." && pwd) +cd "$root/rust" +BENCHMARK_TRACE=1 RUST_BACKTRACE=1 cargo test --locked --test same_behavior \ + all_doublets_variants_return_same_results -- --test-threads=1 --nocapture diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 68fa651..b1f4d4d 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -11,7 +11,7 @@ harness = false [dependencies] # SpacetimeDB 2.0 official Rust client SDK. -spacetimedb-sdk = "2" +spacetimedb-sdk = "=2.10.1" # Use patched doublets that is compatible with modern nightly (>= 1.93.0). # The pinned git revision uses removed nightly features (generators, const_trait_impl, # const_deref, etc.) incompatible with Rust >= 1.80.0. The local patch: diff --git a/rust/benches/bench.rs b/rust/benches/bench.rs index 4792ff0..27f7e9a 100644 --- a/rust/benches/bench.rs +++ b/rust/benches/bench.rs @@ -21,8 +21,6 @@ //! - `SPACETIMEDB_URI` — SpacetimeDB server URI (default: `http://localhost:3000`) //! - `SPACETIMEDB_DB` — database name (default: `benchmark-links`) -#![feature(allocator_api)] - use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use spacetimedb_vs_doublets::{ benched::{ @@ -33,6 +31,16 @@ use spacetimedb_vs_doublets::{ }; use std::time::{Duration, Instant}; +// Select a backend before constructing it, so Doublets VMs need no server. +fn backend_enabled(backend: &str) -> bool { + match std::env::var("BENCHMARK_BACKEND").as_deref() { + Ok("all") | Err(_) => true, + Ok("spacetimedb") => backend == "spacetimedb", + Ok("doublets") => backend == "doublets", + Ok(other) => panic!("Unknown BENCHMARK_BACKEND: {other}"), + } +} + // ===================== HELPERS ===================== /// Populate background links before each measured iteration. @@ -48,6 +56,9 @@ macro_rules! setup_background { // ===================== CREATE ===================== fn spacetimedb_create(c: &mut Criterion) { + if !backend_enabled("spacetimedb") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = SpacetimeDbBenched::setup(()); c.bench_with_input( @@ -72,6 +83,9 @@ fn spacetimedb_create(c: &mut Criterion) { } fn doublets_united_create(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedVolatileBenched::setup(()); c.bench_with_input( @@ -96,6 +110,9 @@ fn doublets_united_create(c: &mut Criterion) { } fn doublets_split_create(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitVolatileBenched::setup(()); c.bench_with_input( @@ -120,6 +137,9 @@ fn doublets_split_create(c: &mut Criterion) { } fn doublets_united_non_volatile_create(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedNonVolatileBenched::setup( "/tmp/bench_united_non_volatile_create.links".to_string(), @@ -146,6 +166,9 @@ fn doublets_united_non_volatile_create(c: &mut Criterion) { } fn doublets_split_non_volatile_create(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitNonVolatileBenched::setup(( "/tmp/bench_split_non_volatile_create_data.links".to_string(), @@ -175,6 +198,9 @@ fn doublets_split_non_volatile_create(c: &mut Criterion) { // ===================== DELETE ===================== fn spacetimedb_delete(c: &mut Criterion) { + if !backend_enabled("spacetimedb") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = SpacetimeDbBenched::setup(()); c.bench_with_input( @@ -200,6 +226,9 @@ fn spacetimedb_delete(c: &mut Criterion) { } fn doublets_united_delete(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedVolatileBenched::setup(()); c.bench_with_input( @@ -225,6 +254,9 @@ fn doublets_united_delete(c: &mut Criterion) { } fn doublets_split_delete(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitVolatileBenched::setup(()); c.bench_with_input( @@ -250,6 +282,9 @@ fn doublets_split_delete(c: &mut Criterion) { } fn doublets_united_non_volatile_delete(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedNonVolatileBenched::setup( "/tmp/bench_united_non_volatile_delete.links".to_string(), @@ -277,6 +312,9 @@ fn doublets_united_non_volatile_delete(c: &mut Criterion) { } fn doublets_split_non_volatile_delete(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitNonVolatileBenched::setup(( "/tmp/bench_split_non_volatile_delete_data.links".to_string(), @@ -307,6 +345,9 @@ fn doublets_split_non_volatile_delete(c: &mut Criterion) { // ===================== UPDATE ===================== fn spacetimedb_update(c: &mut Criterion) { + if !backend_enabled("spacetimedb") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = SpacetimeDbBenched::setup(()); c.bench_with_input( @@ -335,6 +376,9 @@ fn spacetimedb_update(c: &mut Criterion) { } fn doublets_united_update(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedVolatileBenched::setup(()); c.bench_with_input( @@ -363,6 +407,9 @@ fn doublets_united_update(c: &mut Criterion) { } fn doublets_split_update(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitVolatileBenched::setup(()); c.bench_with_input( @@ -391,6 +438,9 @@ fn doublets_split_update(c: &mut Criterion) { } fn doublets_united_non_volatile_update(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedNonVolatileBenched::setup( "/tmp/bench_united_non_volatile_update.links".to_string(), @@ -421,6 +471,9 @@ fn doublets_united_non_volatile_update(c: &mut Criterion) { } fn doublets_split_non_volatile_update(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitNonVolatileBenched::setup(( "/tmp/bench_split_non_volatile_update_data.links".to_string(), @@ -454,6 +507,9 @@ fn doublets_split_non_volatile_update(c: &mut Criterion) { // ===================== QUERY ALL ===================== fn spacetimedb_query_all(c: &mut Criterion) { + if !backend_enabled("spacetimedb") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = SpacetimeDbBenched::setup(()); c.bench_with_input( @@ -479,6 +535,9 @@ fn spacetimedb_query_all(c: &mut Criterion) { } fn doublets_united_query_all(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedVolatileBenched::setup(()); c.bench_with_input( @@ -504,6 +563,9 @@ fn doublets_united_query_all(c: &mut Criterion) { } fn doublets_split_query_all(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitVolatileBenched::setup(()); c.bench_with_input( @@ -529,6 +591,9 @@ fn doublets_split_query_all(c: &mut Criterion) { } fn doublets_united_non_volatile_query_all(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedNonVolatileBenched::setup( "/tmp/bench_united_non_volatile_query_all.links".to_string(), @@ -556,6 +621,9 @@ fn doublets_united_non_volatile_query_all(c: &mut Criterion) { } fn doublets_split_non_volatile_query_all(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitNonVolatileBenched::setup(( "/tmp/bench_split_non_volatile_query_all_data.links".to_string(), @@ -586,6 +654,9 @@ fn doublets_split_non_volatile_query_all(c: &mut Criterion) { // ===================== QUERY BY ID ===================== fn spacetimedb_query_by_id(c: &mut Criterion) { + if !backend_enabled("spacetimedb") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = SpacetimeDbBenched::setup(()); c.bench_with_input( @@ -600,7 +671,7 @@ fn spacetimedb_query_by_id(c: &mut Criterion) { let ids: Vec = (0..n).map(|_| fork.create_point()).collect(); let start = Instant::now(); for id in ids { - let _ = fork.query_by_id(id); + std::hint::black_box(fork.query_by_id(id)); } total += start.elapsed(); } @@ -611,6 +682,9 @@ fn spacetimedb_query_by_id(c: &mut Criterion) { } fn doublets_united_query_by_id(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedVolatileBenched::setup(()); c.bench_with_input( @@ -625,7 +699,7 @@ fn doublets_united_query_by_id(c: &mut Criterion) { let ids: Vec = (0..n).map(|_| fork.create_point()).collect(); let start = Instant::now(); for id in ids { - let _ = fork.query_by_id(id); + std::hint::black_box(fork.query_by_id(id)); } total += start.elapsed(); } @@ -636,6 +710,9 @@ fn doublets_united_query_by_id(c: &mut Criterion) { } fn doublets_split_query_by_id(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitVolatileBenched::setup(()); c.bench_with_input( @@ -650,7 +727,7 @@ fn doublets_split_query_by_id(c: &mut Criterion) { let ids: Vec = (0..n).map(|_| fork.create_point()).collect(); let start = Instant::now(); for id in ids { - let _ = fork.query_by_id(id); + std::hint::black_box(fork.query_by_id(id)); } total += start.elapsed(); } @@ -661,6 +738,9 @@ fn doublets_split_query_by_id(c: &mut Criterion) { } fn doublets_united_non_volatile_query_by_id(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedNonVolatileBenched::setup( "/tmp/bench_united_non_volatile_query_by_id.links".to_string(), @@ -677,7 +757,7 @@ fn doublets_united_non_volatile_query_by_id(c: &mut Criterion) { let ids: Vec = (0..n).map(|_| fork.create_point()).collect(); let start = Instant::now(); for id in ids { - let _ = fork.query_by_id(id); + std::hint::black_box(fork.query_by_id(id)); } total += start.elapsed(); } @@ -688,6 +768,9 @@ fn doublets_united_non_volatile_query_by_id(c: &mut Criterion) { } fn doublets_split_non_volatile_query_by_id(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitNonVolatileBenched::setup(( "/tmp/bench_split_non_volatile_query_by_id_data.links".to_string(), @@ -705,7 +788,7 @@ fn doublets_split_non_volatile_query_by_id(c: &mut Criterion) { let ids: Vec = (0..n).map(|_| fork.create_point()).collect(); let start = Instant::now(); for id in ids { - let _ = fork.query_by_id(id); + std::hint::black_box(fork.query_by_id(id)); } total += start.elapsed(); } @@ -718,6 +801,9 @@ fn doublets_split_non_volatile_query_by_id(c: &mut Criterion) { // ===================== QUERY BY SOURCE ===================== fn spacetimedb_query_by_source(c: &mut Criterion) { + if !backend_enabled("spacetimedb") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = SpacetimeDbBenched::setup(()); c.bench_with_input( @@ -728,14 +814,25 @@ fn spacetimedb_query_by_source(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed sources for i in 1..=n as u64 { - fork.create(i % 10 + 1, i % 7 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + true, + ); + fork.create(source, target); } let start = Instant::now(); - for src in 1..=(n.min(10) as u64) { - let _ = fork.query_by_source(src); + for &src in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_source(src)); } total += start.elapsed(); } @@ -746,6 +843,9 @@ fn spacetimedb_query_by_source(c: &mut Criterion) { } fn doublets_united_query_by_source(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedVolatileBenched::setup(()); c.bench_with_input( @@ -756,14 +856,25 @@ fn doublets_united_query_by_source(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed sources for i in 1..=n as u64 { - fork.create(i % 10 + 1, i % 7 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + true, + ); + fork.create(source, target); } let start = Instant::now(); - for src in 1..=(n.min(10) as u64) { - let _ = fork.query_by_source(src); + for &src in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_source(src)); } total += start.elapsed(); } @@ -774,6 +885,9 @@ fn doublets_united_query_by_source(c: &mut Criterion) { } fn doublets_split_query_by_source(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitVolatileBenched::setup(()); c.bench_with_input( @@ -784,14 +898,25 @@ fn doublets_split_query_by_source(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed sources for i in 1..=n as u64 { - fork.create(i % 10 + 1, i % 7 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + true, + ); + fork.create(source, target); } let start = Instant::now(); - for src in 1..=(n.min(10) as u64) { - let _ = fork.query_by_source(src); + for &src in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_source(src)); } total += start.elapsed(); } @@ -802,6 +927,9 @@ fn doublets_split_query_by_source(c: &mut Criterion) { } fn doublets_united_non_volatile_query_by_source(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedNonVolatileBenched::setup( "/tmp/bench_united_non_volatile_query_by_source.links".to_string(), @@ -814,14 +942,25 @@ fn doublets_united_non_volatile_query_by_source(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed sources for i in 1..=n as u64 { - fork.create(i % 10 + 1, i % 7 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + true, + ); + fork.create(source, target); } let start = Instant::now(); - for src in 1..=(n.min(10) as u64) { - let _ = fork.query_by_source(src); + for &src in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_source(src)); } total += start.elapsed(); } @@ -832,6 +971,9 @@ fn doublets_united_non_volatile_query_by_source(c: &mut Criterion) { } fn doublets_split_non_volatile_query_by_source(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitNonVolatileBenched::setup(( "/tmp/bench_split_non_volatile_query_by_source_data.links".to_string(), @@ -845,14 +987,25 @@ fn doublets_split_non_volatile_query_by_source(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed sources for i in 1..=n as u64 { - fork.create(i % 10 + 1, i % 7 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + true, + ); + fork.create(source, target); } let start = Instant::now(); - for src in 1..=(n.min(10) as u64) { - let _ = fork.query_by_source(src); + for &src in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_source(src)); } total += start.elapsed(); } @@ -865,6 +1018,9 @@ fn doublets_split_non_volatile_query_by_source(c: &mut Criterion) { // ===================== QUERY BY TARGET ===================== fn spacetimedb_query_by_target(c: &mut Criterion) { + if !backend_enabled("spacetimedb") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = SpacetimeDbBenched::setup(()); c.bench_with_input( @@ -875,14 +1031,25 @@ fn spacetimedb_query_by_target(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed targets for i in 1..=n as u64 { - fork.create(i % 7 + 1, i % 10 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + false, + ); + fork.create(source, target); } let start = Instant::now(); - for tgt in 1..=(n.min(10) as u64) { - let _ = fork.query_by_target(tgt); + for &tgt in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_target(tgt)); } total += start.elapsed(); } @@ -893,6 +1060,9 @@ fn spacetimedb_query_by_target(c: &mut Criterion) { } fn doublets_united_query_by_target(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedVolatileBenched::setup(()); c.bench_with_input( @@ -903,14 +1073,25 @@ fn doublets_united_query_by_target(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed targets for i in 1..=n as u64 { - fork.create(i % 7 + 1, i % 10 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + false, + ); + fork.create(source, target); } let start = Instant::now(); - for tgt in 1..=(n.min(10) as u64) { - let _ = fork.query_by_target(tgt); + for &tgt in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_target(tgt)); } total += start.elapsed(); } @@ -921,6 +1102,9 @@ fn doublets_united_query_by_target(c: &mut Criterion) { } fn doublets_split_query_by_target(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitVolatileBenched::setup(()); c.bench_with_input( @@ -931,14 +1115,25 @@ fn doublets_split_query_by_target(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed targets for i in 1..=n as u64 { - fork.create(i % 7 + 1, i % 10 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + false, + ); + fork.create(source, target); } let start = Instant::now(); - for tgt in 1..=(n.min(10) as u64) { - let _ = fork.query_by_target(tgt); + for &tgt in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_target(tgt)); } total += start.elapsed(); } @@ -949,6 +1144,9 @@ fn doublets_split_query_by_target(c: &mut Criterion) { } fn doublets_united_non_volatile_query_by_target(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsUnitedNonVolatileBenched::setup( "/tmp/bench_united_non_volatile_query_by_target.links".to_string(), @@ -961,14 +1159,25 @@ fn doublets_united_non_volatile_query_by_target(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed targets for i in 1..=n as u64 { - fork.create(i % 7 + 1, i % 10 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + false, + ); + fork.create(source, target); } let start = Instant::now(); - for tgt in 1..=(n.min(10) as u64) { - let _ = fork.query_by_target(tgt); + for &tgt in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_target(tgt)); } total += start.elapsed(); } @@ -979,6 +1188,9 @@ fn doublets_united_non_volatile_query_by_target(c: &mut Criterion) { } fn doublets_split_non_volatile_query_by_target(c: &mut Criterion) { + if !backend_enabled("doublets") { + return; + } let count = *BENCHMARK_LINK_COUNT; let mut benched = DoubletsSplitNonVolatileBenched::setup(( "/tmp/bench_split_non_volatile_query_by_target_data.links".to_string(), @@ -992,14 +1204,25 @@ fn doublets_split_non_volatile_query_by_target(c: &mut Criterion) { let mut total = Duration::ZERO; for _ in 0..iters { let mut fork = Benched::fork(&mut benched); - setup_background!(fork); + let background_ids: Vec = (0..*BACKGROUND_LINK_COUNT) + .map(|_| fork.create_point()) + .collect(); + assert!( + background_ids.len() >= 10 + n.div_ceil(10), + "query workloads require at least 10 + ceil(N/10) background links" + ); // Create links with distributed targets for i in 1..=n as u64 { - fork.create(i % 7 + 1, i % 10 + 1); + let (source, target) = spacetimedb_vs_doublets::workload::distributed_link( + (i - 1) as usize, + &background_ids, + false, + ); + fork.create(source, target); } let start = Instant::now(); - for tgt in 1..=(n.min(10) as u64) { - let _ = fork.query_by_target(tgt); + for &tgt in background_ids.iter().take(n.min(10)) { + std::hint::black_box(fork.query_by_target(tgt)); } total += start.elapsed(); } diff --git a/rust/doublets-patched/PATCHES.md b/rust/doublets-patched/PATCHES.md index 750bb2c..3326a56 100644 --- a/rust/doublets-patched/PATCHES.md +++ b/rust/doublets-patched/PATCHES.md @@ -26,8 +26,20 @@ The `doublets` crate (and its internal dependencies `platform-mem`, `platform-da The `doublets` crate is not actively maintained for compatibility with current Rust nightly. Attempts to use the published crates.io version (or the upstream git repository) result in compilation failures on Rust nightly ≥ 1.80.0. Upstream issues have been filed but not addressed. -The patches are minimal and surgical — they do not change any algorithmic behavior, data structures, or storage semantics. The only changes are the removal of feature flags and the replacement of removed syntax with equivalent stable alternatives. +The compatibility patches remove feature flags and replace removed syntax with +equivalent alternatives. The Split endpoint-classification correction below is +also required for the shared update workload to behave correctly. ## Future Resolution Once the upstream `doublets-rs` crates are updated for modern Rust nightly compatibility, the `doublets-patched/` directory can be removed and the dependency replaced with a direct crates.io reference. This is tracked in the [doublets-rs repository](https://github.com/linksplatform/doublets-rs). + +### Split store endpoint classification (issue #16) + +`Store::update_links` now classifies the new source and target before detaching +from the index trees. Detaching a self-referential point temporarily clears its +target-tree size; `is_unused` then incorrectly classifies that live point as an +external endpoint. The next update tries to detach from the wrong tree and can +underflow its size. `rust/tests/same_behavior.rs` reproduces the failure with +12 points, indexed updates, and resets to `(0, 0)`; it checks both Split layouts +against United and SpacetimeDB. Optional `BENCHMARK_TRACE=1` prints each update. diff --git a/rust/doublets-patched/doublets/src/mem/split/store.rs b/rust/doublets-patched/doublets/src/mem/split/store.rs index 628f7c4..f53b14f 100644 --- a/rust/doublets-patched/doublets/src/mem/split/store.rs +++ b/rust/doublets-patched/doublets/src/mem/split/store.rs @@ -748,6 +748,11 @@ impl< let new_target = change[2]; let link = self.try_get_link(index)?; + // Classify endpoints while the link is still indexed. Detaching a point + // temporarily sets its target-tree size to zero, which is_unused treats + // as a free link. A self-reference must stay in the internal trees. + let virtual_source = self.is_virtual(new_source); + let virtual_target = self.is_virtual(new_target); if link.source != T::funty(0) { // SAFETY: Here index attach to source @@ -774,8 +779,6 @@ impl< } } - let virtual_source = self.is_virtual(new_source); - let virtual_target = self.is_virtual(new_target); let place = self.mut_data_part(index); place.source = new_source; place.target = new_target; diff --git a/rust/spacetime-module/Cargo.toml b/rust/spacetime-module/Cargo.toml index b1c28a5..9b4929d 100644 --- a/rust/spacetime-module/Cargo.toml +++ b/rust/spacetime-module/Cargo.toml @@ -9,5 +9,5 @@ description = "SpacetimeDB module defining the links table and reducers for benc crate-type = ["cdylib"] [dependencies] -spacetimedb = "2" +spacetimedb = "=2.10.1" log = "0.4" diff --git a/rust/src/benched/doublets_benched.rs b/rust/src/benched/doublets_benched.rs index 7952d67..0a0eeb4 100644 --- a/rust/src/benched/doublets_benched.rs +++ b/rust/src/benched/doublets_benched.rs @@ -25,7 +25,7 @@ impl Benched for DoubletsUnitedVolatileBenched { } } - fn fork(&mut self) -> Fork { + fn fork(&mut self) -> Fork<'_, Self> { Fork::new(self) } @@ -62,7 +62,7 @@ impl Benched for DoubletsSplitVolatileBenched { } } - fn fork(&mut self) -> Fork { + fn fork(&mut self) -> Fork<'_, Self> { Fork::new(self) } @@ -106,7 +106,7 @@ impl Benched for DoubletsUnitedNonVolatileBenched { } } - fn fork(&mut self) -> Fork { + fn fork(&mut self) -> Fork<'_, Self> { Fork::new(self) } @@ -161,7 +161,7 @@ impl Benched for DoubletsSplitNonVolatileBenched { } } - fn fork(&mut self) -> Fork { + fn fork(&mut self) -> Fork<'_, Self> { Fork::new(self) } diff --git a/rust/src/benched/mod.rs b/rust/src/benched/mod.rs index 449a1a3..ea96e3e 100644 --- a/rust/src/benched/mod.rs +++ b/rust/src/benched/mod.rs @@ -26,7 +26,7 @@ pub trait Benched: Sized { fn setup(builder: Self::Builder) -> Self; /// Create a fork for a single isolated benchmark iteration. - fn fork(&mut self) -> Fork; + fn fork(&mut self) -> Fork<'_, Self>; /// Reset database state after an iteration (called by `Fork::drop`). /// diff --git a/rust/src/benched/spacetimedb_benched.rs b/rust/src/benched/spacetimedb_benched.rs index 634d07e..9c2076c 100644 --- a/rust/src/benched/spacetimedb_benched.rs +++ b/rust/src/benched/spacetimedb_benched.rs @@ -20,7 +20,7 @@ impl Benched for SpacetimeDbBenched { } } - fn fork(&mut self) -> Fork { + fn fork(&mut self) -> Fork<'_, Self> { Fork::new(self) } diff --git a/rust/src/doublets_impl.rs b/rust/src/doublets_impl.rs index 1706879..c5bf5df 100644 --- a/rust/src/doublets_impl.rs +++ b/rust/src/doublets_impl.rs @@ -67,7 +67,7 @@ impl DoubletsLinks { impl + DoubletsExt> Links for DoubletsLinks { fn create(&mut self, source: u64, target: u64) -> u64 { self.store - .create_by([source as usize, target as usize]) + .create_link(source as usize, target as usize) .expect("Failed to create link") as u64 } @@ -184,6 +184,21 @@ pub fn create_split_non_volatile( mod tests { use super::*; + #[test] + fn create_preserves_endpoints() { + for mut db in [ + Box::new(create_united_volatile()) as Box, + Box::new(create_split_volatile()), + ] { + let source = db.create_point(); + let target = db.create_point(); + let id = db.create(source, target); + assert_eq!(db.query_by_id(id), Some(Link::new(id, source, target))); + assert_eq!(db.query_by_source(source).len(), 2); + assert_eq!(db.query_by_target(target).len(), 2); + } + } + #[test] fn test_create_and_query_united() { let mut db = create_united_volatile(); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index e957413..94c0e99 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -11,6 +11,7 @@ pub mod exclusive; pub mod fork; pub mod module_bindings; pub mod spacetimedb_impl; +pub mod workload; pub use benched::Benched; pub use exclusive::Exclusive; diff --git a/rust/src/workload.rs b/rust/src/workload.rs new file mode 100644 index 0000000..81ad1d3 --- /dev/null +++ b/rust/src/workload.rs @@ -0,0 +1,17 @@ +//! Shared dataset for the source/target query benchmarks and differential tests. + +/// Return a unique pair referencing existing background links. +/// +/// The first ten background links are query keys. Subsequent links provide the +/// other endpoint, so pairs never duplicate the background points or each other. +/// Use actual IDs because SpacetimeDB's sequence survives `delete_all`. +#[must_use] +pub fn distributed_link(index: usize, background: &[u64], by_source: bool) -> (u64, u64) { + let key = background[index % 10]; + let other = background[index / 10 + 10]; + if by_source { + (key, other) + } else { + (other, key) + } +} diff --git a/rust/tests/same_behavior.rs b/rust/tests/same_behavior.rs new file mode 100644 index 0000000..20ea4ec --- /dev/null +++ b/rust/tests/same_behavior.rs @@ -0,0 +1,153 @@ +//! Differential checks for every benchmark backend. Server tests run explicitly in CI. +use spacetimedb_vs_doublets::{doublets_impl::*, spacetimedb_impl::SpacetimeDbLinks, Link, Links}; + +fn sorted(mut links: Vec) -> Vec { + links.sort_by_key(|link| (link.id, link.source, link.target)); + links +} + +// IDs are auto-incremented by SpacetimeDB and reused by Doublets. Compare identities +// by creation order, while preserving source/target relationships between links. +fn exercise(db: &mut dyn Links) -> Vec> { + db.delete_all(); + let ids: Vec<_> = (0..12).map(|_| db.create_point()).collect(); + let normalize = |links: Vec| { + let identity = |id| { + ids.iter() + .position(|&candidate| candidate == id) + .map_or(id, |i| i as u64 + 1) + }; + sorted( + links + .into_iter() + .map(|link| { + Link::new( + identity(link.id), + identity(link.source), + identity(link.target), + ) + }) + .collect(), + ) + }; + assert_eq!(db.count(), 12); + let mut trace = vec![normalize(db.query_all())]; + for &id in &ids { + let point = Link::new(id, id, id); + assert_eq!(db.query_by_id(id), Some(point)); + assert_eq!(db.query_by_source(id), vec![point]); + assert_eq!(db.query_by_target(id), vec![point]); + } + for (i, &id) in ids.iter().enumerate() { + if std::env::var_os("BENCHMARK_TRACE").is_some() { + eprintln!("distributed update {id}"); + } + db.update(id, ids[i % 3], ids[i % 5]); + } + trace.push(normalize(db.query_all())); + for &id in &ids { + trace.push(normalize(db.query_by_id(id).into_iter().collect())); + trace.push(normalize(db.query_by_source(id))); + trace.push(normalize(db.query_by_target(id))); + trace.push(normalize(db.query_by_source_target(id, ids[0]))); + } + assert_eq!(db.query_by_id(u64::MAX - 1), None); + assert!(db.query_by_source(99999).is_empty()); + assert!(db.query_by_target(99999).is_empty()); + // The benchmark updates each point to (0, 0), then restores it. + for &id in &ids { + if std::env::var_os("BENCHMARK_TRACE").is_some() { + eprintln!("reset update {id}"); + } + db.update(id, 0, 0); + } + trace.push(normalize(db.query_all())); + for &id in &ids { + db.update(id, id, id); + } + for &id in ids.iter().rev().take(6) { + db.delete(id); + } + assert_eq!(db.count(), 6); + trace.push(normalize(db.query_all())); + db.delete_all(); + assert_eq!(db.count(), 0); + assert!(db.query_all().is_empty()); + trace +} + +fn distributed_queries(db: &mut dyn Links) -> Vec> { + let mut trace = Vec::new(); + // Two repetitions exercise ID sequences that do not restart at one. + for _ in 0..2 { + for by_source in [true, false] { + db.delete_all(); + let mut ids: Vec = (0..30).map(|_| db.create_point()).collect(); + let background = ids.clone(); + for i in 0..100 { + let (source, target) = + spacetimedb_vs_doublets::workload::distributed_link(i, &background, by_source); + ids.push(db.create(source, target)); + } + assert_eq!(db.count(), 130); + for &key in &background[..10] { + let rows = if by_source { + db.query_by_source(key) + } else { + db.query_by_target(key) + }; + assert_eq!(rows.len(), 11); // one point plus ten distributed links + let identity = |id| ids.iter().position(|&value| value == id).unwrap() as u64 + 1; + trace.push(sorted( + rows.into_iter() + .map(|row| { + Link::new(identity(row.id), identity(row.source), identity(row.target)) + }) + .collect(), + )); + } + } + } + db.delete_all(); + trace +} + +fn full_trace(db: &mut dyn Links) -> Vec> { + let mut trace = exercise(db); + trace.extend(distributed_queries(db)); + trace +} + +fn doublets_traces() -> Vec>> { + let directory = std::env::temp_dir().join(format!("same-behavior-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let path = |name| directory.join(name).to_str().unwrap().to_string(); + let traces = vec![ + full_trace(&mut create_united_volatile()), + full_trace(&mut create_split_volatile()), + full_trace(&mut create_united_non_volatile(&path("united"))), + full_trace(&mut create_split_non_volatile( + &path("data"), + &path("index"), + )), + ]; + std::fs::remove_dir_all(directory).unwrap(); + traces +} + +#[test] +fn all_doublets_variants_return_same_results() { + let traces = doublets_traces(); + for trace in &traces[1..] { + assert_eq!(trace, &traces[0]); + } +} + +#[test] +#[ignore = "requires a published SpacetimeDB module; CI runs with --include-ignored"] +fn spacetimedb_and_all_doublets_variants_return_same_results() { + let actual = full_trace(&mut SpacetimeDbLinks::connect()); + for trace in doublets_traces() { + assert_eq!(trace, actual); + } +} From c367778588a7ff2b3ac5b379f17da3acc972a760 Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:37:47 +0000 Subject: [PATCH 3/7] Add C# SDK and Doublets benchmarks with shared behavior tests --- .../HarnessTests.cs | 31 + .../SameBehaviorTests.cs | 130 ++++ .../SpacetimeDBVSDoublets.Tests.csproj | 14 + .../packages.lock.json | 324 +++++++++ csharp/SpacetimeDBVSDoublets.slnx | 4 + csharp/SpacetimeDBVSDoublets/Benchmarks.cs | 64 ++ csharp/SpacetimeDBVSDoublets/DoubletsLinks.cs | 110 +++ csharp/SpacetimeDBVSDoublets/Harness.cs | 93 +++ csharp/SpacetimeDBVSDoublets/ILinks.cs | 19 + .../ModuleBindings/Reducers/CreateLink.g.cs | 73 ++ .../Reducers/DeleteAllLinks.g.cs | 53 ++ .../ModuleBindings/Reducers/DeleteLink.g.cs | 66 ++ .../ModuleBindings/Reducers/UpdateLink.g.cs | 78 +++ .../ModuleBindings/SpacetimeDBClient.g.cs | 631 ++++++++++++++++++ .../ModuleBindings/Tables/Links.g.cs | 87 +++ .../ModuleBindings/Types/Link.g.cs | 38 ++ csharp/SpacetimeDBVSDoublets/Program.cs | 28 + .../SpacetimeDBVSDoublets.csproj | 14 + .../SpacetimeDBVSDoublets/SpacetimeLinks.cs | 99 +++ .../SpacetimeDBVSDoublets/packages.lock.json | 192 ++++++ csharp/global.json | 6 + 21 files changed, 2154 insertions(+) create mode 100644 csharp/SpacetimeDBVSDoublets.Tests/HarnessTests.cs create mode 100644 csharp/SpacetimeDBVSDoublets.Tests/SameBehaviorTests.cs create mode 100644 csharp/SpacetimeDBVSDoublets.Tests/SpacetimeDBVSDoublets.Tests.csproj create mode 100644 csharp/SpacetimeDBVSDoublets.Tests/packages.lock.json create mode 100644 csharp/SpacetimeDBVSDoublets.slnx create mode 100644 csharp/SpacetimeDBVSDoublets/Benchmarks.cs create mode 100644 csharp/SpacetimeDBVSDoublets/DoubletsLinks.cs create mode 100644 csharp/SpacetimeDBVSDoublets/Harness.cs create mode 100644 csharp/SpacetimeDBVSDoublets/ILinks.cs create mode 100644 csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/CreateLink.g.cs create mode 100644 csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/DeleteAllLinks.g.cs create mode 100644 csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/DeleteLink.g.cs create mode 100644 csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/UpdateLink.g.cs create mode 100644 csharp/SpacetimeDBVSDoublets/ModuleBindings/SpacetimeDBClient.g.cs create mode 100644 csharp/SpacetimeDBVSDoublets/ModuleBindings/Tables/Links.g.cs create mode 100644 csharp/SpacetimeDBVSDoublets/ModuleBindings/Types/Link.g.cs create mode 100644 csharp/SpacetimeDBVSDoublets/Program.cs create mode 100644 csharp/SpacetimeDBVSDoublets/SpacetimeDBVSDoublets.csproj create mode 100644 csharp/SpacetimeDBVSDoublets/SpacetimeLinks.cs create mode 100644 csharp/SpacetimeDBVSDoublets/packages.lock.json create mode 100644 csharp/global.json diff --git a/csharp/SpacetimeDBVSDoublets.Tests/HarnessTests.cs b/csharp/SpacetimeDBVSDoublets.Tests/HarnessTests.cs new file mode 100644 index 0000000..f283a35 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets.Tests/HarnessTests.cs @@ -0,0 +1,31 @@ +using Comparisons.SpacetimeDBVSDoublets; +using Xunit; + +namespace Comparisons.SpacetimeDBVSDoublets.Tests; + +public sealed class HarnessTests +{ + [Fact] + public void StatisticsMatchKnownSamples() + { + var estimate = Estimate.Of([10, 20, 30]); + Assert.Equal(20, estimate.Median); + Assert.Equal(10, estimate.StandardDeviation); + Assert.Contains("test create/SpacetimeDB/10 ... bench:", estimate.Bencher("create", "SpacetimeDB/10")); + Assert.Contains("ns/iter (+/- 10)", estimate.Bencher("create", "SpacetimeDB/10")); + Assert.Equal(25, Estimate.Of([10, 20, 30, 40]).Median); + } + + [Fact] + public void SamplingUsesCriterionLinearIterationCounts() + { + var sampling = new Sampling(10, SamplingMode.Linear, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2)); + Assert.Equal(Enumerable.Range(1, 10).Select(i => i * 37L), sampling.IterationCounts(1_000_000)); + } + + [Fact] + public void SlowIterationsUseCriterionAutomaticFlatSampling() + { + Assert.Equal(Enumerable.Repeat(2L, 10), Sampling.Quick.IterationCounts(100_000_000)); + } +} diff --git a/csharp/SpacetimeDBVSDoublets.Tests/SameBehaviorTests.cs b/csharp/SpacetimeDBVSDoublets.Tests/SameBehaviorTests.cs new file mode 100644 index 0000000..0bb9a7e --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets.Tests/SameBehaviorTests.cs @@ -0,0 +1,130 @@ +using Comparisons.SpacetimeDBVSDoublets; +using Xunit; + +namespace Comparisons.SpacetimeDBVSDoublets.Tests; + +public sealed class SameBehaviorTests +{ + private static List Exercise(ILinks store) + { + store.DeleteAll(); + var ids = Enumerable.Range(0, 12).Select(_ => store.CreatePoint()).ToArray(); + ulong Normalize(ulong id) => Array.IndexOf(ids, id) is var index && index >= 0 ? (ulong)index + 1 : id; + Link[] Snapshot(IEnumerable rows) => rows.Select(row => new Link(Normalize(row.Id), Normalize(row.Source), Normalize(row.Target))) + .OrderBy(row => row.Id).ToArray(); + Assert.Equal(12UL, store.Count()); + var trace = new List { Snapshot(store.QueryAll()) }; + foreach (var id in ids) + { + var point = new Link(id, id, id); + Assert.Equal(point, store.QueryById(id)); + Assert.Equal([point], store.QueryBySource(id)); + Assert.Equal([point], store.QueryByTarget(id)); + } + for (var i = 0; i < ids.Length; i++) store.Update(ids[i], ids[i % 3], ids[i % 5]); + trace.Add(Snapshot(store.QueryAll())); + foreach (var id in ids) + { + trace.Add(Snapshot([store.QueryById(id)!.Value])); + trace.Add(Snapshot(store.QueryBySource(id))); + trace.Add(Snapshot(store.QueryByTarget(id))); + trace.Add(Snapshot(store.QueryBySourceTarget(id, ids[0]))); + } + Assert.Null(store.QueryById(ulong.MaxValue - 1)); + Assert.Empty(store.QueryBySource(99999)); + Assert.Empty(store.QueryByTarget(99999)); + foreach (var id in ids) store.Update(id, 0, 0); + trace.Add(Snapshot(store.QueryAll())); + foreach (var id in ids) store.Update(id, id, id); + foreach (var id in ids.Reverse().Take(6)) store.Delete(id); + Assert.Equal(6UL, store.Count()); + trace.Add(Snapshot(store.QueryAll())); + store.DeleteAll(); + Assert.Empty(store.QueryAll()); + Assert.Equal(0UL, store.Count()); + return trace; + } + + private static List DistributedQueries(ILinks store) + { + var trace = new List(); + for (var repetition = 0; repetition < 2; repetition++) + foreach (var bySource in new[] { true, false }) + { + store.DeleteAll(); + var ids = Enumerable.Range(0, 30).Select(_ => store.CreatePoint()).ToList(); + var background = ids.ToArray(); + for (var i = 0; i < 100; i++) + { + var pair = Benchmarks.DistributedLink(i, background, bySource); + ids.Add(store.Create(pair.Source, pair.Target)); + } + Assert.Equal(130UL, store.Count()); + foreach (var key in background.Take(10)) + { + var rows = bySource ? store.QueryBySource(key) : store.QueryByTarget(key); + Assert.Equal(11, rows.Length); + ulong Identity(ulong id) => (ulong)ids.IndexOf(id) + 1; + trace.Add(rows.Select(row => new Link(Identity(row.Id), Identity(row.Source), Identity(row.Target))).OrderBy(row => row.Id).ToArray()); + } + } + store.DeleteAll(); + return trace; + } + + private static List FullTrace(ILinks store) + { + var trace = Exercise(store); + trace.AddRange(DistributedQueries(store)); + return trace; + } + + [Fact] + public void AllBackendsReturnSameResults() + { + var directory = Path.Combine(Path.GetTempPath(), $"same-behavior-{Guid.NewGuid()}"); + Directory.CreateDirectory(directory); + try + { + List? expected = null; + foreach (var variant in DoubletsLinks.Variants) + { + using var store = DoubletsLinks.Open(variant, directory); + var actual = FullTrace(store); + expected ??= actual; + for (var i = 0; i < expected.Count; i++) Assert.Equal(expected[i], actual[i]); + } + if (Environment.GetEnvironmentVariable("SPACETIMEDB_URI") is not null) + { + using var store = new SpacetimeLinks(); + var actual = FullTrace(store); + for (var i = 0; i < expected!.Count; i++) Assert.Equal(expected[i], actual[i]); + } + } + finally { Directory.Delete(directory, true); } + } + + [Fact] + public void EveryBenchmarkIterationResetsTheStore() + { + var directory = Path.Combine(Path.GetTempPath(), $"iterations-{Guid.NewGuid()}"); + Directory.CreateDirectory(directory); + try + { + var variants = DoubletsLinks.Variants.Concat(Environment.GetEnvironmentVariable("SPACETIMEDB_URI") is null + ? Array.Empty() : ["SpacetimeDB"]); + foreach (var variant in variants) + { + using ILinks store = variant == "SpacetimeDB" ? new SpacetimeLinks() : DoubletsLinks.Open(variant, directory); + store.DeleteAll(); + foreach (var operation in Benchmarks.Operations) + for (var i = 0; i < 2; i++) + { + Benchmarks.Iteration(store, operation, 10, 30); + Assert.Equal(0UL, store.Count()); + } + } + } + finally { Directory.Delete(directory, true); } + } +} diff --git a/csharp/SpacetimeDBVSDoublets.Tests/SpacetimeDBVSDoublets.Tests.csproj b/csharp/SpacetimeDBVSDoublets.Tests/SpacetimeDBVSDoublets.Tests.csproj new file mode 100644 index 0000000..38df9b1 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets.Tests/SpacetimeDBVSDoublets.Tests.csproj @@ -0,0 +1,14 @@ + + + Exe + net10.0 + enable + enable + true + true + + + + + + diff --git a/csharp/SpacetimeDBVSDoublets.Tests/packages.lock.json b/csharp/SpacetimeDBVSDoublets.Tests/packages.lock.json new file mode 100644 index 0000000..924916e --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets.Tests/packages.lock.json @@ -0,0 +1,324 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "xunit.v3": { + "type": "Direct", + "requested": "[4.0.1, )", + "resolved": "4.0.1", + "contentHash": "KpDYf6jFjYfOchuRJOAXPOpwN03hLE1AF4kWzCUSnhr8JZMiGH4BT1YidCcm6BE1H4E90GyJzuKoCwXbg9PXmA==", + "dependencies": { + "xunit.v3.mtp-v2": "[4.0.1]" + } + }, + "Microsoft.ApplicationInsights": { + "type": "Transitive", + "resolved": "2.23.0", + "contentHash": "nWArUZTdU7iqZLycLKWe0TDms48KKGE6pONH2terYNa8REXiqixrMOkf1sk5DHGMaUTqONU2YkS4SAXBhLStgw==" + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg==" + }, + "Microsoft.Testing.Extensions.Telemetry": { + "type": "Transitive", + "resolved": "2.4.0", + "contentHash": "JeP1RFqBa11fWmBk8xEfZcMKr4rxWSyI6OZ+659V069CaMkTEOQBW2UdSSeNz3absOsygcn7JJkzerC4LGnZ9w==", + "dependencies": { + "Microsoft.ApplicationInsights": "2.23.0", + "Microsoft.Testing.Platform": "[2.4.0, 3.0.0)" + } + }, + "Microsoft.Testing.Extensions.TrxReport.Abstractions": { + "type": "Transitive", + "resolved": "2.4.0", + "contentHash": "uRb+4qM42dFDg4kWJZ2kFEcwESNVCRZJjItp5vETorN3rSJGysP617LqYirOcrCYmxg+obPreHMM5JcsNDKtBw==", + "dependencies": { + "Microsoft.Testing.Platform": "[2.4.0, 3.0.0)" + } + }, + "Microsoft.Testing.Platform": { + "type": "Transitive", + "resolved": "2.4.0", + "contentHash": "dp1N3P1ujb0ztwFgqz2o/ItEvq+pm19/AiA+Xq7Zpcy7oPMcxDBZLYGtf0LlU1MveEBJuKVjZhI+LnkfcH/0jQ==" + }, + "Microsoft.Testing.Platform.MSBuild": { + "type": "Transitive", + "resolved": "2.4.0", + "contentHash": "qr5M6h16YHMJLFDcWELFVMMpGte2BUmveBZKT5YoBV+bmuJRPu9bv/Zqke4yQuOEKRNxoAETrG5jr+/6Rnr3Hg==", + "dependencies": { + "Microsoft.Testing.Platform": "[2.4.0, 3.0.0)" + } + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==" + }, + "Platform.Collections": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "ofU+XTf0CJry3dnpSq5pdiz8cArZDlrd3IaGtYoV8L6e+OhTdx2n544s+XLOZK90KJUsNv/docu2Y97Asdbvuw==", + "dependencies": { + "Platform.Disposables": "0.4.0", + "Platform.Interfaces": "0.5.0", + "Platform.Random": "0.3.0", + "Platform.Ranges": "0.5.0" + } + }, + "Platform.Collections.Methods": { + "type": "Transitive", + "resolved": "0.7.3", + "contentHash": "GG+XkIiZWlg96w9HxBeCeGrWdXr6+ST8L8piPjvC4lVrB0XaJNWWj+FYSnQOcX7MZOXs70Y6+e5eD0t1GygB5g==", + "dependencies": { + "Platform.Numbers": "0.9.0" + } + }, + "Platform.Converters": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "Rz4gLL2zPB/KsqORljmhMaQR8EVcTBnRRO2rz91F/nu+qJXuzpurx4VgtnL0lyso0NfkmNJffTiLucxwYX9pWQ==", + "dependencies": { + "Platform.Reflection": "0.7.0" + } + }, + "Platform.Data": { + "type": "Transitive", + "resolved": "0.16.1", + "contentHash": "1JdpgdWbowNgVx+aJatRyTEL5CS+hB5mjOJD57+R26XfbCpIsscJwL+Kcmk/ivaop6OABd3SX7M2DaAhYHXKtw==", + "dependencies": { + "Platform.Delegates": "0.1.1", + "Platform.Numbers": "0.9.0", + "Platform.Setters": "0.5.0", + "Platform.Threading": "0.4.0" + } + }, + "Platform.Data.Doublets": { + "type": "Transitive", + "resolved": "0.18.1", + "contentHash": "b6p7GgLDonEnRdhuy3Xsn/N5g5Qq+aivapAlkFH/Mk/MDTDiSJ0C1wnViC4Ff9/d6IXfmh2Sv/TeGIkyKbTOTw==", + "dependencies": { + "Platform.Collections.Methods": "0.7.3", + "Platform.Data": "0.16.1", + "Platform.Data.Doublets.FFI": "0.9.0", + "Platform.Memory": "0.4.1", + "Platform.Random": "0.3.0", + "Platform.Singletons": "0.3.0", + "Platform.Timestamps": "0.3.0" + } + }, + "Platform.Data.Doublets.FFI": { + "type": "Transitive", + "resolved": "0.9.0", + "contentHash": "NRgGyihV3sdmvNW9SLnwBV14zkq89GJlTgyeiB+EKT2cMOkCjZtd55C/M0Sy5tLkZ076rBcup4jLFKmejnS2tg==" + }, + "Platform.Delegates": { + "type": "Transitive", + "resolved": "0.1.1", + "contentHash": "+182klZgGEJ+yWtxz3BMrTCYcpSXjvJzrYzpK+W2UecAaDKLvLyehjPlMfta+dM/XlKbPt53Wf20vhI1kul0SQ==" + }, + "Platform.Disposables": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "ko0W4cw5XhNxJANdO1T8srpWRVqpPiXtCohumlbTu33kivRBEmk1ivDsySaNhKTyuYjJnrjRYrgA1iEb2sS+/Q==", + "dependencies": { + "Platform.Exceptions": "0.5.0" + } + }, + "Platform.Exceptions": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "htu3iyPVJinbLxU1I1cpc2qFlur7IF2fTc9u2jqQ+3X1Lyzd3aeK9Y7gJu9ZkdnIO3tbdO6wxr1HZG4UhaIG8A==" + }, + "Platform.Interfaces": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "OBViyjgCYWaXH8xhibYL/fbwHfPP/+qVUDxcFGH22IaYzbrFbfNR5TxW6gsk3LR9QXLANnrLsuI7phZueoj/Tw==" + }, + "Platform.IO": { + "type": "Transitive", + "resolved": "0.5.1", + "contentHash": "MZ+uZIrDZuNZZu8J8gv8RcXwGihSTEo9UvFoRHCGIXQnaQxnJ0gBIF5s/zSjJ1+h8c78lv/wmlSvh4zMzwyJIQ==", + "dependencies": { + "Platform.Threading": "0.5.0", + "Platform.Unsafe": "0.6.1" + } + }, + "Platform.Memory": { + "type": "Transitive", + "resolved": "0.4.1", + "contentHash": "Jz4Xb0B8pmh6BU9hhvqMgK2L+R0fCxSk1bGWL7WK/WYKAYSuGaJUAvzWdJLTECyfs+w6PVUxXK53q1OXxHRmMQ==", + "dependencies": { + "Platform.IO": "0.5.1" + } + }, + "Platform.Numbers": { + "type": "Transitive", + "resolved": "0.9.0", + "contentHash": "0GgZSH0p+jP933DKtLzCfOAdkJ1cNKrSrisXgMKy0zTMBZOxMxzs83IVXwBju66vm5N/w6lFq8HwCLW0gQsYVA==", + "dependencies": { + "Platform.Converters": "0.4.0" + } + }, + "Platform.Random": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "GX24Xx45cUIr9EZl55evdwb7oYN/nimpoNqDG4zoMDDLqeEpL35zX8IiHHDYITV9F4bjrCrSEiRCVsRFAsi7SQ==", + "dependencies": { + "Platform.Ranges": "0.5.0" + } + }, + "Platform.Ranges": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "CUhZM/4F714NAyrtWtkBx78L4xJpFZ1wp+6ivFvpAXx6DsaBcBsyDxjn5eoLB9LSUYxB11gFkwwmXmJhMgZ3qw==", + "dependencies": { + "Platform.Exceptions": "0.5.0" + } + }, + "Platform.Reflection": { + "type": "Transitive", + "resolved": "0.7.0", + "contentHash": "CvpRnDWH5YArcXlY7lcY0mDYQCtnJAeZsqZ4/IdllJoj7z8JwTufeo/zK62Vn6K0ulTyHNhb+/T3HFusxnrfxA==", + "dependencies": { + "Platform.Collections": "0.4.0" + } + }, + "Platform.Setters": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "vZG+VnYc4u2x5vnBY7rqaFHtOS2AeoLwGV+02JBFa9pxluHwVTu4nVcVBTJTlenZSfAoZUNVsV1StnUWvINkCQ==", + "dependencies": { + "Platform.Delegates": "0.1.1", + "Platform.Interfaces": "0.5.0" + } + }, + "Platform.Singletons": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "fCfNw4CqPL6HzsUJVVYI/OLYXGkjmdDCe+QWThW8hPoNNlqMZ+FWbfBXbXpCuDYw9EYhLDlhuTAQMbqNt7kCzg==", + "dependencies": { + "Platform.Reflection": "0.7.0" + } + }, + "Platform.Threading": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "Q0p5oVDbhQ76IGHi/vYIX+de9T270XZmNgjNbHtobc+AoJbIWGf+kZXTzq+MmEfXS2ANtGNe9uWN1a7Nzp7olQ==", + "dependencies": { + "Platform.Collections": "0.4.0" + } + }, + "Platform.Timestamps": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "Lmhm8Vp0DobTlxSVTPZZE+3ABTrvuofcmTQOWZv6TyiZ5NyFBH6m61vZdnIkMa/3lQguKvGfgMsUT97p0t5x7w==", + "dependencies": { + "Platform.Interfaces": "0.5.0" + } + }, + "Platform.Unsafe": { + "type": "Transitive", + "resolved": "0.6.1", + "contentHash": "GbKgPC7lfw+q2GqQEch0NSxHOZXxoLTj3ChjdRZIQBvZnGWzYH6nat8glfTJseOmgdabIxlwogFECoai0O3akA==", + "dependencies": { + "Platform.Numbers": "0.7.0" + } + }, + "SpacetimeDB.BSATN.Runtime": { + "type": "Transitive", + "resolved": "2.10.1", + "contentHash": "EvVdnyTOy+qK1OOYMBC3HxqjkzoCffXhS8phh7yefwoToF297qMeSLd9uSAw8MJOwJ+3Nu050fSzO8wIm3sCfw==" + }, + "SpacetimeDB.ClientSDK": { + "type": "Transitive", + "resolved": "2.10.1", + "contentHash": "w9AyBZzKQR74gws6jXqaAJ9Bc5C9SmJOoN9Xr/RvXrXnKmvTjFfXceOa/VBYrtTwou4tZmuaj3/ljI9EphYMDQ==", + "dependencies": { + "SpacetimeDB.BSATN.Runtime": "2.10.1" + } + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "6.0.1", + "contentHash": "IQ4NXP/B3Ayzvw0rDQzVTYsCKyy0Jp9KI6aYcK7UnGVlR9+Awz++TIPCQtPYfLJfOpm8ajowMR09V7quD3sEHw==" + }, + "xunit.analyzers": { + "type": "Transitive", + "resolved": "2.1.0", + "contentHash": "X7QXEcZQGz0G/HL4HUyK+aAvNa/IMGbOCnFIq4jD/Evktq12xANKwzOUr7b08vCmC1LXu/47qHWOdjm3KfaJ0A==" + }, + "xunit.v3.assert": { + "type": "Transitive", + "resolved": "4.0.1", + "contentHash": "nC7d3cY06Oo7hkdwWPZkBR0Ud75xNhgx0P7i0GmMMZC3puCGlc4szFiT30biH03IN/eDnr8h+nv3A1UD17bk/A==" + }, + "xunit.v3.common": { + "type": "Transitive", + "resolved": "4.0.1", + "contentHash": "Qf25TVdadDQYf9zVxSd7L9RNQhuP0jCUHmo96XTmIKLxGefylIEV63jRbXBPFzdXmV09TCdcurk+AQ9LiWS2Hg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0" + } + }, + "xunit.v3.core.mtp-v2": { + "type": "Transitive", + "resolved": "4.0.1", + "contentHash": "7qfTfrIfS2wybpSVyRLmhXHbSudD2eNw66LfukixmKsRCTcfvOrLsx0qGL/lObcEh3Z2F4SIXgoFnLIiB8yfjQ==", + "dependencies": { + "Microsoft.Testing.Extensions.Telemetry": "2.4.0", + "Microsoft.Testing.Extensions.TrxReport.Abstractions": "2.4.0", + "Microsoft.Testing.Platform": "2.4.0", + "Microsoft.Testing.Platform.MSBuild": "2.4.0", + "xunit.v3.extensibility.core": "[4.0.1]", + "xunit.v3.runner.inproc.console": "[4.0.1]" + } + }, + "xunit.v3.extensibility.core": { + "type": "Transitive", + "resolved": "4.0.1", + "contentHash": "J0d5OfFcp920nZxCRIvU14+TZhiSPQR0wvqEMZ3MRiJLPu1VmYlcRNkmcSW0i1DwkB81gAHc+fHUQf/cU64MYg==", + "dependencies": { + "xunit.v3.common": "[4.0.1]" + } + }, + "xunit.v3.mtp-v2": { + "type": "Transitive", + "resolved": "4.0.1", + "contentHash": "s88KiWwDDYgOWV3A+ViJiqCe9cLU/Rt6Gb5TfOC7Abv/HKSoj4IHVPcOQk7jPt7HhZHHEts5IuaVpB30eO5B1w==", + "dependencies": { + "xunit.analyzers": "2.1.0", + "xunit.v3.assert": "[4.0.1]", + "xunit.v3.core.mtp-v2": "[4.0.1]" + } + }, + "xunit.v3.runner.common": { + "type": "Transitive", + "resolved": "4.0.1", + "contentHash": "p9AyfBpj2e5Iws8B57SbLiSngg7mEdvegHInKR4T2cjt2mis9h8htVgCnmu//agZO4zri6p6VQOj5EkHHTy3gA==", + "dependencies": { + "Microsoft.Win32.Registry": "[5.0.0]", + "System.Security.AccessControl": "[6.0.1]", + "xunit.v3.common": "[4.0.1]" + } + }, + "xunit.v3.runner.inproc.console": { + "type": "Transitive", + "resolved": "4.0.1", + "contentHash": "Hqwfd6ehMIhPmVWUQ2dgmknzuLFTWeyp8ES1q3D4YR5bQVyiXDcaIoaFwqAz2zgLr49WaW4Mz7VVncP7u/Q8/Q==", + "dependencies": { + "xunit.v3.extensibility.core": "[4.0.1]", + "xunit.v3.runner.common": "[4.0.1]" + } + }, + "spacetimedbvsdoublets": { + "type": "Project", + "dependencies": { + "Platform.Data.Doublets": "[0.18.1, )", + "SpacetimeDB.ClientSDK": "[2.10.1, )" + } + } + } + } +} \ No newline at end of file diff --git a/csharp/SpacetimeDBVSDoublets.slnx b/csharp/SpacetimeDBVSDoublets.slnx new file mode 100644 index 0000000..c74c1d3 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets.slnx @@ -0,0 +1,4 @@ + + + + diff --git a/csharp/SpacetimeDBVSDoublets/Benchmarks.cs b/csharp/SpacetimeDBVSDoublets/Benchmarks.cs new file mode 100644 index 0000000..70cece0 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/Benchmarks.cs @@ -0,0 +1,64 @@ +using System.Diagnostics; + +namespace Comparisons.SpacetimeDBVSDoublets; + +/// Mirrors rust/benches/bench.rs, including the setup and reset outside measurement. +public static class Benchmarks +{ + public static readonly string[] Operations = + ["create", "update", "delete", "query_all", "query_by_id", "query_by_source", "query_by_target"]; + + /// Unique pairs referencing actual background IDs; matches rust/src/workload.rs. + public static (ulong Source, ulong Target) DistributedLink(int index, ulong[] background, bool bySource) + { + var key = background[index % 10]; + var other = background[index / 10 + 10]; + return bySource ? (key, other) : (other, key); + } + + public static TimeSpan Iteration(ILinks store, string operation, int count, int background) + { + var backgroundIds = Enumerable.Range(0, background).Select(_ => store.CreatePoint()).ToArray(); + if (operation is "query_by_source" or "query_by_target" && background < 10 + (count + 9) / 10) + throw new ArgumentException("Query workloads require at least 10 + ceil(N/10) background links"); + var ids = new List(); + if (operation is "update" or "delete" or "query_all" or "query_by_id") + for (var i = 0; i < count; i++) ids.Add(store.CreatePoint()); + if (operation == "query_by_source") + for (ulong i = 1; i <= (ulong)count; i++) { var pair = DistributedLink((int)i - 1, backgroundIds, true); store.Create(pair.Source, pair.Target); } + if (operation == "query_by_target") + for (ulong i = 1; i <= (ulong)count; i++) { var pair = DistributedLink((int)i - 1, backgroundIds, false); store.Create(pair.Source, pair.Target); } + try + { + var started = Stopwatch.GetTimestamp(); + switch (operation) + { + case "create": + for (var i = 0; i < count; i++) store.CreatePoint(); + break; + case "update": + foreach (var id in ids) store.Update(id, 0, 0); + foreach (var id in ids) store.Update(id, id, id); + break; + case "delete": + foreach (var id in ids) store.Delete(id); + break; + case "query_all": + GC.KeepAlive(store.QueryAll()); + break; + case "query_by_id": + foreach (var id in ids) GC.KeepAlive(store.QueryById(id)); + break; + case "query_by_source": + foreach (var source in backgroundIds.Take(Math.Min(count, 10))) GC.KeepAlive(store.QueryBySource(source)); + break; + case "query_by_target": + foreach (var target in backgroundIds.Take(Math.Min(count, 10))) GC.KeepAlive(store.QueryByTarget(target)); + break; + default: throw new ArgumentException($"Unknown operation {operation}"); + } + return Stopwatch.GetElapsedTime(started); + } + finally { store.DeleteAll(); } + } +} diff --git a/csharp/SpacetimeDBVSDoublets/DoubletsLinks.cs b/csharp/SpacetimeDBVSDoublets/DoubletsLinks.cs new file mode 100644 index 0000000..f82ad9c --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/DoubletsLinks.cs @@ -0,0 +1,110 @@ +using Platform.Data; +using Platform.Data.Doublets; +using Platform.Data.Doublets.Memory.Split.Generic; +using Platform.Data.Doublets.Memory.United.Generic; +using Platform.Memory; + +namespace Comparisons.SpacetimeDBVSDoublets; + +/// +/// A Doublets store of Platform.Data.Doublets. +/// The store runs inside the benchmark process: every operation reads and writes the store's memory directly, +/// without a server, a network connection or a query language. +/// +/// +/// +/// Doublets_United_VolatileUnitedMemoryLinks in RAM +/// Doublets_United_NonVolatileUnitedMemoryLinks in a memory-mapped file +/// Doublets_Split_VolatileSplitMemoryLinks in RAM +/// Doublets_Split_NonVolatileSplitMemoryLinks in two memory-mapped files +/// +/// Writes to a memory-mapped file go to the OS page cache; the benchmarks do not flush them to the disk. +/// +public sealed class DoubletsLinks(ILinks links) : ILinks +{ + public static readonly string[] Variants = + [ + "Doublets_United_Volatile", + "Doublets_United_NonVolatile", + "Doublets_Split_Volatile", + "Doublets_Split_NonVolatile", + ]; + + private readonly ulong _continue = links.Constants.Continue; + + public ulong Any { get; } = links.Constants.Any; + + /// Opens the store of ; the non-volatile stores are kept in . + public static DoubletsLinks Open(string variant, string directory) + { + static FileMappedResizableDirectMemory Mapped(string directory, string name) + { + var path = Path.Combine(directory, name); + File.Delete(path); + return new FileMappedResizableDirectMemory(path); + } + + ILinks links = variant switch + { + "Doublets_United_Volatile" => new UnitedMemoryLinks(new HeapResizableDirectMemory()), + "Doublets_United_NonVolatile" => new UnitedMemoryLinks(Mapped(directory, "united.links")), + "Doublets_Split_Volatile" => new SplitMemoryLinks(new HeapResizableDirectMemory(), new HeapResizableDirectMemory()), + "Doublets_Split_NonVolatile" => new SplitMemoryLinks(Mapped(directory, "split_data.links"), Mapped(directory, "split_index.links")), + _ => throw new ArgumentException($"unknown Doublets store {variant}", nameof(variant)), + }; + return new DoubletsLinks(links); + } + + public void Populate(int backgroundLinks) + { + for (var i = 0; i < backgroundLinks; i++) + { + links.CreatePoint(); + } + } + + /// Deletes the links from the highest id down, so the next created link gets id 1 again. + public void DeleteAll() + { + var ids = new List(); + Each(Any, Any, Any, link => ids.Add(link.Id)); + foreach (var id in ids.OrderDescending()) + { + Delete(id); + } + } + + public ulong Create(ulong source, ulong target) => links.CreateAndUpdate(source, target); + + public ulong CreatePoint() => links.CreatePoint(); + + public void Update(ulong id, ulong source, ulong target) => links.Update(id, source, target); + + /// + /// Resets the link to (0, 0), which removes it from the index trees, and then frees it. + /// + public void Delete(ulong id) => links.Delete(id, handler: null); + + public void Each(ulong id, ulong source, ulong target, Action visit) => links.Each(link => + { + visit(new Link(link![0], link[1], link[2])); + return _continue; + }, id, source, target); + + private Link[] Query(ulong id, ulong source, ulong target) + { + var result = new List(); + Each(id, source, target, result.Add); + return result.ToArray(); + } + + public Link[] QueryAll() => Query(Any, Any, Any); + public Link? QueryById(ulong id) => Query(id, Any, Any).Select(link => (Link?)link).FirstOrDefault(); + public Link[] QueryBySource(ulong source) => Query(Any, source, Any); + public Link[] QueryByTarget(ulong target) => Query(Any, Any, target); + public Link[] QueryBySourceTarget(ulong source, ulong target) => Query(Any, source, target); + + public ulong Count() => links.Count(); + + public void Dispose() => (links as IDisposable)?.Dispose(); +} diff --git a/csharp/SpacetimeDBVSDoublets/Harness.cs b/csharp/SpacetimeDBVSDoublets/Harness.cs new file mode 100644 index 0000000..b6eff64 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/Harness.cs @@ -0,0 +1,93 @@ +using System.Diagnostics; +using System.Globalization; + +namespace Comparisons.SpacetimeDBVSDoublets; + +public enum SamplingMode { Auto, Linear, Flat } + +/// +/// How a benchmark is sampled. The values match the Rust workflow, +/// and the iteration counts are chosen as Criterion 0.8 chooses them, so both languages measure the same way. +/// +/// Number of samples. +/// Auto chooses flat sampling when a linear schedule would exceed twice the target time. +/// How long the benchmark runs before it is measured. +/// The measured time that all samples take together. +public sealed record Sampling(int SampleSize, SamplingMode Mode, TimeSpan WarmUp, TimeSpan MeasurementTime) +{ + /// PR validation uses ten samples, one second warm-up and two seconds measurement. + public static Sampling Quick { get; } = new(10, SamplingMode.Auto, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2)); + + /// Full runs use twenty samples, three seconds warm-up and five seconds measurement. + public static Sampling Full { get; } = new(20, SamplingMode.Auto, TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(5)); + + /// The number of iterations of every sample, for iterations that take each. + public long[] IterationCounts(double nanoseconds) + { + var measurement = MeasurementTime.TotalNanoseconds; + var totalRuns = (double)SampleSize * (SampleSize + 1) / 2; + var step = Math.Max(1, (long)Math.Ceiling(measurement / nanoseconds / totalRuns)); + if (Mode == SamplingMode.Flat || Mode == SamplingMode.Auto && totalRuns * step * nanoseconds > 2 * measurement) + { + var iterations = Math.Max(1, (long)Math.Ceiling(measurement / (nanoseconds * SampleSize))); + return Enumerable.Repeat(iterations, SampleSize).ToArray(); + } + return Enumerable.Range(1, SampleSize).Select(sample => sample * step).ToArray(); + } +} + +/// The median time of one iteration and the standard deviation of the samples, in nanoseconds. +public readonly record struct Estimate(double Median, double StandardDeviation) +{ + /// Estimates the time of one iteration from the average iteration time of every sample. + public static Estimate Of(IReadOnlyCollection samples) + { + var sorted = samples.Order().ToArray(); + var middle = sorted.Length / 2; + var median = sorted.Length % 2 == 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; + var mean = sorted.Average(); + var variance = sorted.Length > 1 ? sorted.Sum(sample => (sample - mean) * (sample - mean)) / (sorted.Length - 1) : 0; + return new Estimate(median, Math.Sqrt(variance)); + } + + /// + /// The line that Criterion prints with --output-format bencher, for example + /// test create/SpacetimeDB/1000 ... bench: 44,055,505 ns/iter (+/- 5,345,991). + /// + public string Bencher(string group, string id) => + string.Create(CultureInfo.InvariantCulture, + $"test {group}/{id} ... bench: {Math.Round(Median),11:N0} ns/iter (+/- {Math.Round(StandardDeviation):N0})"); +} + +public static class Harness +{ + /// + /// Measures , which runs one iteration and returns its measured time: + /// first warms up for , doubling the number of iterations each round, + /// then takes samples. + /// + public static Estimate Measure(Sampling sampling, Func iteration) + { + TimeSpan Run(long iterations) + { + var elapsed = TimeSpan.Zero; + for (long i = 0; i < iterations; i++) + { + elapsed += iteration(); + } + return elapsed; + } + + // Criterion calibrates iter_custom using wall time, which includes untimed setup. + var warmUp = Stopwatch.StartNew(); + long warmUpIterations = 0; + for (long iterations = 1; warmUp.Elapsed < sampling.WarmUp; iterations = checked(iterations * 2)) + { + Run(iterations); + warmUpIterations += iterations; + } + var counts = sampling.IterationCounts(Math.Max(1, warmUp.Elapsed.TotalNanoseconds / warmUpIterations)); + return Estimate.Of(counts.Select(count => Run(count).TotalNanoseconds / count).ToArray()); + } + +} diff --git a/csharp/SpacetimeDBVSDoublets/ILinks.cs b/csharp/SpacetimeDBVSDoublets/ILinks.cs new file mode 100644 index 0000000..67a7acb --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/ILinks.cs @@ -0,0 +1,19 @@ +namespace Comparisons.SpacetimeDBVSDoublets; + +public readonly record struct Link(ulong Id, ulong Source, ulong Target); + +/// The same operations as the Rust Links trait; queries materialize their results. +public interface ILinks : IDisposable +{ + ulong Create(ulong source, ulong target); + ulong CreatePoint(); + void Update(ulong id, ulong source, ulong target); + void Delete(ulong id); + void DeleteAll(); + Link[] QueryAll(); + Link? QueryById(ulong id); + Link[] QueryBySource(ulong source); + Link[] QueryByTarget(ulong target); + Link[] QueryBySourceTarget(ulong source, ulong target); + ulong Count(); +} diff --git a/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/CreateLink.g.cs b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/CreateLink.g.cs new file mode 100644 index 0000000..1264fb7 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/CreateLink.g.cs @@ -0,0 +1,73 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Types +{ + public sealed partial class RemoteReducers : RemoteBase + { + public delegate void CreateLinkHandler(ReducerEventContext ctx, ulong source, ulong target); + public event CreateLinkHandler? OnCreateLink; + + public void CreateLink(ulong source, ulong target) + { + conn.InternalCallReducer(new Reducer.CreateLink(source, target)); + } + + public bool InvokeCreateLink(ReducerEventContext ctx, Reducer.CreateLink args) + { + if (OnCreateLink == null) + { + if (InternalOnUnhandledReducerError != null) + { + switch (ctx.Event.Status) + { + case Status.Failed(var reason): InternalOnUnhandledReducerError(ctx, new Exception(reason)); break; + case Status.OutOfEnergy(var _): InternalOnUnhandledReducerError(ctx, new Exception("out of energy")); break; + } + } + return false; + } + OnCreateLink( + ctx, + args.Source, + args.Target + ); + return true; + } + } + + public abstract partial class Reducer + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class CreateLink : Reducer, IReducerArgs + { + [DataMember(Name = "source")] + public ulong Source; + [DataMember(Name = "target")] + public ulong Target; + + public CreateLink( + ulong Source, + ulong Target + ) + { + this.Source = Source; + this.Target = Target; + } + + public CreateLink() + { + } + + string IReducerArgs.ReducerName => "create_link"; + } + } +} diff --git a/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/DeleteAllLinks.g.cs b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/DeleteAllLinks.g.cs new file mode 100644 index 0000000..85089ce --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/DeleteAllLinks.g.cs @@ -0,0 +1,53 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Types +{ + public sealed partial class RemoteReducers : RemoteBase + { + public delegate void DeleteAllLinksHandler(ReducerEventContext ctx); + public event DeleteAllLinksHandler? OnDeleteAllLinks; + + public void DeleteAllLinks() + { + conn.InternalCallReducer(new Reducer.DeleteAllLinks()); + } + + public bool InvokeDeleteAllLinks(ReducerEventContext ctx, Reducer.DeleteAllLinks args) + { + if (OnDeleteAllLinks == null) + { + if (InternalOnUnhandledReducerError != null) + { + switch (ctx.Event.Status) + { + case Status.Failed(var reason): InternalOnUnhandledReducerError(ctx, new Exception(reason)); break; + case Status.OutOfEnergy(var _): InternalOnUnhandledReducerError(ctx, new Exception("out of energy")); break; + } + } + return false; + } + OnDeleteAllLinks( + ctx + ); + return true; + } + } + + public abstract partial class Reducer + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class DeleteAllLinks : Reducer, IReducerArgs + { + string IReducerArgs.ReducerName => "delete_all_links"; + } + } +} diff --git a/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/DeleteLink.g.cs b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/DeleteLink.g.cs new file mode 100644 index 0000000..15050e5 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/DeleteLink.g.cs @@ -0,0 +1,66 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Types +{ + public sealed partial class RemoteReducers : RemoteBase + { + public delegate void DeleteLinkHandler(ReducerEventContext ctx, ulong id); + public event DeleteLinkHandler? OnDeleteLink; + + public void DeleteLink(ulong id) + { + conn.InternalCallReducer(new Reducer.DeleteLink(id)); + } + + public bool InvokeDeleteLink(ReducerEventContext ctx, Reducer.DeleteLink args) + { + if (OnDeleteLink == null) + { + if (InternalOnUnhandledReducerError != null) + { + switch (ctx.Event.Status) + { + case Status.Failed(var reason): InternalOnUnhandledReducerError(ctx, new Exception(reason)); break; + case Status.OutOfEnergy(var _): InternalOnUnhandledReducerError(ctx, new Exception("out of energy")); break; + } + } + return false; + } + OnDeleteLink( + ctx, + args.Id + ); + return true; + } + } + + public abstract partial class Reducer + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class DeleteLink : Reducer, IReducerArgs + { + [DataMember(Name = "id")] + public ulong Id; + + public DeleteLink(ulong Id) + { + this.Id = Id; + } + + public DeleteLink() + { + } + + string IReducerArgs.ReducerName => "delete_link"; + } + } +} diff --git a/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/UpdateLink.g.cs b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/UpdateLink.g.cs new file mode 100644 index 0000000..c912847 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Reducers/UpdateLink.g.cs @@ -0,0 +1,78 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Types +{ + public sealed partial class RemoteReducers : RemoteBase + { + public delegate void UpdateLinkHandler(ReducerEventContext ctx, ulong id, ulong source, ulong target); + public event UpdateLinkHandler? OnUpdateLink; + + public void UpdateLink(ulong id, ulong source, ulong target) + { + conn.InternalCallReducer(new Reducer.UpdateLink(id, source, target)); + } + + public bool InvokeUpdateLink(ReducerEventContext ctx, Reducer.UpdateLink args) + { + if (OnUpdateLink == null) + { + if (InternalOnUnhandledReducerError != null) + { + switch (ctx.Event.Status) + { + case Status.Failed(var reason): InternalOnUnhandledReducerError(ctx, new Exception(reason)); break; + case Status.OutOfEnergy(var _): InternalOnUnhandledReducerError(ctx, new Exception("out of energy")); break; + } + } + return false; + } + OnUpdateLink( + ctx, + args.Id, + args.Source, + args.Target + ); + return true; + } + } + + public abstract partial class Reducer + { + [SpacetimeDB.Type] + [DataContract] + public sealed partial class UpdateLink : Reducer, IReducerArgs + { + [DataMember(Name = "id")] + public ulong Id; + [DataMember(Name = "source")] + public ulong Source; + [DataMember(Name = "target")] + public ulong Target; + + public UpdateLink( + ulong Id, + ulong Source, + ulong Target + ) + { + this.Id = Id; + this.Source = Source; + this.Target = Target; + } + + public UpdateLink() + { + } + + string IReducerArgs.ReducerName => "update_link"; + } + } +} diff --git a/csharp/SpacetimeDBVSDoublets/ModuleBindings/SpacetimeDBClient.g.cs b/csharp/SpacetimeDBVSDoublets/ModuleBindings/SpacetimeDBClient.g.cs new file mode 100644 index 0000000..3d85439 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/ModuleBindings/SpacetimeDBClient.g.cs @@ -0,0 +1,631 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb cli version 2.10.1 (commit 3d7607082ab47257adeb7c25164536511237f2b5). + +#nullable enable + +using System; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Types +{ + public sealed partial class RemoteReducers : RemoteBase + { + internal RemoteReducers(DbConnection conn) : base(conn) { } + internal event Action? InternalOnUnhandledReducerError; + } + + public sealed partial class RemoteProcedures : RemoteBase + { + internal RemoteProcedures(DbConnection conn) : base(conn) { } + } + + public sealed partial class RemoteTables : RemoteTablesBase + { + public RemoteTables(DbConnection conn) + { + AddTable(Links = new(conn)); + } + } + + + public interface IRemoteDbContext : IDbContext + { + public event Action? OnUnhandledReducerError; + } + + public sealed class EventContext : IEventContext, IRemoteDbContext + { + private readonly DbConnection conn; + + /// + /// The event that caused this callback to run. + /// + public readonly Event Event; + + /// + /// Access to tables in the client cache, which stores a read-only replica of the remote database state. + /// + /// The returned DbView will have a method to access each table defined by the module. + /// + public RemoteTables Db => conn.Db; + /// + /// Access to reducers defined by the module. + /// + /// The returned RemoteReducers will have a method to invoke each reducer defined by the module, + /// plus methods for adding and removing callbacks on each of those reducers. + /// + public RemoteReducers Reducers => conn.Reducers; + /// + /// Access to procedures defined by the module. + /// + /// The returned RemoteProcedures will have a method to invoke each procedure defined by the module, + /// with a callback for when the procedure completes and returns a value. + /// + public RemoteProcedures Procedures => conn.Procedures; + /// + /// Returns true if the connection is active, i.e. has not yet disconnected. + /// + public bool IsActive => conn.IsActive; + /// + /// Close the connection. + /// + /// Throws an error if the connection is already closed. + /// + public void Disconnect() + { + conn.Disconnect(); + } + /// + /// Start building a subscription. + /// + /// A builder-pattern constructor for subscribing to queries, + /// causing matching rows to be replicated into the client cache. + public SubscriptionBuilder SubscriptionBuilder() => conn.SubscriptionBuilder(); + /// + /// Get the Identity of this connection. + /// + /// This method returns null if the connection was constructed anonymously + /// and we have not yet received our newly-generated Identity from the host. + /// + public Identity? Identity => conn.Identity; + /// + /// Get this connection's ConnectionId. + /// + public ConnectionId ConnectionId => conn.ConnectionId; + /// + /// Register a callback to be called when a reducer with no handler returns an error. + /// + public event Action? OnUnhandledReducerError + { + add => Reducers.InternalOnUnhandledReducerError += value; + remove => Reducers.InternalOnUnhandledReducerError -= value; + } + + internal EventContext(DbConnection conn, Event Event) + { + this.conn = conn; + this.Event = Event; + } + } + + public sealed class ReducerEventContext : IReducerEventContext, IRemoteDbContext + { + private readonly DbConnection conn; + /// + /// The reducer event that caused this callback to run. + /// + public readonly ReducerEvent Event; + + /// + /// Access to tables in the client cache, which stores a read-only replica of the remote database state. + /// + /// The returned DbView will have a method to access each table defined by the module. + /// + public RemoteTables Db => conn.Db; + /// + /// Access to reducers defined by the module. + /// + /// The returned RemoteReducers will have a method to invoke each reducer defined by the module, + /// plus methods for adding and removing callbacks on each of those reducers. + /// + public RemoteReducers Reducers => conn.Reducers; + /// + /// Access to procedures defined by the module. + /// + /// The returned RemoteProcedures will have a method to invoke each procedure defined by the module, + /// with a callback for when the procedure completes and returns a value. + /// + public RemoteProcedures Procedures => conn.Procedures; + /// + /// Returns true if the connection is active, i.e. has not yet disconnected. + /// + public bool IsActive => conn.IsActive; + /// + /// Close the connection. + /// + /// Throws an error if the connection is already closed. + /// + public void Disconnect() + { + conn.Disconnect(); + } + /// + /// Start building a subscription. + /// + /// A builder-pattern constructor for subscribing to queries, + /// causing matching rows to be replicated into the client cache. + public SubscriptionBuilder SubscriptionBuilder() => conn.SubscriptionBuilder(); + /// + /// Get the Identity of this connection. + /// + /// This method returns null if the connection was constructed anonymously + /// and we have not yet received our newly-generated Identity from the host. + /// + public Identity? Identity => conn.Identity; + /// + /// Get this connection's ConnectionId. + /// + public ConnectionId ConnectionId => conn.ConnectionId; + /// + /// Register a callback to be called when a reducer with no handler returns an error. + /// + public event Action? OnUnhandledReducerError + { + add => Reducers.InternalOnUnhandledReducerError += value; + remove => Reducers.InternalOnUnhandledReducerError -= value; + } + + internal ReducerEventContext(DbConnection conn, ReducerEvent reducerEvent) + { + this.conn = conn; + Event = reducerEvent; + } + } + + public sealed class ErrorContext : IErrorContext, IRemoteDbContext + { + private readonly DbConnection conn; + /// + /// The Exception that caused this error callback to be run. + /// + public readonly Exception Event; + Exception IErrorContext.Event + { + get + { + return Event; + } + } + + /// + /// Access to tables in the client cache, which stores a read-only replica of the remote database state. + /// + /// The returned DbView will have a method to access each table defined by the module. + /// + public RemoteTables Db => conn.Db; + /// + /// Access to reducers defined by the module. + /// + /// The returned RemoteReducers will have a method to invoke each reducer defined by the module, + /// plus methods for adding and removing callbacks on each of those reducers. + /// + public RemoteReducers Reducers => conn.Reducers; + /// + /// Access to procedures defined by the module. + /// + /// The returned RemoteProcedures will have a method to invoke each procedure defined by the module, + /// with a callback for when the procedure completes and returns a value. + /// + public RemoteProcedures Procedures => conn.Procedures; + /// + /// Returns true if the connection is active, i.e. has not yet disconnected. + /// + public bool IsActive => conn.IsActive; + /// + /// Close the connection. + /// + /// Throws an error if the connection is already closed. + /// + public void Disconnect() + { + conn.Disconnect(); + } + /// + /// Start building a subscription. + /// + /// A builder-pattern constructor for subscribing to queries, + /// causing matching rows to be replicated into the client cache. + public SubscriptionBuilder SubscriptionBuilder() => conn.SubscriptionBuilder(); + /// + /// Get the Identity of this connection. + /// + /// This method returns null if the connection was constructed anonymously + /// and we have not yet received our newly-generated Identity from the host. + /// + public Identity? Identity => conn.Identity; + /// + /// Get this connection's ConnectionId. + /// + public ConnectionId ConnectionId => conn.ConnectionId; + /// + /// Register a callback to be called when a reducer with no handler returns an error. + /// + public event Action? OnUnhandledReducerError + { + add => Reducers.InternalOnUnhandledReducerError += value; + remove => Reducers.InternalOnUnhandledReducerError -= value; + } + + internal ErrorContext(DbConnection conn, Exception error) + { + this.conn = conn; + Event = error; + } + } + + public sealed class SubscriptionEventContext : ISubscriptionEventContext, IRemoteDbContext + { + private readonly DbConnection conn; + + /// + /// Access to tables in the client cache, which stores a read-only replica of the remote database state. + /// + /// The returned DbView will have a method to access each table defined by the module. + /// + public RemoteTables Db => conn.Db; + /// + /// Access to reducers defined by the module. + /// + /// The returned RemoteReducers will have a method to invoke each reducer defined by the module, + /// plus methods for adding and removing callbacks on each of those reducers. + /// + public RemoteReducers Reducers => conn.Reducers; + /// + /// Access to procedures defined by the module. + /// + /// The returned RemoteProcedures will have a method to invoke each procedure defined by the module, + /// with a callback for when the procedure completes and returns a value. + /// + public RemoteProcedures Procedures => conn.Procedures; + /// + /// Returns true if the connection is active, i.e. has not yet disconnected. + /// + public bool IsActive => conn.IsActive; + /// + /// Close the connection. + /// + /// Throws an error if the connection is already closed. + /// + public void Disconnect() + { + conn.Disconnect(); + } + /// + /// Start building a subscription. + /// + /// A builder-pattern constructor for subscribing to queries, + /// causing matching rows to be replicated into the client cache. + public SubscriptionBuilder SubscriptionBuilder() => conn.SubscriptionBuilder(); + /// + /// Get the Identity of this connection. + /// + /// This method returns null if the connection was constructed anonymously + /// and we have not yet received our newly-generated Identity from the host. + /// + public Identity? Identity => conn.Identity; + /// + /// Get this connection's ConnectionId. + /// + public ConnectionId ConnectionId => conn.ConnectionId; + /// + /// Register a callback to be called when a reducer with no handler returns an error. + /// + public event Action? OnUnhandledReducerError + { + add => Reducers.InternalOnUnhandledReducerError += value; + remove => Reducers.InternalOnUnhandledReducerError -= value; + } + + internal SubscriptionEventContext(DbConnection conn) + { + this.conn = conn; + } + } + + public sealed class ProcedureEventContext : IProcedureEventContext, IRemoteDbContext + { + private readonly DbConnection conn; + /// + /// The procedure event that caused this callback to run. + /// + public readonly ProcedureEvent Event; + + /// + /// Access to tables in the client cache, which stores a read-only replica of the remote database state. + /// + /// The returned DbView will have a method to access each table defined by the module. + /// + public RemoteTables Db => conn.Db; + /// + /// Access to reducers defined by the module. + /// + /// The returned RemoteReducers will have a method to invoke each reducer defined by the module, + /// plus methods for adding and removing callbacks on each of those reducers. + /// + public RemoteReducers Reducers => conn.Reducers; + /// + /// Access to procedures defined by the module. + /// + /// The returned RemoteProcedures will have a method to invoke each procedure defined by the module, + /// with a callback for when the procedure completes and returns a value. + /// + public RemoteProcedures Procedures => conn.Procedures; + /// + /// Returns true if the connection is active, i.e. has not yet disconnected. + /// + public bool IsActive => conn.IsActive; + /// + /// Close the connection. + /// + /// Throws an error if the connection is already closed. + /// + public void Disconnect() + { + conn.Disconnect(); + } + /// + /// Start building a subscription. + /// + /// A builder-pattern constructor for subscribing to queries, + /// causing matching rows to be replicated into the client cache. + public SubscriptionBuilder SubscriptionBuilder() => conn.SubscriptionBuilder(); + /// + /// Get the Identity of this connection. + /// + /// This method returns null if the connection was constructed anonymously + /// and we have not yet received our newly-generated Identity from the host. + /// + public Identity? Identity => conn.Identity; + /// + /// Get this connection's ConnectionId. + /// + public ConnectionId ConnectionId => conn.ConnectionId; + /// + /// Register a callback to be called when a reducer with no handler returns an error. + /// + public event Action? OnUnhandledReducerError + { + add => Reducers.InternalOnUnhandledReducerError += value; + remove => Reducers.InternalOnUnhandledReducerError -= value; + } + + internal ProcedureEventContext(DbConnection conn, ProcedureEvent Event) + { + this.conn = conn; + this.Event = Event; + } + } + + /// + /// Builder-pattern constructor for subscription queries. + /// + public sealed class SubscriptionBuilder + { + private readonly IDbConnection conn; + + private event Action? Applied; + private event Action? Error; + + /// + /// Private API, use conn.SubscriptionBuilder() instead. + /// + public SubscriptionBuilder(IDbConnection conn) + { + this.conn = conn; + } + + /// + /// Register a callback to run when the subscription is applied. + /// + public SubscriptionBuilder OnApplied( + Action callback + ) + { + Applied += callback; + return this; + } + + /// + /// Register a callback to run when the subscription fails. + /// + /// Note that this callback may run either when attempting to apply the subscription, + /// in which case Self::on_applied will never run, + /// or later during the subscription's lifetime if the module's interface changes, + /// in which case Self::on_applied may have already run. + /// + public SubscriptionBuilder OnError( + Action callback + ) + { + Error += callback; + return this; + } + + /// + /// Add a typed query to this subscription. + /// + /// This is the entry point for building subscriptions without writing SQL by hand. + /// Once a typed query is added, only typed queries may follow (SQL and typed queries cannot be mixed). + /// + public TypedSubscriptionBuilder AddQuery( + Func> build + ) + { + var typed = new TypedSubscriptionBuilder(conn, Applied, Error); + return typed.AddQuery(build); + } + + /// + /// Subscribe to the following SQL queries. + /// + /// This method returns immediately, with the data not yet added to the DbConnection. + /// The provided callbacks will be invoked once the data is returned from the remote server. + /// Data from all the provided queries will be returned at the same time. + /// + /// See the SpacetimeDB SQL docs for more information on SQL syntax: + /// https://spacetimedb.com/docs/sql + /// + public SubscriptionHandle Subscribe( + string[] querySqls + ) => new(conn, Applied, Error, querySqls); + + /// + /// Subscribe to all rows from all tables. + /// + /// This method is intended as a convenience + /// for applications where client-side memory use and network bandwidth are not concerns. + /// Applications where these resources are a constraint + /// should register more precise queries via Self.Subscribe + /// in order to replicate only the subset of data which the client needs to function. + /// + /// This method should not be combined with Self.Subscribe on the same DbConnection. + /// A connection may either Self.Subscribe to particular queries, + /// or Self.SubscribeToAllTables, but not both. + /// Attempting to call Self.Subscribe + /// on a DbConnection that has previously used Self.SubscribeToAllTables, + /// or vice versa, may misbehave in any number of ways, + /// including dropping subscriptions, corrupting the client cache, or panicking. + /// + public SubscriptionHandle SubscribeToAllTables() => + new(conn, Applied, Error, QueryBuilder.AllTablesSqlQueries()); + } + + public sealed class SubscriptionHandle : SubscriptionHandleBase + { + /// + /// Internal API. Construct SubscriptionHandles using conn.SubscriptionBuilder. + /// + public SubscriptionHandle( + IDbConnection conn, + Action? onApplied, + Action? onError, + string[] querySqls + ) : base(conn, onApplied, onError, querySqls) + { } + } + + public sealed class QueryBuilder + { + public From From { get; } = new(); + + internal static string[] AllTablesSqlQueries() => new string[] + { + new QueryBuilder().From.Links().ToSql(), + } + ; + } + + public sealed class From + { + public global::SpacetimeDB.Table Links() => new("links", new LinksCols("links"), new LinksIxCols("links")); + } + + public sealed class TypedSubscriptionBuilder + { + private readonly IDbConnection conn; + private Action? Applied; + private Action? Error; + private readonly List querySqls = new(); + + internal TypedSubscriptionBuilder(IDbConnection conn, Action? applied, Action? error) + { + this.conn = conn; + Applied = applied; + Error = error; + } + + public TypedSubscriptionBuilder OnApplied(Action callback) + { + Applied += callback; + return this; + } + + public TypedSubscriptionBuilder OnError(Action callback) + { + Error += callback; + return this; + } + + public TypedSubscriptionBuilder AddQuery(Func> build) + { + var qb = new QueryBuilder(); + querySqls.Add(build(qb).ToSql()); + return this; + } + + public SubscriptionHandle Subscribe() => new(conn, Applied, Error, querySqls.ToArray()); + } + + public abstract partial class Reducer + { + private Reducer() { } + } + + public abstract partial class Procedure + { + private Procedure() { } + } + + public sealed class DbConnection : DbConnectionBase + { + public override RemoteTables Db { get; } + public readonly RemoteReducers Reducers; + public readonly RemoteProcedures Procedures; + + public DbConnection() + { + Db = new(this); + Reducers = new(this); + Procedures = new(this); + } + + protected override IEventContext ToEventContext(Event Event) => + new EventContext(this, Event); + + protected override IReducerEventContext ToReducerEventContext(ReducerEvent reducerEvent) => + new ReducerEventContext(this, reducerEvent); + + protected override ISubscriptionEventContext MakeSubscriptionEventContext() => + new SubscriptionEventContext(this); + + protected override IErrorContext ToErrorContext(Exception exception) => + new ErrorContext(this, exception); + + protected override IProcedureEventContext ToProcedureEventContext(ProcedureEvent procedureEvent) => + new ProcedureEventContext(this, procedureEvent); + + protected override bool Dispatch(IReducerEventContext context, Reducer reducer) + { + var eventContext = (ReducerEventContext)context; + return reducer switch + { + Reducer.CreateLink args => Reducers.InvokeCreateLink(eventContext, args), + Reducer.DeleteAllLinks args => Reducers.InvokeDeleteAllLinks(eventContext, args), + Reducer.DeleteLink args => Reducers.InvokeDeleteLink(eventContext, args), + Reducer.UpdateLink args => Reducers.InvokeUpdateLink(eventContext, args), + _ => throw new ArgumentOutOfRangeException("Reducer", $"Unknown reducer {reducer}") + }; + } + + public SubscriptionBuilder SubscriptionBuilder() => new(this); + public event Action OnUnhandledReducerError + { + add => Reducers.InternalOnUnhandledReducerError += value; + remove => Reducers.InternalOnUnhandledReducerError -= value; + } + } +} diff --git a/csharp/SpacetimeDBVSDoublets/ModuleBindings/Tables/Links.g.cs b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Tables/Links.g.cs new file mode 100644 index 0000000..90e4776 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Tables/Links.g.cs @@ -0,0 +1,87 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using SpacetimeDB.BSATN; +using SpacetimeDB.ClientApi; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Types +{ + public sealed partial class RemoteTables + { + public sealed class LinksHandle : RemoteTableHandle + { + public override string RemoteTableName => "links"; + + public sealed class IdUniqueIndex : UniqueIndexBase + { + protected override ulong GetKey(Link row) => row.Id; + + public IdUniqueIndex(LinksHandle table) : base(table) { } + } + + public readonly IdUniqueIndex Id; + + public sealed class SourceIndex : BTreeIndexBase + { + protected override ulong GetKey(Link row) => row.Source; + + public SourceIndex(LinksHandle table) : base(table) { } + } + + public readonly SourceIndex Source; + + public sealed class TargetIndex : BTreeIndexBase + { + protected override ulong GetKey(Link row) => row.Target; + + public TargetIndex(LinksHandle table) : base(table) { } + } + + public readonly TargetIndex Target; + + internal LinksHandle(DbConnection conn) : base(conn) + { + Id = new(this); + Source = new(this); + Target = new(this); + } + + protected override object GetPrimaryKey(Link row) => row.Id; + } + + public readonly LinksHandle Links; + } + + public sealed class LinksCols + { + public global::SpacetimeDB.Col Id { get; } + public global::SpacetimeDB.Col Source { get; } + public global::SpacetimeDB.Col Target { get; } + + public LinksCols(string tableName) + { + Id = new global::SpacetimeDB.Col(tableName, "id"); + Source = new global::SpacetimeDB.Col(tableName, "source"); + Target = new global::SpacetimeDB.Col(tableName, "target"); + } + } + + public sealed class LinksIxCols + { + public global::SpacetimeDB.IxCol Id { get; } + public global::SpacetimeDB.IxCol Source { get; } + public global::SpacetimeDB.IxCol Target { get; } + + public LinksIxCols(string tableName) + { + Id = new global::SpacetimeDB.IxCol(tableName, "id"); + Source = new global::SpacetimeDB.IxCol(tableName, "source"); + Target = new global::SpacetimeDB.IxCol(tableName, "target"); + } + } +} diff --git a/csharp/SpacetimeDBVSDoublets/ModuleBindings/Types/Link.g.cs b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Types/Link.g.cs new file mode 100644 index 0000000..551826c --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/ModuleBindings/Types/Link.g.cs @@ -0,0 +1,38 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Types +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class Link + { + [DataMember(Name = "id")] + public ulong Id; + [DataMember(Name = "source")] + public ulong Source; + [DataMember(Name = "target")] + public ulong Target; + + public Link( + ulong Id, + ulong Source, + ulong Target + ) + { + this.Id = Id; + this.Source = Source; + this.Target = Target; + } + + public Link() + { + } + } +} diff --git a/csharp/SpacetimeDBVSDoublets/Program.cs b/csharp/SpacetimeDBVSDoublets/Program.cs new file mode 100644 index 0000000..0ba9047 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/Program.cs @@ -0,0 +1,28 @@ +using Comparisons.SpacetimeDBVSDoublets; + +var backend = args.FirstOrDefault(arg => arg.StartsWith("--backend="))?.Split('=')[1] ?? "all"; +if (backend is not ("all" or "doublets" or "spacetimedb")) throw new ArgumentException("Invalid backend"); +var quick = args.Contains("--quick"); +int Size(string name, int fallback) => int.TryParse(Environment.GetEnvironmentVariable(name), out var value) && value > 0 + ? value : fallback; +var count = Size("BENCHMARK_LINK_COUNT", quick ? 10 : 1000); +var background = Size("BACKGROUND_LINK_COUNT", quick ? 30 : 3000); +var variants = (backend == "spacetimedb" ? Array.Empty() : DoubletsLinks.Variants) + .Concat(backend == "doublets" ? Array.Empty() : ["SpacetimeDB"]); +var directory = Path.Combine(Path.GetTempPath(), $"spacetimedb-vs-doublets-{Guid.NewGuid()}"); +Directory.CreateDirectory(directory); +try +{ + foreach (var variant in variants) + { + using ILinks store = variant == "SpacetimeDB" ? new SpacetimeLinks() : DoubletsLinks.Open(variant, directory); + store.DeleteAll(); + foreach (var operation in Benchmarks.Operations) + { + var estimate = Harness.Measure(quick ? Sampling.Quick : Sampling.Full, + () => Benchmarks.Iteration(store, operation, count, background)); + Console.WriteLine(estimate.Bencher(operation, $"{variant}/{count}")); + } + } +} +finally { Directory.Delete(directory, true); } diff --git a/csharp/SpacetimeDBVSDoublets/SpacetimeDBVSDoublets.csproj b/csharp/SpacetimeDBVSDoublets/SpacetimeDBVSDoublets.csproj new file mode 100644 index 0000000..57aa9ff --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/SpacetimeDBVSDoublets.csproj @@ -0,0 +1,14 @@ + + + Exe + net10.0 + enable + enable + true + true + + + + + + diff --git a/csharp/SpacetimeDBVSDoublets/SpacetimeLinks.cs b/csharp/SpacetimeDBVSDoublets/SpacetimeLinks.cs new file mode 100644 index 0000000..29c0e34 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/SpacetimeLinks.cs @@ -0,0 +1,99 @@ +using System.Diagnostics; +using SpacetimeDB; +using SpacetimeDB.Types; +using DbLink = SpacetimeDB.Types.Link; + +namespace Comparisons.SpacetimeDBVSDoublets; + +/// Official C# SDK: reducer writes and reads from the subscribed client cache. +public sealed class SpacetimeLinks : ILinks +{ + private readonly DbConnection connection; + private bool completed; + private Exception? failure; + + public SpacetimeLinks() + { + var ready = false; + var connected = false; + connection = DbConnection.Builder() + .WithUri(Environment.GetEnvironmentVariable("SPACETIMEDB_URI") ?? "http://localhost:3000") + .WithDatabaseName(Environment.GetEnvironmentVariable("SPACETIMEDB_DB") ?? "benchmark-links") + .OnConnect((_, _, _) => connected = true) + .OnConnectError(error => failure = error) + .OnDisconnect((_, error) => failure = error ?? new IOException("SpacetimeDB disconnected")) + .Build(); + connection.Reducers.OnCreateLink += (context, _, _) => Complete(context); + connection.Reducers.OnUpdateLink += (context, _, _, _) => Complete(context); + connection.Reducers.OnDeleteLink += (context, _) => Complete(context); + connection.Reducers.OnDeleteAllLinks += Complete; + try + { + PumpUntil(() => connected); + connection.SubscriptionBuilder() + .OnApplied(_ => ready = true) + .OnError((_, error) => failure = error) + .Subscribe(["SELECT * FROM links"]); + PumpUntil(() => ready); + } + catch + { + connection.Disconnect(); + throw; + } + } + + private void Complete(ReducerEventContext context) + { + switch (context.Event.Status) + { + case Status.Failed(var reason): failure = new InvalidOperationException(reason); break; + case Status.OutOfEnergy: failure = new InvalidOperationException("SpacetimeDB out of energy"); break; + } + completed = true; + } + + private void PumpUntil(Func done) + { + var elapsed = Stopwatch.StartNew(); + while (!done()) + { + connection.FrameTick(); + if (failure is not null) throw new InvalidOperationException("SpacetimeDB operation failed", failure); + if (elapsed.Elapsed > TimeSpan.FromSeconds(30)) throw new TimeoutException("SpacetimeDB operation timed out"); + Thread.Yield(); + } + if (failure is not null) throw new InvalidOperationException("SpacetimeDB operation failed", failure); + } + + private void Invoke(Action reducer) + { + completed = false; + reducer(); + PumpUntil(() => completed); + } + + private static Link Convert(DbLink row) => new(row.Id, row.Source, row.Target); + public ulong Create(ulong source, ulong target) + { + Invoke(() => connection.Reducers.CreateLink(source, target)); + return connection.Db.Links.Iter().Where(row => row.Source == source && row.Target == target).Max(row => row.Id); + } + public ulong CreatePoint() + { + var id = Create(0, 0); + Update(id, id, id); + return id; + } + public void Update(ulong id, ulong source, ulong target) => Invoke(() => connection.Reducers.UpdateLink(id, source, target)); + public void Delete(ulong id) => Invoke(() => connection.Reducers.DeleteLink(id)); + public void DeleteAll() => Invoke(connection.Reducers.DeleteAllLinks); + public Link[] QueryAll() => connection.Db.Links.Iter().Select(Convert).ToArray(); + public Link? QueryById(ulong id) => connection.Db.Links.Iter().Where(row => row.Id == id).Select(row => (Link?)Convert(row)).FirstOrDefault(); + public Link[] QueryBySource(ulong source) => connection.Db.Links.Iter().Where(row => row.Source == source).Select(Convert).ToArray(); + public Link[] QueryByTarget(ulong target) => connection.Db.Links.Iter().Where(row => row.Target == target).Select(Convert).ToArray(); + public Link[] QueryBySourceTarget(ulong source, ulong target) => connection.Db.Links.Iter() + .Where(row => row.Source == source && row.Target == target).Select(Convert).ToArray(); + public ulong Count() => (ulong)connection.Db.Links.Count; + public void Dispose() => connection.Disconnect(); +} diff --git a/csharp/SpacetimeDBVSDoublets/packages.lock.json b/csharp/SpacetimeDBVSDoublets/packages.lock.json new file mode 100644 index 0000000..6b1efe6 --- /dev/null +++ b/csharp/SpacetimeDBVSDoublets/packages.lock.json @@ -0,0 +1,192 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Platform.Data.Doublets": { + "type": "Direct", + "requested": "[0.18.1, )", + "resolved": "0.18.1", + "contentHash": "b6p7GgLDonEnRdhuy3Xsn/N5g5Qq+aivapAlkFH/Mk/MDTDiSJ0C1wnViC4Ff9/d6IXfmh2Sv/TeGIkyKbTOTw==", + "dependencies": { + "Platform.Collections.Methods": "0.7.3", + "Platform.Data": "0.16.1", + "Platform.Data.Doublets.FFI": "0.9.0", + "Platform.Memory": "0.4.1", + "Platform.Random": "0.3.0", + "Platform.Singletons": "0.3.0", + "Platform.Timestamps": "0.3.0" + } + }, + "SpacetimeDB.ClientSDK": { + "type": "Direct", + "requested": "[2.10.1, )", + "resolved": "2.10.1", + "contentHash": "w9AyBZzKQR74gws6jXqaAJ9Bc5C9SmJOoN9Xr/RvXrXnKmvTjFfXceOa/VBYrtTwou4tZmuaj3/ljI9EphYMDQ==", + "dependencies": { + "SpacetimeDB.BSATN.Runtime": "2.10.1" + } + }, + "Platform.Collections": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "ofU+XTf0CJry3dnpSq5pdiz8cArZDlrd3IaGtYoV8L6e+OhTdx2n544s+XLOZK90KJUsNv/docu2Y97Asdbvuw==", + "dependencies": { + "Platform.Disposables": "0.4.0", + "Platform.Interfaces": "0.5.0", + "Platform.Random": "0.3.0", + "Platform.Ranges": "0.5.0" + } + }, + "Platform.Collections.Methods": { + "type": "Transitive", + "resolved": "0.7.3", + "contentHash": "GG+XkIiZWlg96w9HxBeCeGrWdXr6+ST8L8piPjvC4lVrB0XaJNWWj+FYSnQOcX7MZOXs70Y6+e5eD0t1GygB5g==", + "dependencies": { + "Platform.Numbers": "0.9.0" + } + }, + "Platform.Converters": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "Rz4gLL2zPB/KsqORljmhMaQR8EVcTBnRRO2rz91F/nu+qJXuzpurx4VgtnL0lyso0NfkmNJffTiLucxwYX9pWQ==", + "dependencies": { + "Platform.Reflection": "0.7.0" + } + }, + "Platform.Data": { + "type": "Transitive", + "resolved": "0.16.1", + "contentHash": "1JdpgdWbowNgVx+aJatRyTEL5CS+hB5mjOJD57+R26XfbCpIsscJwL+Kcmk/ivaop6OABd3SX7M2DaAhYHXKtw==", + "dependencies": { + "Platform.Delegates": "0.1.1", + "Platform.Numbers": "0.9.0", + "Platform.Setters": "0.5.0", + "Platform.Threading": "0.4.0" + } + }, + "Platform.Data.Doublets.FFI": { + "type": "Transitive", + "resolved": "0.9.0", + "contentHash": "NRgGyihV3sdmvNW9SLnwBV14zkq89GJlTgyeiB+EKT2cMOkCjZtd55C/M0Sy5tLkZ076rBcup4jLFKmejnS2tg==" + }, + "Platform.Delegates": { + "type": "Transitive", + "resolved": "0.1.1", + "contentHash": "+182klZgGEJ+yWtxz3BMrTCYcpSXjvJzrYzpK+W2UecAaDKLvLyehjPlMfta+dM/XlKbPt53Wf20vhI1kul0SQ==" + }, + "Platform.Disposables": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "ko0W4cw5XhNxJANdO1T8srpWRVqpPiXtCohumlbTu33kivRBEmk1ivDsySaNhKTyuYjJnrjRYrgA1iEb2sS+/Q==", + "dependencies": { + "Platform.Exceptions": "0.5.0" + } + }, + "Platform.Exceptions": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "htu3iyPVJinbLxU1I1cpc2qFlur7IF2fTc9u2jqQ+3X1Lyzd3aeK9Y7gJu9ZkdnIO3tbdO6wxr1HZG4UhaIG8A==" + }, + "Platform.Interfaces": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "OBViyjgCYWaXH8xhibYL/fbwHfPP/+qVUDxcFGH22IaYzbrFbfNR5TxW6gsk3LR9QXLANnrLsuI7phZueoj/Tw==" + }, + "Platform.IO": { + "type": "Transitive", + "resolved": "0.5.1", + "contentHash": "MZ+uZIrDZuNZZu8J8gv8RcXwGihSTEo9UvFoRHCGIXQnaQxnJ0gBIF5s/zSjJ1+h8c78lv/wmlSvh4zMzwyJIQ==", + "dependencies": { + "Platform.Threading": "0.5.0", + "Platform.Unsafe": "0.6.1" + } + }, + "Platform.Memory": { + "type": "Transitive", + "resolved": "0.4.1", + "contentHash": "Jz4Xb0B8pmh6BU9hhvqMgK2L+R0fCxSk1bGWL7WK/WYKAYSuGaJUAvzWdJLTECyfs+w6PVUxXK53q1OXxHRmMQ==", + "dependencies": { + "Platform.IO": "0.5.1" + } + }, + "Platform.Numbers": { + "type": "Transitive", + "resolved": "0.9.0", + "contentHash": "0GgZSH0p+jP933DKtLzCfOAdkJ1cNKrSrisXgMKy0zTMBZOxMxzs83IVXwBju66vm5N/w6lFq8HwCLW0gQsYVA==", + "dependencies": { + "Platform.Converters": "0.4.0" + } + }, + "Platform.Random": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "GX24Xx45cUIr9EZl55evdwb7oYN/nimpoNqDG4zoMDDLqeEpL35zX8IiHHDYITV9F4bjrCrSEiRCVsRFAsi7SQ==", + "dependencies": { + "Platform.Ranges": "0.5.0" + } + }, + "Platform.Ranges": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "CUhZM/4F714NAyrtWtkBx78L4xJpFZ1wp+6ivFvpAXx6DsaBcBsyDxjn5eoLB9LSUYxB11gFkwwmXmJhMgZ3qw==", + "dependencies": { + "Platform.Exceptions": "0.5.0" + } + }, + "Platform.Reflection": { + "type": "Transitive", + "resolved": "0.7.0", + "contentHash": "CvpRnDWH5YArcXlY7lcY0mDYQCtnJAeZsqZ4/IdllJoj7z8JwTufeo/zK62Vn6K0ulTyHNhb+/T3HFusxnrfxA==", + "dependencies": { + "Platform.Collections": "0.4.0" + } + }, + "Platform.Setters": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "vZG+VnYc4u2x5vnBY7rqaFHtOS2AeoLwGV+02JBFa9pxluHwVTu4nVcVBTJTlenZSfAoZUNVsV1StnUWvINkCQ==", + "dependencies": { + "Platform.Delegates": "0.1.1", + "Platform.Interfaces": "0.5.0" + } + }, + "Platform.Singletons": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "fCfNw4CqPL6HzsUJVVYI/OLYXGkjmdDCe+QWThW8hPoNNlqMZ+FWbfBXbXpCuDYw9EYhLDlhuTAQMbqNt7kCzg==", + "dependencies": { + "Platform.Reflection": "0.7.0" + } + }, + "Platform.Threading": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "Q0p5oVDbhQ76IGHi/vYIX+de9T270XZmNgjNbHtobc+AoJbIWGf+kZXTzq+MmEfXS2ANtGNe9uWN1a7Nzp7olQ==", + "dependencies": { + "Platform.Collections": "0.4.0" + } + }, + "Platform.Timestamps": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "Lmhm8Vp0DobTlxSVTPZZE+3ABTrvuofcmTQOWZv6TyiZ5NyFBH6m61vZdnIkMa/3lQguKvGfgMsUT97p0t5x7w==", + "dependencies": { + "Platform.Interfaces": "0.5.0" + } + }, + "Platform.Unsafe": { + "type": "Transitive", + "resolved": "0.6.1", + "contentHash": "GbKgPC7lfw+q2GqQEch0NSxHOZXxoLTj3ChjdRZIQBvZnGWzYH6nat8glfTJseOmgdabIxlwogFECoai0O3akA==", + "dependencies": { + "Platform.Numbers": "0.7.0" + } + }, + "SpacetimeDB.BSATN.Runtime": { + "type": "Transitive", + "resolved": "2.10.1", + "contentHash": "EvVdnyTOy+qK1OOYMBC3HxqjkzoCffXhS8phh7yefwoToF297qMeSLd9uSAw8MJOwJ+3Nu050fSzO8wIm3sCfw==" + } + } + } +} \ No newline at end of file diff --git a/csharp/global.json b/csharp/global.json new file mode 100644 index 0000000..512142d --- /dev/null +++ b/csharp/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "version": "10.0.100", + "rollForward": "latestFeature" + } +} From 4239795d00ee04767ab1f4cb3bae9b537cfe5dfd Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:38:06 +0000 Subject: [PATCH 4/7] Report readable durations, measurement noise and recorded provenance --- rust/out.py | 180 +++++++++++++++++++++++++++++++++++++---------- rust/results.md | 16 ++--- rust/test_out.py | 109 +++++++++++++++++++++++++--- 3 files changed, 248 insertions(+), 57 deletions(-) diff --git a/rust/out.py b/rust/out.py index 5dfc796..6cfd2fe 100644 --- a/rust/out.py +++ b/rust/out.py @@ -18,10 +18,15 @@ """ import argparse +import json +import math +from dataclasses import dataclass import os import re import shutil import sys +import tomllib +from pathlib import Path from datetime import datetime, timezone try: @@ -49,7 +54,7 @@ BENCHER_PATTERN = re.compile( r"test\s+(\w+)/(\w+)/(\d+)\s+\.\.\.\s*" r"(?:(?!\btest\s)[\s\S])*?" - r"bench:\s+([\d,]+)\s+ns/iter" + r"bench:\s+([\d,]+)\s+ns/iter\s+\(\+/-\s+([\d,]+)\)" ) # Operations in the order they are reported, mapped to human readable labels. @@ -79,14 +84,37 @@ README_END_MARKER = "" +@dataclass(frozen=True) +class Measurement: + """Median and standard deviation, both in nanoseconds.""" + median: int + deviation: int = 0 + + +def measurement(value): + return value if isinstance(value, Measurement) else Measurement(value) + + def parse_text(content): - """Parse bencher-format text into ``{operation: {variant: ns_per_iter}}``.""" + """Parse bencher-format text into ``{operation: {variant: Measurement}}``.""" results = {op: {} for op, _ in OPERATIONS} + sizes = set() for match in BENCHER_PATTERN.finditer(content): - operation, variant, _size, ns_str = match.groups() + operation, variant, _size, ns_str, deviation = match.groups() + sizes.add(_size) + if len(sizes) != 1: + raise ValueError("Mixed benchmark sizes") + if operation not in results: + raise ValueError(f"Unknown benchmark operation: {operation}") if operation in results: - results[operation][variant] = int(ns_str.replace(",", "")) + if variant not in {key for key, _, _ in VARIANTS}: + raise ValueError(f"Unknown benchmark variant: {variant}") + if variant in results[operation]: + raise ValueError(f"Duplicate benchmark: {operation}/{variant}") + results[operation][variant] = Measurement( + int(ns_str.replace(",", "")), int(deviation.replace(",", "")) + ) return results @@ -121,19 +149,43 @@ def has_any_results(results): return any(measurements for measurements in results.values()) +def format_duration(value): + """Format nanoseconds with three significant digits, promoting rounded units.""" + value = measurement(value).median + units = ("ns", "µs", "ms", "s") + index = 0 + while value >= 1000 and index < len(units) - 1: + value /= 1000 + index += 1 + if float(f"{value:.3g}") >= 1000 and index < len(units) - 1: + value /= 1000 + index += 1 + value = float(f"{value:.3g}") + decimals = max(0, 2 - math.floor(math.log10(value))) if value > 0 else 2 + return f"{value:.{decimals}f} {units[index]}" + + def format_speedup(value, baseline): - """Annotate ``value`` with how it compares to the ``baseline`` measurement.""" - if not value: + """Compare medians; differences under 5% or overlapping ±1 SD are noise.""" + value, baseline = measurement(value), measurement(baseline) + if not value.median: return "N/A" - if not baseline: - return f"{value}" - if value <= baseline: - return f"{value} ({baseline / value:.1f}x faster)" - return f"{value} ({value / baseline:.1f}x slower)" + duration = format_duration(value) + if not baseline.median: + return duration + overlap = max(value.median - value.deviation, baseline.median - baseline.deviation) <= min( + value.median + value.deviation, baseline.median + baseline.deviation + ) + if abs(value.median - baseline.median) / baseline.median < 0.05 or overlap: + return f"{duration} (≈ same)" + ratio = max(value.median, baseline.median) / min(value.median, baseline.median) + factor = f"{ratio:,.0f}" if ratio >= 100 else f"{ratio:.3g}" + direction = "faster" if value.median < baseline.median else "slower" + return f"{duration} ({factor}× {direction})" def format_results_table(results): - """Render the Markdown results table (all numbers in nanoseconds).""" + """Render the Markdown results table with readable durations and comparisons.""" labels = [label for _, label, _ in VARIANTS] widths = [max(len(label), 13) for label in labels] operation_width = max(len(label) for _, label in OPERATIONS) @@ -151,7 +203,7 @@ def format_results_table(results): for key, _label, _color in VARIANTS: value = results[op].get(key, 0) if key == BASELINE: - cells.append(str(value) if value else "N/A") + cells.append(format_duration(value) if value else "N/A") else: cells.append(format_speedup(value, baseline)) row = "| " + op_label.ljust(operation_width) + " | " @@ -162,13 +214,14 @@ def format_results_table(results): return "\n".join(lines) -def build_provenance(benchmark_links=None, background_links=None, generated_at=None): +def build_provenance(benchmark_links=None, background_links=None, generated_at=None, metadata=None): """Describe how and when the committed results were produced.""" - benchmark_links = benchmark_links or os.environ.get("BENCHMARK_LINK_COUNT", "1000") - background_links = background_links or os.environ.get( + metadata = metadata or {} + benchmark_links = metadata.get("benchmark_links") or benchmark_links or os.environ.get("BENCHMARK_LINK_COUNT", "1000") + background_links = metadata.get("background_links") or background_links or os.environ.get( "BACKGROUND_LINK_COUNT", "3000" ) - generated_at = generated_at or datetime.now(timezone.utc).strftime( + generated_at = metadata.get("date") or generated_at or datetime.now(timezone.utc).strftime( "%Y-%m-%d %H:%M UTC" ) @@ -181,10 +234,21 @@ def build_provenance(benchmark_links=None, background_links=None, generated_at=N f"(https://github.com/{repository}/actions/runs/{run_id})" ) + source = metadata.get("source", source) + root = Path(__file__).resolve().parents[1] + packages = tomllib.loads((root / "rust/Cargo.lock").read_text())["package"] + versions = metadata.get("versions", { + "SpacetimeDB server/CLI": "unknown (not recorded)", + "Rust SDK": next(p["version"] for p in packages if p["name"] == "spacetimedb-sdk"), + "Doublets (patched)": next(p["version"] for p in packages if p["name"] == "doublets"), + }) + cpus = metadata.get("cpus", {"local": "unknown (not recorded)"}) + details = "; ".join(f"{name} {version}" for name, version in versions.items()) + cpu_details = "; ".join(f"{backend}: {cpu}" for backend, cpu in cpus.items()) return ( f"_Generated {generated_at} by {source} — " f"{benchmark_links} benchmarked links, " - f"{background_links} background links._" + f"{background_links} background links. {details}; CPU — {cpu_details}._" ) @@ -194,7 +258,7 @@ def render_results_section(results, provenance=None): return f"{provenance}\n\n{format_results_table(results)}" -def update_readme(readme_path, section): +def update_readme(readme_path, section, language="rust"): """Replace the marked results section of the README. Returns ``True`` when the file was modified. @@ -202,17 +266,19 @@ def update_readme(readme_path, section): with open(readme_path, "r", encoding="utf-8") as handle: readme = handle.read() - if README_START_MARKER not in readme or README_END_MARKER not in readme: + start = README_START_MARKER if language == "rust" else "" + end = README_END_MARKER if language == "rust" else "" + if start not in readme or end not in readme: raise ValueError( f"{readme_path} does not contain the " - f"{README_START_MARKER} / {README_END_MARKER} markers" + f"{start} / {end} markers" ) pattern = re.compile( - re.escape(README_START_MARKER) + r".*?" + re.escape(README_END_MARKER), + re.escape(start) + r".*?" + re.escape(end), re.DOTALL, ) - replacement = f"{README_START_MARKER}\n{section}\n{README_END_MARKER}" + replacement = f"{start}\n{section}\n{end}" updated = pattern.sub(lambda _match: replacement, readme, count=1) if updated == readme: @@ -225,7 +291,7 @@ def update_readme(readme_path, section): def _series(results, variant): """Measurements of one variant across all operations, 0 when missing.""" - return [results[op].get(variant, 0) for op, _ in OPERATIONS] + return [measurement(results[op].get(variant, 0)).median for op, _ in OPERATIONS] def _ensure_min_visible(values, minimum): @@ -233,7 +299,7 @@ def _ensure_min_visible(values, minimum): return [max(value, minimum) if value > 0 else 0 for value in values] -def _plot(results, filename, log_scale, output_dir): +def _plot(results, filename, log_scale, output_dir, language): positions = np.arange(len(OPERATIONS)) width = 0.15 figure, axes = plt.subplots(figsize=(12, 8)) @@ -259,7 +325,8 @@ def _plot(results, filename, log_scale, output_dir): axes.barh(positions + offset, plotted[key], width, label=label, color=color) axes.set_xlabel("Time (ns) – log scale" if log_scale else "Time (ns)") - axes.set_title("Benchmark Comparison: SpacetimeDB vs Doublets (Rust)") + label = "Rust" if language == "rust" else "C#" + axes.set_title(f"Benchmark Comparison: SpacetimeDB vs Doublets ({label})") axes.set_yticks(positions) axes.set_yticklabels([label for _, label in OPERATIONS]) if log_scale: @@ -273,18 +340,18 @@ def _plot(results, filename, log_scale, output_dir): print(f"Generated {path}") -def generate_charts(results, output_dir=""): +def generate_charts(results, output_dir="", language="rust"): """Generate the linear and logarithmic comparison charts.""" if not HAS_MATPLOTLIB: return - _plot(results, "bench_rust.png", log_scale=False, output_dir=output_dir) - _plot(results, "bench_rust_log_scale.png", log_scale=True, output_dir=output_dir) + _plot(results, f"bench_{language}.png", log_scale=False, output_dir=output_dir, language=language) + _plot(results, f"bench_{language}_log_scale.png", log_scale=True, output_dir=output_dir, language=language) -def copy_charts(docs_dir, output_dir=""): +def copy_charts(docs_dir, output_dir="", language="rust"): """Copy generated charts into the documentation directory.""" os.makedirs(docs_dir, exist_ok=True) - for chart in ("bench_rust.png", "bench_rust_log_scale.png"): + for chart in (f"bench_{language}.png", f"bench_{language}_log_scale.png"): source = os.path.join(output_dir, chart) if output_dir else chart if os.path.exists(source): shutil.copyfile(source, os.path.join(docs_dir, chart)) @@ -295,8 +362,8 @@ def parse_args(argv): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "input", - nargs="?", - default="out.txt", + nargs="*", + default=["out.txt"], help="criterion bencher output file (default: out.txt)", ) parser.add_argument( @@ -319,19 +386,46 @@ def parse_args(argv): default="", help="directory the charts are written to (default: working directory)", ) + parser.add_argument("--language", choices=["rust", "csharp"], default="rust") + parser.add_argument("--metadata", nargs="+", default=[]) return parser.parse_args(argv) def main(argv=None): args = parse_args(argv if argv is not None else sys.argv[1:]) - results = parse_results(args.input) + results = {op: {} for op, _ in OPERATIONS} + metadata = {} + sizes = set() + for filename in args.input: + if os.path.exists(filename): + sizes.update(match.group(3) for match in BENCHER_PATTERN.finditer(Path(filename).read_text(encoding="utf-8"))) + if len(sizes) > 1: + raise ValueError("Mixed benchmark sizes across input files") + for op, variants in parse_results(filename).items(): + if results[op].keys() & variants.keys(): + raise ValueError(f"Duplicate results for {op}") + results[op].update(variants) + for filename in args.metadata: + item = json.loads(Path(filename).read_text(encoding="utf-8")) + for key in ("benchmark_links", "background_links", "source", "versions"): + if key in metadata and metadata[key] != item[key]: + raise ValueError(f"Inconsistent metadata: {key}") + metadata.update({key: value for key, value in item.items() if key != "cpus"}) + metadata.setdefault("cpus", {}).update(item["cpus"]) + if not has_any_results(results): print(f"No benchmark data found in {args.input}") - print(report_input_excerpt(args.input)) + print("\n".join(report_input_excerpt(path) for path in args.input)) return 1 - section = render_results_section(results) + missing = [f"{op}/{key}" for op, _ in OPERATIONS for key, _, _ in VARIANTS if key not in results[op]] + if missing: + raise ValueError("Missing benchmark results: " + ", ".join(missing)) + size = next(iter(sizes)) + if "benchmark_links" in metadata and str(metadata["benchmark_links"]) != size: + raise ValueError("Benchmark size does not match metadata") + section = render_results_section(results, build_provenance(benchmark_links=size, metadata=metadata)) table = format_results_table(results) print(table) @@ -339,13 +433,21 @@ def main(argv=None): handle.write(section + "\n") print(f"Generated {args.results}") - generate_charts(results, args.output_dir) + generate_charts(results, args.output_dir, args.language) if args.docs_dir: - copy_charts(args.docs_dir, args.output_dir) + copy_charts(args.docs_dir, args.output_dir, args.language) if args.readme: - changed = update_readme(args.readme, section) + readme_section = section + if args.language == "csharp" and HAS_MATPLOTLIB: + chart_dir = args.docs_dir or args.output_dir or "." + relative = os.path.relpath(chart_dir, Path(args.readme).resolve().parent) + readme_section += ( + f"\n\n![C# benchmark (linear scale)]({relative}/bench_csharp.png)" + f"\n![C# benchmark (log scale)]({relative}/bench_csharp_log_scale.png)" + ) + changed = update_readme(args.readme, readme_section, args.language) print( f"{'Updated' if changed else 'No changes needed in'} {args.readme}", ) diff --git a/rust/results.md b/rust/results.md index 4cac6dd..85b4084 100644 --- a/rust/results.md +++ b/rust/results.md @@ -1,11 +1,11 @@ -_Generated 2026-09-16 09:20 UTC by [GitHub Actions run 35075414684](https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/actions/runs/35075414684) — 1000 benchmarked links, 3000 background links._ +_Generated 2026-09-16 09:20 UTC by [GitHub Actions run 35075414684](https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/actions/runs/35075414684) — 1000 benchmarked links, 3000 background links. SpacetimeDB server/CLI unknown (not recorded); Rust SDK 2.10.1 (lockfile); Doublets (patched) 0.1.0-pre+beta.15; CPU — historical run: unknown (not recorded)._ | Operation | Doublets United Volatile | Doublets United NonVolatile | Doublets Split Volatile | Doublets Split NonVolatile | SpacetimeDB | |-----------------|--------------------------|-----------------------------|-------------------------|----------------------------|---------------| -| Create | 76689 (33385.0x faster) | 76567 (33438.2x faster) | 47102 (54355.7x faster) | 47395 (54019.7x faster) | 2560262853 | -| Update | 247101 (10703.3x faster) | 247021 (10706.8x faster) | 35819 (73838.1x faster) | 36029 (73407.7x faster) | 2644805261 | -| Delete | 180271 (7169.4x faster) | 181398 (7124.8x faster) | 94116 (13732.3x faster) | 100143 (12905.9x faster) | 1292432593 | -| Query All | 21580 (1.0x faster) | 21592 (1.0x faster) | 28388 (1.3x slower) | 28312 (1.3x slower) | 22126 | -| Query by Id | 55 (198863.3x faster) | 59 (185381.0x faster) | 1018 (10744.1x faster) | 1018 (10744.1x faster) | 10937480 | -| Query by Source | 1476 (110.8x faster) | 1453 (112.5x faster) | 480 (340.7x faster) | 478 (342.1x faster) | 163528 | -| Query by Target | 1514 (109.3x faster) | 1478 (111.9x faster) | 445 (371.8x faster) | 393 (420.9x faster) | 165432 | +| Create | 76.7 µs (33,385× faster) | 76.6 µs (33,438× faster) | 47.1 µs (54,356× faster) | 47.4 µs (54,020× faster) | 2.56 s | +| Update | 247 µs (10,703× faster) | 247 µs (10,707× faster) | 35.8 µs (73,838× faster) | 36.0 µs (73,408× faster) | 2.64 s | +| Delete | 180 µs (7,169× faster) | 181 µs (7,125× faster) | 94.1 µs (13,732× faster) | 100 µs (12,906× faster) | 1.29 s | +| Query All | 21.6 µs (≈ same) | 21.6 µs (≈ same) | 28.4 µs (1.28× slower) | 28.3 µs (1.28× slower) | 22.1 µs | +| Query by Id | 55.0 ns (198,863× faster) | 59.0 ns (185,381× faster) | 1.02 µs (10,744× faster) | 1.02 µs (10,744× faster) | 10.9 ms | +| Query by Source | 1.48 µs (111× faster) | 1.45 µs (113× faster) | 480 ns (341× faster) | 478 ns (342× faster) | 164 µs | +| Query by Target | 1.51 µs (109× faster) | 1.48 µs (112× faster) | 445 ns (372× faster) | 393 ns (421× faster) | 165 µs | diff --git a/rust/test_out.py b/rust/test_out.py index cd202b2..d6ef7e5 100644 --- a/rust/test_out.py +++ b/rust/test_out.py @@ -10,8 +10,10 @@ """ import os +import json import tempfile import unittest +from pathlib import Path import out @@ -96,8 +98,8 @@ def test_parses_every_operation_and_variant(self): def test_parses_thousand_separators(self): results = out.parse_results(self.path) - self.assertEqual(results["create"]["SpacetimeDB"], 3_159_452_615) - self.assertEqual(results["create"]["Doublets_United_Volatile"], 84_823) + self.assertEqual(results["create"]["SpacetimeDB"], out.Measurement(3_159_452_615, 12_345)) + self.assertEqual(results["create"]["Doublets_United_Volatile"], out.Measurement(84_823, 1_000)) def test_missing_file_yields_empty_results(self): results = out.parse_results(os.path.join(self.tmp.name, "absent.txt")) @@ -133,8 +135,8 @@ def test_interleaved_criterion_errors_do_not_hide_measurements(self): results = out.parse_results(path) - self.assertEqual(results["create"]["SpacetimeDB"], 24_992_765) - self.assertEqual(results["create"]["Doublets_United_Volatile"], 464) + self.assertEqual(results["create"]["SpacetimeDB"], out.Measurement(24_992_765, 1_790_323_177)) + self.assertEqual(results["create"]["Doublets_United_Volatile"], out.Measurement(464, 12)) def test_measurement_is_not_borrowed_from_the_next_benchmark(self): """An aborted benchmark must not take the following benchmark's number.""" @@ -147,7 +149,51 @@ def test_measurement_is_not_borrowed_from_the_next_benchmark(self): results = out.parse_results(path) self.assertNotIn("SpacetimeDB", results["create"]) - self.assertEqual(results["create"]["Doublets_United_Volatile"], 464) + self.assertEqual(results["create"]["Doublets_United_Volatile"], out.Measurement(464, 12)) + + +class ReportValidationTests(unittest.TestCase): + def test_unknown_operation_and_variant_rejected(self): + for text in [SAMPLE_OUT_TXT.replace("create/SpacetimeDB", "typo/SpacetimeDB"), SAMPLE_OUT_TXT.replace("create/SpacetimeDB", "create/Typo")]: + with self.assertRaises(ValueError): + out.parse_text(text) + + def test_duplicates_and_mixed_sizes_rejected(self): + for text in [SAMPLE_OUT_TXT + SAMPLE_OUT_TXT, SAMPLE_OUT_TXT.replace("create/SpacetimeDB/1000", "create/SpacetimeDB/10")]: + with self.assertRaises(ValueError): + out.parse_text(text) + + def test_five_percent_boundary_and_non_overlapping_deviations(self): + self.assertIn("slower", out.format_speedup(105, 100)) + self.assertIn("≈ same", out.format_speedup(104, 100)) + self.assertIn("faster", out.format_speedup(out.Measurement(100, 1), out.Measurement(140, 1))) + + def test_csharp_update_preserves_rust_section(self): + with tempfile.TemporaryDirectory() as directory: + path = write(os.path.join(directory, "README.md"), README_TEMPLATE + "\n\nold\n\n") + out.update_readme(path, "C# table", "csharp") + content = open(path, encoding="utf-8").read() + self.assertIn("Benchmark results will be generated", content) + self.assertIn("C# table", content) + + def test_recorded_metadata_survives_report_generation(self): + provenance = out.build_provenance(metadata={"date": "2026-01-01", "source": "recorded run", "benchmark_links": "10", "background_links": "30", "versions": {"SDK": "2.10.1"}, "cpus": {"doublets": "CPU A", "spacetimedb": "CPU B"}}) + for value in ["2026-01-01", "recorded run", "10 benchmarked", "30 background", "SDK 2.10.1", "CPU A", "CPU B"]: + self.assertIn(value, provenance) + + def test_incomplete_input_cannot_publish(self): + with tempfile.TemporaryDirectory() as directory: + path = write(os.path.join(directory, "out.txt"), SAMPLE_OUT_TXT.splitlines()[3] + "\n") + with self.assertRaisesRegex(ValueError, "Missing benchmark results"): + out.main([path]) + + def test_separate_backends_with_different_sizes_cannot_publish(self): + with tempfile.TemporaryDirectory() as directory: + lines = SAMPLE_OUT_TXT.splitlines() + baseline = write(os.path.join(directory, "server.txt"), "\n".join(line for line in lines if "/SpacetimeDB/" in line)) + doublets = write(os.path.join(directory, "doublets.txt"), "\n".join(line.replace("/1000", "/10") for line in lines if "/Doublets_" in line)) + with self.assertRaisesRegex(ValueError, "Mixed benchmark sizes"): + out.main([baseline, doublets, "--results", os.path.join(directory, "results.md"), "--output-dir", directory]) class InputExcerptTests(unittest.TestCase): @@ -180,16 +226,33 @@ def test_reports_the_tail(self): class SpeedupFormattingTests(unittest.TestCase): def test_faster_than_baseline(self): - self.assertEqual(out.format_speedup(100, 1000), "100 (10.0x faster)") + self.assertEqual(out.format_speedup(100, 1000), "100 ns (10× faster)") def test_slower_than_baseline(self): - self.assertEqual(out.format_speedup(1000, 100), "1000 (10.0x slower)") + self.assertEqual(out.format_speedup(1000, 100), "1.00 µs (10× slower)") def test_missing_measurement(self): self.assertEqual(out.format_speedup(0, 1000), "N/A") def test_missing_baseline_reports_raw_value(self): - self.assertEqual(out.format_speedup(100, 0), "100") + self.assertEqual(out.format_speedup(100, 0), "100 ns") + + +class NoiseAndUnitsRegressionTests(unittest.TestCase): + def test_query_all_difference_within_five_percent_is_same(self): + self.assertEqual(out.format_speedup(21580, 22126), "21.6 µs (≈ same)") + + def test_overlapping_standard_deviations_are_same(self): + self.assertEqual(out.format_speedup(out.Measurement(100, 25), out.Measurement(140, 20)), "100 ns (≈ same)") + + def test_units_have_three_significant_digits(self): + for value, expected in [(55, "55.0 ns"), (76689, "76.7 µs"), (2560262853, "2.56 s"), (999999, "1.00 ms")]: + with self.subTest(value=value): + self.assertEqual(out.format_duration(value), expected) + + def test_deviation_is_preserved_by_parser(self): + result = out.parse_text("test query_all/SpacetimeDB/1000 ... bench: 22,126 ns/iter (+/- 1,250)") + self.assertEqual(result["query_all"]["SpacetimeDB"].deviation, 1250) class ResultsTableTests(unittest.TestCase): @@ -218,8 +281,8 @@ def test_doublets_cells_are_annotated_with_speedup(self): ) # 3_159_452_615 / 84_823 = 37,247.6x - self.assertIn("84823 (37247.6x faster)", create_row) - self.assertIn("3159452615", create_row) + self.assertIn("84.8 µs (37,248× faster)", create_row) + self.assertIn("3.16 s", create_row) def test_missing_measurements_render_as_not_available(self): partial = {operation: {} for operation, _label in out.OPERATIONS} @@ -306,6 +369,32 @@ def setUp(self): self.results = os.path.join(self.tmp.name, "results.md") self.docs = os.path.join(self.tmp.name, "docs", "benchmarks") + def test_combines_csharp_backends_and_vm_metadata(self): + metadata_paths, input_paths = [], [] + for backend in ("spacetimedb", "doublets"): + lines = [line for line in SAMPLE_OUT_TXT.splitlines() if + ("/SpacetimeDB/" if backend == "spacetimedb" else "/Doublets_") in line] + input_paths.append(write(os.path.join(self.tmp.name, backend + ".txt"), "\n".join(lines))) + item = {"date": "2026-01-01", "source": "recorded CI run", "benchmark_links": "1000", "background_links": "3000", + "versions": {"C# SDK": "2.10.1", "Doublets": "0.18.1"}, "cpus": {backend: backend + " CPU"}} + metadata_paths.append(write(os.path.join(self.tmp.name, backend + ".json"), json.dumps(item))) + write(self.readme, README_TEMPLATE + "\n\nold\n\n") + self.assertEqual(out.main(input_paths + ["--metadata"] + metadata_paths + [ + "--language", "csharp", "--results", self.results, "--readme", self.readme, + "--output-dir", self.tmp.name, "--docs-dir", self.docs]), 0) + content = Path(self.readme).read_text(encoding="utf-8") + for value in ("spacetimedb CPU", "doublets CPU", "C# SDK 2.10.1", "Doublets 0.18.1", "Benchmark results will be generated"): + self.assertIn(value, content) + if out.HAS_MATPLOTLIB: + self.assertIn("![C# benchmark (linear scale)](docs/benchmarks/bench_csharp.png)", content) + self.assertTrue(Path(self.docs, "bench_csharp_log_scale.png").exists()) + + def test_recorded_size_must_match_measurements(self): + metadata = write(os.path.join(self.tmp.name, "metadata.json"), json.dumps({ + "benchmark_links": "10", "background_links": "30", "source": "run", "versions": {}, "cpus": {}})) + with self.assertRaisesRegex(ValueError, "size does not match metadata"): + out.main([self.input, "--metadata", metadata, "--results", self.results]) + def test_writes_results_and_updates_readme(self): exit_code = out.main( [ From f13f071bca5c50fc43fb9a140d1ce22f718f5f38 Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:38:06 +0000 Subject: [PATCH 5/7] Run isolated backend benchmarks and remove template release scaffolding --- .github/workflows/release.yml | 411 ------------------ .github/workflows/rust-benchmark.yml | 490 +++++++--------------- .gitignore | 9 + .pre-commit-config.yaml | 6 +- CONTRIBUTING.md | 320 ++------------ Cargo.lock | 101 ----- Cargo.toml | 45 -- README.md | 194 +++++---- changelog.d/20261006_comparison_parity.md | 14 + examples/basic_usage.rs | 34 -- scripts/benchmark-metadata.py | 55 +++ scripts/install-spacetimedb.sh | 21 + scripts/spacetimedb-version | 1 + scripts/start-spacetimedb.sh | 18 + src/lib.rs | 136 ------ src/main.rs | 29 -- tests/integration_test.rs | 88 ---- 17 files changed, 423 insertions(+), 1549 deletions(-) delete mode 100644 .github/workflows/release.yml delete mode 100644 Cargo.lock delete mode 100644 Cargo.toml create mode 100644 changelog.d/20261006_comparison_parity.md delete mode 100644 examples/basic_usage.rs create mode 100644 scripts/benchmark-metadata.py create mode 100755 scripts/install-spacetimedb.sh create mode 100644 scripts/spacetimedb-version create mode 100755 scripts/start-spacetimedb.sh delete mode 100644 src/lib.rs delete mode 100644 src/main.rs delete mode 100644 tests/integration_test.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 9012f21..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,411 +0,0 @@ -name: CI/CD Pipeline - -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened] - workflow_dispatch: - inputs: - release_mode: - description: 'Manual release mode' - required: true - type: choice - default: 'instant' - options: - - instant - - changelog-pr - bump_type: - description: 'Version bump type' - required: true - type: choice - options: - - patch - - minor - - major - description: - description: 'Release description (optional)' - required: false - type: string - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - CARGO_TERM_COLOR: always - RUSTFLAGS: -Dwarnings - # Support both CARGO_REGISTRY_TOKEN (cargo's native env var) and CARGO_TOKEN (for backwards compatibility) - CARGO_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN || secrets.CARGO_TOKEN }} - -jobs: - # === DETECT CHANGES - determines which jobs should run === - detect-changes: - name: Detect Changes - runs-on: ubuntu-latest - if: github.event_name != 'workflow_dispatch' - outputs: - rs-changed: ${{ steps.changes.outputs.rs-changed }} - toml-changed: ${{ steps.changes.outputs.toml-changed }} - mjs-changed: ${{ steps.changes.outputs.mjs-changed }} - docs-changed: ${{ steps.changes.outputs.docs-changed }} - workflow-changed: ${{ steps.changes.outputs.workflow-changed }} - any-code-changed: ${{ steps.changes.outputs.any-code-changed }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Detect changes - id: changes - env: - GITHUB_EVENT_NAME: ${{ github.event_name }} - GITHUB_BASE_SHA: ${{ github.event.pull_request.base.sha }} - GITHUB_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: node scripts/detect-code-changes.mjs - - # === CHANGELOG CHECK - only runs on PRs with code changes === - # Docs-only PRs (./docs folder, markdown files) don't require changelog fragments - changelog: - name: Changelog Fragment Check - runs-on: ubuntu-latest - needs: [detect-changes] - if: github.event_name == 'pull_request' && needs.detect-changes.outputs.any-code-changed == 'true' - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Check for changelog fragments - env: - GITHUB_BASE_REF: ${{ github.base_ref }} - run: node scripts/check-changelog-fragment.mjs - - # === VERSION CHECK - prevents manual version modification in PRs === - # This ensures versions are only modified by the automated release pipeline - version-check: - name: Version Modification Check - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Check for manual version changes - env: - GITHUB_EVENT_NAME: ${{ github.event_name }} - GITHUB_HEAD_REF: ${{ github.head_ref }} - GITHUB_BASE_REF: ${{ github.base_ref }} - run: node scripts/check-version-modification.mjs - - # === LINT AND FORMAT CHECK === - # Lint runs independently of changelog check - it's a fast check that should always run - # See: https://github.com/link-assistant/hive-mind/pull/1024 for why this dependency was removed - lint: - name: Lint and Format Check - runs-on: ubuntu-latest - needs: [detect-changes] - # Note: always() is required because detect-changes is skipped on workflow_dispatch, - # and without always(), this job would also be skipped even though its condition includes workflow_dispatch. - # See: https://github.com/actions/runner/issues/491 - if: | - always() && !cancelled() && ( - github.event_name == 'push' || - github.event_name == 'workflow_dispatch' || - needs.detect-changes.outputs.rs-changed == 'true' || - needs.detect-changes.outputs.toml-changed == 'true' || - needs.detect-changes.outputs.mjs-changed == 'true' || - needs.detect-changes.outputs.docs-changed == 'true' || - needs.detect-changes.outputs.workflow-changed == 'true' - ) - steps: - - uses: actions/checkout@v4 - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Cache cargo registry - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - - name: Check formatting - run: cargo fmt --all -- --check - - - name: Run Clippy - run: cargo clippy --all-targets --all-features - - - name: Check file size limit - run: node scripts/check-file-size.mjs - - # === TEST === - # Test runs independently of changelog check - test: - name: Test (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - needs: [detect-changes, changelog] - # Run if: push event, OR changelog succeeded, OR changelog was skipped (docs-only PR) - if: always() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' || needs.changelog.result == 'success' || needs.changelog.result == 'skipped') - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - steps: - - uses: actions/checkout@v4 - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache cargo registry - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - - name: Run tests - run: cargo test --all-features --verbose - - - name: Run doc tests - run: cargo test --doc --verbose - - # === BUILD === - # Build package - only runs if lint and test pass - build: - name: Build Package - runs-on: ubuntu-latest - needs: [lint, test] - if: always() && !cancelled() && needs.lint.result == 'success' && needs.test.result == 'success' - steps: - - uses: actions/checkout@v4 - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache cargo registry - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-build- - - - name: Build release - run: cargo build --release --verbose - - - name: Check package - run: cargo package --list - - # === AUTO RELEASE === - # Automatic release on push to main using changelog fragments - # This job automatically bumps version based on fragments in changelog.d/ - auto-release: - name: Auto Release - needs: [lint, test, build] - # Note: always() ensures consistent behavior with other jobs that depend on jobs using always(). - if: | - always() && !cancelled() && - github.event_name == 'push' && - github.ref == 'refs/heads/main' && - needs.build.result == 'success' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Configure git - run: node scripts/git-config.mjs - - - name: Determine bump type from changelog fragments - id: bump_type - run: node scripts/get-bump-type.mjs - - - name: Check if version already released or no fragments - id: check - env: - HAS_FRAGMENTS: ${{ steps.bump_type.outputs.has_fragments }} - run: node scripts/check-release-needed.mjs - - - name: Collect changelog and bump version - id: version - if: steps.check.outputs.should_release == 'true' && steps.check.outputs.skip_bump != 'true' - run: | - node scripts/version-and-commit.mjs \ - --bump-type "${{ steps.bump_type.outputs.bump_type }}" - - - name: Get current version - id: current_version - if: steps.check.outputs.should_release == 'true' - run: node scripts/get-version.mjs - - - name: Build release - if: steps.check.outputs.should_release == 'true' - run: cargo build --release - - - name: Publish to Crates.io - if: steps.check.outputs.should_release == 'true' - id: publish-crate - env: - CARGO_TOKEN: ${{ secrets.CARGO_TOKEN }} - run: node scripts/publish-crate.mjs - - - name: Create GitHub Release - if: steps.check.outputs.should_release == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: node scripts/create-github-release.mjs --release-version "${{ steps.current_version.outputs.version }}" --repository "${{ github.repository }}" - - # === MANUAL INSTANT RELEASE === - # Manual release via workflow_dispatch - only after CI passes - manual-release: - name: Instant Release - needs: [lint, test, build] - # Note: always() is required to evaluate the condition when dependencies use always(). - # The build job ensures lint and test passed before this job runs. - if: | - always() && !cancelled() && - github.event_name == 'workflow_dispatch' && - github.event.inputs.release_mode == 'instant' && - needs.build.result == 'success' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Configure git - run: node scripts/git-config.mjs - - - name: Collect changelog fragments - run: node scripts/collect-changelog.mjs - - - name: Version and commit - id: version - env: - BUMP_TYPE: ${{ github.event.inputs.bump_type }} - DESCRIPTION: ${{ github.event.inputs.description }} - run: node scripts/version-and-commit.mjs --bump-type "${{ github.event.inputs.bump_type }}" --description "${{ github.event.inputs.description }}" - - - name: Build release - if: steps.version.outputs.version_committed == 'true' || steps.version.outputs.already_released == 'true' - run: cargo build --release - - - name: Publish to Crates.io - if: steps.version.outputs.version_committed == 'true' || steps.version.outputs.already_released == 'true' - id: publish-crate - env: - CARGO_TOKEN: ${{ secrets.CARGO_TOKEN }} - run: node scripts/publish-crate.mjs - - - name: Create GitHub Release - if: steps.version.outputs.version_committed == 'true' || steps.version.outputs.already_released == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: node scripts/create-github-release.mjs --release-version "${{ steps.version.outputs.new_version }}" --repository "${{ github.repository }}" - - # === MANUAL CHANGELOG PR === - changelog-pr: - name: Create Changelog PR - if: github.event_name == 'workflow_dispatch' && github.event.inputs.release_mode == 'changelog-pr' - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '20.x' - - - name: Create changelog fragment - env: - BUMP_TYPE: ${{ github.event.inputs.bump_type }} - DESCRIPTION: ${{ github.event.inputs.description }} - run: node scripts/create-changelog-fragment.mjs --bump-type "${{ github.event.inputs.bump_type }}" --description "${{ github.event.inputs.description }}" - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v7 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: 'chore: add changelog for manual ${{ github.event.inputs.bump_type }} release' - branch: changelog-manual-release-${{ github.run_id }} - delete-branch: true - title: 'chore: manual ${{ github.event.inputs.bump_type }} release' - body: | - ## Manual Release Request - - This PR was created by a manual workflow trigger to prepare a **${{ github.event.inputs.bump_type }}** release. - - ### Release Details - - **Type:** ${{ github.event.inputs.bump_type }} - - **Description:** ${{ github.event.inputs.description || 'Manual release' }} - - **Triggered by:** @${{ github.actor }} - - ### Next Steps - 1. Review the changelog fragment in this PR - 2. Merge this PR to main - 3. The automated release workflow will publish to crates.io and create a GitHub release diff --git a/.github/workflows/rust-benchmark.yml b/.github/workflows/rust-benchmark.yml index b84c108..69ac932 100644 --- a/.github/workflows/rust-benchmark.yml +++ b/.github/workflows/rust-benchmark.yml @@ -1,28 +1,31 @@ -name: Rust Benchmark +name: Benchmarks on: push: branches: [main, master] - paths: ['rust/**', '.github/workflows/rust-benchmark.yml'] + paths: ['rust/**', 'csharp/**', 'scripts/**', '.github/workflows/rust-benchmark.yml'] pull_request: branches: [main, master] - paths: ['rust/**', '.github/workflows/rust-benchmark.yml'] + paths: ['rust/**', 'csharp/**', 'scripts/**', '.github/workflows/rust-benchmark.yml'] + workflow_dispatch: + +concurrency: + group: benchmarks-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 - # Pinned nightly, kept in sync with rust/rust-toolchain.toml. The patched - # doublets crates rely on unstable APIs, so a rolling nightly breaks the build - # without any change in this repository. toolchain: nightly-2026-09-16 - -defaults: - run: - working-directory: rust + SPACETIMEDB_URI: http://localhost:3000 + SPACETIMEDB_DB: benchmark-links jobs: - test: - name: Test (${{ matrix.os }}) + test-rust: + name: Rust tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: @@ -31,375 +34,206 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 - - - name: Setup Rust (pinned nightly) - uses: dtolnay/rust-toolchain@master + - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.toolchain }} components: rustfmt, clippy targets: wasm32-unknown-unknown - - - name: Cache cargo registry - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@v2 with: - workspaces: rust -> target - cache-on-failure: "true" - - - name: Check formatting - run: cargo fmt --all -- --check - - - name: Run Clippy - run: cargo clippy --all-targets - - - name: Install SpacetimeDB CLI - shell: bash - run: | - curl -sSf https://install.spacetimedb.com | sh -s -- -y - echo "$HOME/.local/bin" >> $GITHUB_PATH - working-directory: . - - - name: Build SpacetimeDB module (WASM) - run: cargo build --release --target wasm32-unknown-unknown + workspaces: | + rust -> target + rust/spacetime-module -> target + - name: Rust quality checks + working-directory: rust + run: | + cargo fmt --all -- --check + cargo clippy --locked --all-targets -- -D warnings + - run: scripts/install-spacetimedb.sh + - name: Build module working-directory: rust/spacetime-module + run: cargo build --locked --release --target wasm32-unknown-unknown + - run: scripts/start-spacetimedb.sh + - name: All Rust tests, including backend equivalence + working-directory: rust + run: cargo test --locked -- --test-threads=1 --include-ignored + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: rust-test-logs-${{ matrix.os }} + path: ci-logs/ - - name: Start SpacetimeDB server - shell: bash - run: | - spacetime start & - echo "Waiting for SpacetimeDB server to be ready..." - for i in $(seq 1 30); do - if curl -sf http://localhost:3000/ > /dev/null 2>&1; then - echo "SpacetimeDB server is ready" - break - fi - sleep 1 - done - working-directory: . - - - name: Publish SpacetimeDB module - shell: bash - run: | - spacetime publish \ - --server http://localhost:3000 \ - --bin-path target/wasm32-unknown-unknown/release/spacetime_module.wasm \ - --yes \ - benchmark-links + test-csharp: + name: C# tests + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.toolchain }} + targets: wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust/spacetime-module -> target + - run: dotnet restore csharp/SpacetimeDBVSDoublets.slnx --locked-mode + - run: dotnet format csharp/SpacetimeDBVSDoublets.slnx --no-restore --verify-no-changes + - run: dotnet build csharp/SpacetimeDBVSDoublets.slnx --no-restore --configuration Release --warnaserror + - run: scripts/install-spacetimedb.sh + - name: Build module working-directory: rust/spacetime-module + run: cargo build --locked --release --target wasm32-unknown-unknown + - name: Verify generated bindings + run: | + spacetime generate --lang csharp --yes \ + --bin-path rust/spacetime-module/target/wasm32-unknown-unknown/release/spacetime_module.wasm \ + --out-dir csharp/SpacetimeDBVSDoublets/ModuleBindings + git diff --exit-code -- csharp/SpacetimeDBVSDoublets/ModuleBindings + - run: scripts/start-spacetimedb.sh + - name: All C# tests, including backend equivalence + run: dotnet run --project csharp/SpacetimeDBVSDoublets.Tests --configuration Release --no-build -- -parallelMode none + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: csharp-test-logs + path: ci-logs/ - - name: Run tests - env: - SPACETIMEDB_URI: http://localhost:3000 - SPACETIMEDB_DB: benchmark-links - # Run tests sequentially to avoid parallel interference with shared SpacetimeDB state. - run: cargo test -- --test-threads=1 - - # Unit tests for the reporting pipeline (rust/out.py). They are cheap and must - # pass before a benchmark is run, otherwise a 40 minute benchmark could finish - # only to fail while publishing its results. results-pipeline: name: Results pipeline tests runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.11' - - - name: Install Python dependencies - run: pip install matplotlib numpy - - - name: Run results pipeline tests + - run: pip install matplotlib numpy + - working-directory: rust run: python3 -m unittest test_out -v + - run: node scripts/check-file-size.mjs - # Quick benchmark validation for pull requests. - # Runs benchmarks with reduced scale to verify they work and produce results - # in well under 10 minutes. Results are not committed but uploaded as artifacts. - # - # Parameters chosen to keep total benchmark time under 5 minutes: - # BENCHMARK_LINK_COUNT=10, BACKGROUND_LINK_COUNT=30: reduces SpacetimeDB - # round trips per iteration from ~8000 to ~80, making each iteration ~0.1s. - # --sample-size 10: collect 10 samples per benchmark (instead of 100 default). - # --warm-up-time 1: 1s warm-up instead of 3s default. - # --measurement-time 2: 2s measurement instead of 5s default. - # Expected runtime: ~3-5 minutes total for all 35 benchmarks. - benchmark-pr: - name: Benchmark (PR validation) + benchmark: + name: Benchmark (${{ matrix.language }}, ${{ matrix.backend }}) runs-on: ubuntu-latest - needs: [test, results-pipeline] - if: github.event_name == 'pull_request' - timeout-minutes: 20 + needs: [test-rust, test-csharp, results-pipeline] + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + language: [rust, csharp] + backend: [spacetimedb, doublets] + env: + BENCHMARK_LINK_COUNT: ${{ github.event_name == 'pull_request' && '10' || '1000' }} + BACKGROUND_LINK_COUNT: ${{ github.event_name == 'pull_request' && '30' || '3000' }} + BENCHMARK_BACKEND: ${{ matrix.backend }} steps: - uses: actions/checkout@v4 - - - name: Setup Rust (pinned nightly) - uses: dtolnay/rust-toolchain@master + - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ env.toolchain }} - # rust/rust-toolchain.toml requires these components. Installing them - # with the toolchain avoids rustup adding them lazily on the first - # cargo invocation, which fails on the runner image with - # "detected conflict: 'bin/cargo-fmt'". components: rustfmt, clippy targets: wasm32-unknown-unknown - - - name: Setup Python - uses: actions/setup-python@v5 + - uses: Swatinem/rust-cache@v2 with: - python-version: '3.11' - - - name: Install Python dependencies - run: pip install matplotlib numpy - - - name: Cache cargo registry - uses: Swatinem/rust-cache@v2 + workspaces: | + rust -> target + rust/spacetime-module -> target + - uses: actions/setup-dotnet@v4 + if: matrix.language == 'csharp' with: - workspaces: rust -> target - cache-on-failure: "true" - - - name: Install SpacetimeDB CLI - run: | - curl -sSf https://install.spacetimedb.com | sh -s -- -y - echo "$HOME/.local/bin" >> $GITHUB_PATH - working-directory: . - - - name: Build SpacetimeDB module (WASM) - run: cargo build --release --target wasm32-unknown-unknown - working-directory: rust/spacetime-module - - - name: Start SpacetimeDB server - run: | - spacetime start & - for i in $(seq 1 30); do - if curl -sf http://localhost:3000/ > /dev/null 2>&1; then - echo "SpacetimeDB server is ready" - break - fi - sleep 1 - done - working-directory: . - - - name: Publish SpacetimeDB module - run: | - spacetime publish \ - --server http://localhost:3000 \ - --bin-path target/wasm32-unknown-unknown/release/spacetime_module.wasm \ - --yes \ - benchmark-links + dotnet-version: '10.0.x' + - name: Prepare output + run: mkdir -p results + - if: matrix.backend == 'spacetimedb' + run: scripts/install-spacetimedb.sh + - name: Build module + if: matrix.backend == 'spacetimedb' working-directory: rust/spacetime-module - - - name: Build benchmark - run: cargo build --release - - - name: Run benchmark (quick mode for PR validation) - env: - # Reduced scale: 10 links instead of 1000, 30 background instead of 3000. - # This reduces SpacetimeDB round trips per iteration from ~8000 to ~80, - # keeping each iteration under 0.1s and total benchmark time under 5 minutes. - BENCHMARK_LINK_COUNT: 10 - BACKGROUND_LINK_COUNT: 30 - SPACETIMEDB_URI: http://localhost:3000 - SPACETIMEDB_DB: benchmark-links + run: cargo build --locked --release --target wasm32-unknown-unknown + - if: matrix.backend == 'spacetimedb' + run: scripts/start-spacetimedb.sh + - name: Record the measuring VM's provenance + run: python3 scripts/benchmark-metadata.py ${{ matrix.language }} ${{ matrix.backend }} results/${{ matrix.language }}-${{ matrix.backend }}.json + - name: Rust benchmark + if: matrix.language == 'rust' + working-directory: rust + shell: bash run: | set -o pipefail - # Criterion compares against target/criterion///base and prints - # the failure to *stdout* when that directory was restored incompletely by - # the cache, which splices "Criterion.rs ERROR: ..." into the middle of the - # bencher records (see CI run 35028280108). Start from a clean state. rm -rf target/criterion - cargo bench --bench bench -- \ - --output-format bencher \ - --sample-size 10 \ - --warm-up-time 1 \ - --measurement-time 2 \ - --nresamples 1000 \ - | tee out.txt - - - name: Generate results table and charts - env: - BENCHMARK_LINK_COUNT: 10 - BACKGROUND_LINK_COUNT: 30 - run: python3 out.py out.txt --results results.md - - - name: Publish results to the job summary + options=(--sample-size 20 --nresamples 10000) + if [[ '${{ github.event_name }}' == pull_request ]]; then + options=(--sample-size 10 --warm-up-time 1 --measurement-time 2 --nresamples 1000) + fi + cargo bench --locked --bench bench -- --output-format bencher "${options[@]}" \ + > ../results/rust-${{ matrix.backend }}.txt 2> ../results/rust-${{ matrix.backend }}.log + - name: C# benchmark + if: matrix.language == 'csharp' + shell: bash run: | - { - echo "## Benchmark results (PR validation, reduced scale)" - echo - cat results.md - echo - echo "_Reduced scale: these numbers only prove the benchmark runs;" - echo "the published results come from the full run on \`main\`._" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Upload PR benchmark artifacts - # Keep the raw output even when the results step fails, - # so a broken run can be diagnosed from the artifact. + dotnet restore csharp/SpacetimeDBVSDoublets/SpacetimeDBVSDoublets.csproj --locked-mode + dotnet build csharp/SpacetimeDBVSDoublets/SpacetimeDBVSDoublets.csproj --no-restore -c Release --warnaserror + options=() + if [[ '${{ github.event_name }}' == pull_request ]]; then options=(--quick); fi + dotnet run --project csharp/SpacetimeDBVSDoublets --no-build -c Release -- \ + --backend=${{ matrix.backend }} "${options[@]}" \ + > results/csharp-${{ matrix.backend }}.txt 2> results/csharp-${{ matrix.backend }}.log + - uses: actions/upload-artifact@v4 if: always() - uses: actions/upload-artifact@v4 with: - name: benchmark-results-pr - path: | - rust/out.txt - rust/results.md - rust/bench_rust.png - rust/bench_rust_log_scale.png + name: results-${{ matrix.language }}-${{ matrix.backend }} + path: results/ - # Full benchmark run for commits to main/master. - # Uses full scale (1000 links, 3000 background) with reduced sample count - # to produce statistically meaningful results while fitting within 1 hour. - # - # Parameters: - # BENCHMARK_LINK_COUNT=1000, BACKGROUND_LINK_COUNT=3000: realistic scale. - # --sample-size 20: 20 samples per benchmark (down from 100 default). - # --nresamples 10000: 10k bootstrap resamples (down from 100k default). - # Expected runtime: ~30-45 minutes total for all 35 benchmarks. - benchmark: - name: Benchmark (full) + report: + name: Publish benchmark report runs-on: ubuntu-latest - needs: [test, results-pipeline] - if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') - timeout-minutes: 180 - # Required to push the regenerated README table and docs/benchmarks/ charts back. + needs: benchmark + timeout-minutes: 10 permissions: contents: write steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup Rust (pinned nightly) - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ env.toolchain }} - # rust/rust-toolchain.toml requires these components. Installing them - # with the toolchain avoids rustup adding them lazily on the first - # cargo invocation, which fails on the runner image with - # "detected conflict: 'bin/cargo-fmt'". - components: rustfmt, clippy - targets: wasm32-unknown-unknown - - - name: Setup Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.11' - - - name: Install Python dependencies - run: pip install matplotlib numpy - - - name: Cache cargo registry - uses: Swatinem/rust-cache@v2 + - run: pip install matplotlib numpy + - uses: actions/download-artifact@v4 with: - workspaces: rust -> target - cache-on-failure: "true" - - - name: Install SpacetimeDB CLI - run: | - curl -sSf https://install.spacetimedb.com | sh -s -- -y - echo "$HOME/.local/bin" >> $GITHUB_PATH - working-directory: . - - - name: Build SpacetimeDB module (WASM) - run: cargo build --release --target wasm32-unknown-unknown - working-directory: rust/spacetime-module - - - name: Start SpacetimeDB server + pattern: results-* + path: collected + merge-multiple: true + - name: Combine both backends and render both languages + shell: bash run: | - spacetime start & - for i in $(seq 1 30); do - if curl -sf http://localhost:3000/ > /dev/null 2>&1; then - echo "SpacetimeDB server is ready" - break - fi - sleep 1 + for language in rust csharp; do + cat collected/$language-spacetimedb.txt collected/$language-doublets.txt > $language/out.txt + python3 rust/out.py $language/out.txt --language "$language" \ + --metadata collected/$language-spacetimedb.json collected/$language-doublets.json \ + --results "$language/results.md" --readme README.md --output-dir docs/benchmarks + cat "$language/results.md" >> "$GITHUB_STEP_SUMMARY" done - working-directory: . - - - name: Publish SpacetimeDB module - run: | - spacetime publish \ - --server http://localhost:3000 \ - --bin-path target/wasm32-unknown-unknown/release/spacetime_module.wasm \ - --yes \ - benchmark-links - working-directory: rust/spacetime-module - - - name: Build benchmark - run: cargo build --release - - - name: Run benchmark (full mode for main branch) - env: - # Full scale: 1000 links, 3000 background for realistic results. - # --sample-size 20 reduces total runtime from ~2h (default 100) to ~25-40 min - # while still providing statistically valid measurements. - BENCHMARK_LINK_COUNT: 1000 - BACKGROUND_LINK_COUNT: 3000 - SPACETIMEDB_URI: http://localhost:3000 - SPACETIMEDB_DB: benchmark-links + - name: Commit full results on the default branch + if: github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch run: | - set -o pipefail - # Criterion compares against target/criterion///base and prints - # the failure to *stdout* when that directory was restored incompletely by - # the cache, which splices "Criterion.rs ERROR: ..." into the middle of the - # bencher records (see CI run 35028280108). Start from a clean state. - rm -rf target/criterion - cargo bench --bench bench -- \ - --output-format bencher \ - --sample-size 20 \ - --nresamples 10000 \ - | tee out.txt - - # Regenerates results.md, both charts, copies the charts into docs/benchmarks/ and - # replaces the results section of README.md, so the numbers are readable - # in the repository without running the benchmark locally. - - name: Generate results table and charts - env: - BENCHMARK_LINK_COUNT: 1000 - BACKGROUND_LINK_COUNT: 3000 - run: | - python3 out.py out.txt \ - --results results.md \ - --readme ../README.md \ - --docs-dir ../docs/benchmarks - - - name: Publish results to the job summary - run: | - { - echo "## Benchmark results (full scale)" - echo - cat results.md - } >> "$GITHUB_STEP_SUMMARY" - - - name: Configure git - run: | - git config user.email "linksplatform@gmail.com" - git config user.name "LinksPlatformBencher" - - - name: Commit benchmark results - working-directory: . - run: | - # The charts live in docs/benchmarks/ only; the copies in rust/ are build - # output and are kept as workflow artifacts instead of being committed twice. - git add docs/benchmarks README.md rust/results.md rust/out.txt - if git diff --staged --quiet; then - echo "No changes to commit" - else + git config user.email linksplatform@gmail.com + git config user.name LinksPlatformBencher + git add README.md docs/benchmarks rust/out.txt rust/results.md csharp/out.txt csharp/results.md + if ! git diff --staged --quiet; then git commit -m "Update benchmark results [skip ci]" git push origin HEAD:${GITHUB_REF_NAME} fi - - - name: Upload benchmark artifacts - # Keep the raw output even when the results step fails, - # so a broken run can be diagnosed from the artifact. + - uses: actions/upload-artifact@v4 if: always() - uses: actions/upload-artifact@v4 with: - name: benchmark-results + name: benchmark-report path: | - rust/out.txt + README.md + docs/benchmarks/ rust/results.md - rust/bench_rust.png - rust/bench_rust_log_scale.png + csharp/results.md + collected/ diff --git a/.gitignore b/.gitignore index 5289aa5..5ea4ae2 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,12 @@ doc/ # Log files *.log logs/ +ci-logs/ + +# .NET build output and local benchmark artifacts +**/bin/ +**/obj/ +results/ +collected/ +.spacetime/ +spacetime-data/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ce4da86..4556803 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,21 +14,21 @@ repos: hooks: - id: cargo-fmt name: cargo fmt - entry: cargo fmt --all -- + entry: bash -c 'cd rust && cargo fmt --all --' language: system types: [rust] pass_filenames: false - id: cargo-clippy name: cargo clippy - entry: cargo clippy --all-targets --all-features -- -D warnings + entry: bash -c 'cd rust && cargo clippy --locked --all-targets -- -D warnings' language: system types: [rust] pass_filenames: false - id: cargo-test name: cargo test - entry: cargo test + entry: bash -c 'cd rust && cargo test --locked -- --test-threads=1' language: system types: [rust] pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5823091..eb47e4c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,299 +1,45 @@ -# Contributing to rust-ai-driven-development-pipeline-template +# Contributing -Thank you for your interest in contributing! This document provides guidelines and instructions for contributing to this project. +Work in a pull request branch. The repository contains benchmark applications in +`rust/` and `csharp/`; run Cargo commands from `rust/` to select the pinned nightly. +Install .NET 10 and Python 3.11 or later, then install `matplotlib` and `numpy`. -## Development Setup - -1. **Fork and clone the repository** - - ```bash - git clone https://github.com/YOUR-USERNAME/rust-ai-driven-development-pipeline-template.git - cd rust-ai-driven-development-pipeline-template - ``` - -2. **Install Rust** - - Install Rust using rustup (if not already installed): - - ```bash - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - ``` - -3. **Install development tools** - - ```bash - rustup component add rustfmt clippy - ``` - -4. **Install pre-commit hooks** (optional but recommended) - - ```bash - pip install pre-commit - pre-commit install - ``` - -5. **Build the project** - - ```bash - cargo build - ``` - -## Development Workflow - -1. **Create a feature branch** - - ```bash - git checkout -b feature/my-feature - ``` - -2. **Make your changes** - - - Write code following the project's style guidelines - - Add tests for any new functionality - - Update documentation as needed - -3. **Run quality checks** - - ```bash - # Format code - cargo fmt - - # Run Clippy lints - cargo clippy --all-targets --all-features - - # Check file sizes - node scripts/check-file-size.mjs - - # Run all checks together - cargo fmt --check && cargo clippy --all-targets --all-features && node scripts/check-file-size.mjs - ``` - -4. **Run tests** - - ```bash - # Run all tests - cargo test - - # Run tests with verbose output - cargo test --verbose - - # Run doc tests - cargo test --doc - - # Run a specific test - cargo test test_name - ``` - -5. **Add a changelog fragment** - - For any user-facing changes, create a changelog fragment: - - ```bash - # Create a new file in changelog.d/ - # Format: YYYYMMDD_HHMMSS_description.md - touch changelog.d/$(date +%Y%m%d_%H%M%S)_my_change.md - ``` - - Edit the file to document your changes: - - ```markdown - ### Added - - Description of new feature - - ### Fixed - - Description of bug fix - ``` - - **Why fragments?** This prevents merge conflicts in CHANGELOG.md when multiple PRs are open simultaneously. - -6. **Commit your changes** - - ```bash - git add . - git commit -m "feat: add new feature" - ``` - - Pre-commit hooks will automatically run and check your code. - -7. **Push and create a Pull Request** - - ```bash - git push origin feature/my-feature - ``` - - Then create a Pull Request on GitHub. - -## Code Style Guidelines - -This project uses: - -- **rustfmt** for code formatting -- **Clippy** for linting with pedantic and nursery lints enabled -- **cargo test** for testing - -### Code Standards - -- Follow Rust idioms and best practices -- Use documentation comments (`///`) for all public APIs -- Write tests for all new functionality -- Keep functions focused and reasonably sized -- Keep files under 1000 lines -- Use meaningful variable and function names - -### Documentation Format - -Use Rust documentation comments: - -```rust -/// Brief description of the function. -/// -/// Longer description if needed. -/// -/// # Arguments -/// -/// * `arg1` - Description of arg1 -/// * `arg2` - Description of arg2 -/// -/// # Returns -/// -/// Description of return value -/// -/// # Errors -/// -/// Description of when errors are returned -/// -/// # Examples -/// -/// ``` -/// use my_package::example_function; -/// let result = example_function(1, 2); -/// assert_eq!(result, 3); -/// ``` -pub fn example_function(arg1: i32, arg2: i32) -> i32 { - arg1 + arg2 -} -``` - -## Testing Guidelines - -- Write tests for all new features -- Maintain or improve test coverage -- Use descriptive test names -- Organize tests in modules when appropriate -- Use `#[cfg(test)]` for test-only code - -Example test structure: - -```rust -#[cfg(test)] -mod tests { - use super::*; - - mod my_feature_tests { - use super::*; - - #[test] - fn test_basic_functionality() { - assert_eq!(my_function(), expected_result); - } - - #[test] - fn test_edge_case() { - assert_eq!(my_function(edge_case_input), expected_result); - } - } -} -``` - -## Pull Request Process - -1. Ensure all tests pass locally -2. Update documentation if needed -3. Add a changelog fragment (see step 5 in Development Workflow) -4. Ensure the PR description clearly describes the changes -5. Link any related issues in the PR description -6. Wait for CI checks to pass -7. Address any review feedback - -## Changelog Management - -This project uses a fragment-based changelog system similar to [Scriv](https://scriv.readthedocs.io/) (Python) and [Changesets](https://github.com/changesets/changesets) (JavaScript). - -### Creating a Fragment +Before committing: ```bash -# Create a new fragment with timestamp -touch changelog.d/$(date +%Y%m%d_%H%M%S)_description.md +(cd rust && cargo fmt --all -- --check) +(cd rust && cargo clippy --locked --all-targets -- -D warnings) +(cd rust && python3 -m unittest test_out -v) +node scripts/check-file-size.mjs +dotnet restore csharp/SpacetimeDBVSDoublets.slnx --locked-mode +dotnet format csharp/SpacetimeDBVSDoublets.slnx --no-restore --verify-no-changes +dotnet build csharp/SpacetimeDBVSDoublets.slnx --no-restore -c Release --warnaserror ``` -### Fragment Categories - -Use these categories in your fragments: - -- **Added**: New features -- **Changed**: Changes to existing functionality -- **Deprecated**: Features that will be removed in future -- **Removed**: Features that were removed -- **Fixed**: Bug fixes -- **Security**: Security-related changes - -### During Release - -Fragments are automatically collected into CHANGELOG.md during the release process. The release workflow: - -1. Collects all fragments -2. Updates CHANGELOG.md with the new version entry -3. Removes processed fragment files -4. Bumps the version in Cargo.toml -5. Creates a git tag and GitHub release +For integration checks, install the pinned CLI using `scripts/install-spacetimedb.sh`, +add its directory to `PATH`, then build and publish the shared Rust module: -## Project Structure - -``` -. -├── .github/workflows/ # GitHub Actions CI/CD -├── changelog.d/ # Changelog fragments -│ ├── README.md # Fragment instructions -│ └── *.md # Individual changelog fragments -├── examples/ # Usage examples -├── scripts/ # Utility scripts -├── src/ -│ ├── lib.rs # Library entry point -│ └── main.rs # Binary entry point -├── tests/ # Integration tests -├── .gitignore # Git ignore patterns -├── .pre-commit-config.yaml # Pre-commit hooks -├── Cargo.toml # Project configuration -├── CHANGELOG.md # Project changelog -├── CONTRIBUTING.md # This file -├── LICENSE # Unlicense (public domain) -└── README.md # Project README +```bash +(cd rust/spacetime-module && cargo build --locked --release --target wasm32-unknown-unknown) +scripts/start-spacetimedb.sh +(cd rust && cargo test --locked -- --test-threads=1 --include-ignored) +SPACETIMEDB_URI=http://localhost:3000 \ + dotnet run --project csharp/SpacetimeDBVSDoublets.Tests -c Release -- -parallelMode none ``` -## Release Process - -This project uses semantic versioning (MAJOR.MINOR.PATCH): - -- **MAJOR**: Breaking changes -- **MINOR**: New features (backward compatible) -- **PATCH**: Bug fixes (backward compatible) - -Releases are managed through GitHub releases. To trigger a release: - -1. Manually trigger the release workflow with a version bump type -2. Or: Update the version in Cargo.toml and push to main - -## Getting Help - -- Open an issue for bugs or feature requests -- Use discussions for questions and general help -- Check existing issues and PRs before creating new ones +Run tests sequentially because they reset the shared database. Without a server, +run ordinary `cargo test` and the C# tests without `SPACETIMEDB_URI`; those cover +Doublets and sampling. CI always includes live server tests. -## Code of Conduct +Keep operations, setup, reset and dataset sizes equivalent across languages. Add +regression tests before bug fixes and exercise all four Doublets variants. Generated +C# bindings come from SpacetimeDB 2.10.1 and the module WASM; regenerate them with the +command in the README. Keep code formatting consistent and public APIs documented. -- Be respectful and inclusive -- Provide constructive feedback -- Focus on what is best for the community -- Show empathy towards other community members +Add a fragment to `changelog.d/` for user-facing changes. There is no package release +pipeline: this repository publishes benchmark reports and charts through CI. -Thank you for contributing! +PR validation uses 10 measured links and 30 background links. Full default-branch +runs use 1000 and 3000. Each language/backend is measured on a separate VM. Preserve +raw output and metadata alongside any reported results, and describe validation in +the pull request. diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 82c7d64..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,101 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "futures-core" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" - -[[package]] -name = "my-package" -version = "0.1.0" -dependencies = [ - "tokio", - "tokio-test", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "syn" -version = "3.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tokio" -version = "1.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" -dependencies = [ - "pin-project-lite", - "tokio-macros", -] - -[[package]] -name = "tokio-macros" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-stream" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-test" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" -dependencies = [ - "futures-core", - "tokio", - "tokio-stream", -] - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index 2da51af..0000000 --- a/Cargo.toml +++ /dev/null @@ -1,45 +0,0 @@ -[package] -name = "my-package" -version = "0.1.0" -edition = "2021" -description = "A Rust package template for AI-driven development" -readme = "README.md" -license = "Unlicense" -keywords = ["template", "rust", "ai-driven"] -categories = ["development-tools"] -repository = "https://github.com/link-foundation/rust-ai-driven-development-pipeline-template" -documentation = "https://github.com/link-foundation/rust-ai-driven-development-pipeline-template" -rust-version = "1.70" - -[lib] -name = "my_package" -path = "src/lib.rs" - -[[bin]] -name = "my-package" -path = "src/main.rs" - -[dependencies] -tokio = { version = "1.0", features = ["rt-multi-thread", "macros", "time"] } - -[dev-dependencies] -tokio-test = "0.4" - -[lints.rust] -unsafe_code = "forbid" - -[lints.clippy] -all = { level = "warn", priority = -1 } -pedantic = { level = "warn", priority = -1 } -nursery = { level = "warn", priority = -1 } - -# Allow some common patterns -module_name_repetitions = "allow" -too_many_lines = "allow" -missing_errors_doc = "allow" -missing_panics_doc = "allow" - -[profile.release] -lto = true -codegen-units = 1 -strip = true diff --git a/README.md b/README.md index 170bbba..11d2819 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Benchmark comparing [SpacetimeDB 2](https://github.com/clockworklabs/SpacetimeDB) vs [Doublets](https://github.com/linksplatform/doublets-rs) performance for basic CRUD operations with links. -SpacetimeDB is benchmarked using the official `spacetimedb-sdk` Rust crate connected to a running SpacetimeDB 2.0 server. Doublets is benchmarked with both in-memory (volatile) and file-backed (non-volatile) storage variants. +SpacetimeDB is benchmarked using the official Rust and C# SDKs connected to a running SpacetimeDB 2.10.1 server. Doublets is benchmarked with both in-memory (volatile) and file-backed (non-volatile) storage variants. ## Benchmark Operations @@ -19,9 +19,9 @@ SpacetimeDB is benchmarked using the official `spacetimedb-sdk` Rust crate conne ## Backends Benchmarked ### SpacetimeDB -- **SpacetimeDB** — connects to a running SpacetimeDB 2.0 server via the official `spacetimedb-sdk` Rust crate; uses the `links` table defined in the `spacetime-module` WebAssembly module +- **SpacetimeDB** — connects to a running SpacetimeDB 2.10.1 server via its official Rust or C# client SDK; uses the `links` table defined in the `spacetime-module` WebAssembly module -The benchmark uses the official SpacetimeDB Rust client SDK, calling reducers to mutate data and reading from the client-side subscription cache. +The benchmark uses the official SpacetimeDB client SDKs, calling reducers to mutate data and reading from the client-side subscription cache. ### Doublets - **Doublets United Volatile** — in-memory store; links stored as contiguous `(index, source, target)` units @@ -40,7 +40,11 @@ Each benchmark iteration pre-populates the database with background links to sim ## Results -The numbers below represent the amount of time (ns) a single benchmark iteration takes. +The numbers below represent the time a single benchmark iteration takes, with +three significant digits and human-readable units. Doublets cells compare to +SpacetimeDB: `N× faster`, `N× slower`, or `≈ same` when medians differ by less than +5% of the SpacetimeDB median or their ±1 standard deviation ranges overlap. +These ranges describe measurement variability, not a significance test. - The first chart shows time in a pixel (linear) scale. Doublets bars are drawn with a minimum visible width, otherwise they would not be visible next to SpacetimeDB. @@ -48,7 +52,7 @@ The numbers below represent the amount of time (ns) a single benchmark iteration because it is around 3-5 orders of magnitude. Charts and the table are recalculated by the -[Rust Benchmark workflow](.github/workflows/rust-benchmark.yml) on every push to `main` +[Benchmark workflow](.github/workflows/rust-benchmark.yml) on every push to `main` and committed back to this repository, so the results are visible here without running the benchmark locally. @@ -57,24 +61,50 @@ the benchmark locally. ![Image of Rust benchmark (pixel scale)](https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/blob/main/docs/benchmarks/bench_rust.png?raw=true) ![Image of Rust benchmark (log scale)](https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/blob/main/docs/benchmarks/bench_rust_log_scale.png?raw=true) -### Raw benchmark results (all numbers are in nanoseconds) +### Rust results -_Generated 2026-09-16 09:20 UTC by [GitHub Actions run 35075414684](https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/actions/runs/35075414684) — 1000 benchmarked links, 3000 background links._ +The committed Rust measurements below are historical (2026-09-16), before the +query dataset and optimizer fixes in issue #16. Their CPU was not recorded. +A full run on the default branch replaces them with current measurements. + +_Generated 2026-09-16 09:20 UTC by [GitHub Actions run 35075414684](https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/actions/runs/35075414684) — 1000 benchmarked links, 3000 background links. SpacetimeDB server/CLI unknown (not recorded); Rust SDK 2.10.1 (lockfile); Doublets (patched) 0.1.0-pre+beta.15; CPU — historical run: unknown (not recorded)._ | Operation | Doublets United Volatile | Doublets United NonVolatile | Doublets Split Volatile | Doublets Split NonVolatile | SpacetimeDB | |-----------------|--------------------------|-----------------------------|-------------------------|----------------------------|---------------| -| Create | 76689 (33385.0x faster) | 76567 (33438.2x faster) | 47102 (54355.7x faster) | 47395 (54019.7x faster) | 2560262853 | -| Update | 247101 (10703.3x faster) | 247021 (10706.8x faster) | 35819 (73838.1x faster) | 36029 (73407.7x faster) | 2644805261 | -| Delete | 180271 (7169.4x faster) | 181398 (7124.8x faster) | 94116 (13732.3x faster) | 100143 (12905.9x faster) | 1292432593 | -| Query All | 21580 (1.0x faster) | 21592 (1.0x faster) | 28388 (1.3x slower) | 28312 (1.3x slower) | 22126 | -| Query by Id | 55 (198863.3x faster) | 59 (185381.0x faster) | 1018 (10744.1x faster) | 1018 (10744.1x faster) | 10937480 | -| Query by Source | 1476 (110.8x faster) | 1453 (112.5x faster) | 480 (340.7x faster) | 478 (342.1x faster) | 163528 | -| Query by Target | 1514 (109.3x faster) | 1478 (111.9x faster) | 445 (371.8x faster) | 393 (420.9x faster) | 165432 | +| Create | 76.7 µs (33,385× faster) | 76.6 µs (33,438× faster) | 47.1 µs (54,356× faster) | 47.4 µs (54,020× faster) | 2.56 s | +| Update | 247 µs (10,703× faster) | 247 µs (10,707× faster) | 35.8 µs (73,838× faster) | 36.0 µs (73,408× faster) | 2.64 s | +| Delete | 180 µs (7,169× faster) | 181 µs (7,125× faster) | 94.1 µs (13,732× faster) | 100 µs (12,906× faster) | 1.29 s | +| Query All | 21.6 µs (≈ same) | 21.6 µs (≈ same) | 28.4 µs (1.28× slower) | 28.3 µs (1.28× slower) | 22.1 µs | +| Query by Id | 55.0 ns (198,863× faster) | 59.0 ns (185,381× faster) | 1.02 µs (10,744× faster) | 1.02 µs (10,744× faster) | 10.9 ms | +| Query by Source | 1.48 µs (111× faster) | 1.45 µs (113× faster) | 480 ns (341× faster) | 478 ns (342× faster) | 164 µs | +| Query by Target | 1.51 µs (109× faster) | 1.48 µs (112× faster) | 445 ns (372× faster) | 393 ns (421× faster) | 165 µs | -Each Doublets cell is annotated with how many times faster (or slower) it is than -SpacetimeDB for the same operation. +### C# + +The .NET 10 benchmark uses `SpacetimeDB.ClientSDK` 2.10.1 and +`Platform.Data.Doublets` 0.18.1. It runs all seven operations and the same four +Doublets layouts, dataset sizes, and setup/reset steps as Rust. Both languages +wait for reducer completion on writes and materialize subscription-cache query +results. Rust processes SDK messages on its background thread; C# pumps +`FrameTick` on the calling thread, following the +[official C# SDK guidance](https://spacetimedb.com/docs/clients/c-sharp/). + +C# sampling follows Criterion's automatic linear/flat schedule (PR: 10 samples, +1 s warm-up, 2 s measurement; full: 20 samples, 3 s warm-up, 5 s measurement). +Calibration includes setup/reset wall time; reported samples include only the +operation. This avoids excessive query sampling when setup involves network calls. + + +_No C# measurements published yet. CI validates the reduced workload on this PR; +the first full default-branch run publishes this table and both C# charts._ + + +Each language/backend runs on a separate VM. The line above each generated table +records run, time, sizes, versions and the CPU of both backend VMs. Raw data and +metadata are retained as workflow artifacts. PR reports use reduced sizes and are +uploaded for review; only full default-branch results are committed. ## Conclusion @@ -111,98 +141,88 @@ The algorithmic complexity is the same for volatile and non-volatile Doublets va ### Prerequisites - Rust nightly, pinned in `rust/rust-toolchain.toml` (`rustup` installs it automatically) -- SpacetimeDB CLI: `curl -sSf https://install.spacetimedb.com | sh` +- .NET 10 SDK for C# +- Python 3.11+ with `matplotlib` and `numpy` for reports +- SpacetimeDB CLI: `scripts/install-spacetimedb.sh` (pinned to 2.10.1), then add + `$HOME/.local/bin` to `PATH` ### Start SpacetimeDB server and publish module ```bash -# Start the local SpacetimeDB server -spacetime start & +(cd rust/spacetime-module && cargo build --locked --release --target wasm32-unknown-unknown) +scripts/start-spacetimedb.sh -# Build and publish the links module -spacetime build --project-path rust/spacetime-module -spacetime publish --project-path rust/spacetime-module --yes benchmark-links +# Regenerate the committed C# client bindings after module changes: +spacetime generate --lang csharp --yes \ + --bin-path rust/spacetime-module/target/wasm32-unknown-unknown/release/spacetime_module.wasm \ + --out-dir csharp/SpacetimeDBVSDoublets/ModuleBindings ``` ### Run benchmarks ```bash -cd rust - -# Full benchmark run (1000 links, 3000 background) -SPACETIMEDB_URI=http://localhost:3000 SPACETIMEDB_DB=benchmark-links \ - cargo bench --bench bench -- --output-format bencher | tee out.txt +# Full Rust run: N=1000, B=3000; choose one backend at a time. +(cd rust && BENCHMARK_BACKEND=spacetimedb cargo bench --locked --bench bench -- \ + --output-format bencher --sample-size 20 --nresamples 10000 > ../results-rust-spacetimedb.txt) +(cd rust && BENCHMARK_BACKEND=doublets cargo bench --locked --bench bench -- \ + --output-format bencher --sample-size 20 --nresamples 10000 > ../results-rust-doublets.txt) + +# Full C# run, same defaults and sampling: +dotnet run --project csharp/SpacetimeDBVSDoublets -c Release -- --backend=spacetimedb > results-csharp-spacetimedb.txt +dotnet run --project csharp/SpacetimeDBVSDoublets -c Release -- --backend=doublets > results-csharp-doublets.txt + +# Quick checks (N=10, B=30): +(cd rust && BENCHMARK_BACKEND=doublets BENCHMARK_LINK_COUNT=10 BACKGROUND_LINK_COUNT=30 \ + cargo bench --locked --bench bench -- --sample-size 10 --warm-up-time 1 --measurement-time 2) +dotnet run --project csharp/SpacetimeDBVSDoublets -c Release -- --backend=doublets --quick + +# Record metadata on each backend's machine, with the same size variables: +python3 scripts/benchmark-metadata.py rust spacetimedb results-rust-spacetimedb.json +python3 scripts/benchmark-metadata.py rust doublets results-rust-doublets.json +python3 rust/out.py results-rust-spacetimedb.txt results-rust-doublets.txt \ + --metadata results-rust-spacetimedb.json results-rust-doublets.json \ + --results rust/results.md --readme README.md --output-dir docs/benchmarks +# Use --language csharp, C# input/metadata files and --results csharp/results.md for C#. +``` -# Quick benchmark run (CI scale) -BENCHMARK_LINK_COUNT=10 BACKGROUND_LINK_COUNT=100 \ -SPACETIMEDB_URI=http://localhost:3000 SPACETIMEDB_DB=benchmark-links \ - cargo bench --bench bench +`BENCHMARK_LINK_COUNT` and `BACKGROUND_LINK_COUNT` configure either language. +`SPACETIMEDB_URI` and `SPACETIMEDB_DB` select the server and published database. +Doublets-only runs do not require a server. Run each backend separately to avoid +competing for CPU during measurement. -# Generate the results table and charts from out.txt -python3 out.py out.txt --results results.md +Query by source/target uses unique pairs referring to actual background IDs, +rather than duplicate pairs or hardcoded IDs that differ after SpacetimeDB resets. +Those operations require at least `10 + ceil(N/10)` background links (both default +and PR sizes satisfy this). Setup, dataset generation and reset are outside the +measured interval. Query results are consumed so the optimizer preserves the work. -# Regenerate everything the CI publishes: results.md, docs/benchmarks/ charts -# and the results section of README.md -python3 out.py out.txt --results results.md --readme ../README.md \ - --docs-dir ../docs/benchmarks -``` +### Tests and code quality -### Run tests +See [CONTRIBUTING.md](CONTRIBUTING.md) for all local checks. In particular: ```bash -cd rust -SPACETIMEDB_URI=http://localhost:3000 SPACETIMEDB_DB=benchmark-links cargo test +(cd rust && cargo test --locked -- --test-threads=1 --include-ignored) +SPACETIMEDB_URI=http://localhost:3000 \ + dotnet run --project csharp/SpacetimeDBVSDoublets.Tests -c Release -- -parallelMode none +(cd rust && cargo fmt --all -- --check && cargo clippy --locked --all-targets -- -D warnings) +(cd rust && python3 -m unittest test_out -v) ``` -### Code quality - -```bash -cd rust -cargo fmt --all -cargo clippy --all-targets - -# Unit tests for the results reporting pipeline (no benchmark run required) -python3 -m unittest test_out -v -``` +The same-behavior tests compare all backends' create, update, delete, counts and +queries, including repeated source/target datasets. They normalize IDs by creation +order because SpacetimeDB preserves its sequence and Doublets reuses IDs. CI runs +the live server tests explicitly; they cannot silently skip an unavailable server. ## Project Structure -``` -. -├── docs/ -│ └── benchmarks/ # Benchmark charts published by CI and shown above -│ ├── bench_rust.png -│ └── bench_rust_log_scale.png -├── rust/ -│ ├── spacetime-module/ # SpacetimeDB WASM module (links table + reducers) -│ │ ├── Cargo.toml -│ │ └── src/ -│ │ └── lib.rs # Table definition and reducers using `spacetimedb` crate -│ ├── Cargo.toml # Package manifest with pinned dependencies -│ ├── doublets-patched/ # Local patches to doublets-rs for modern nightly compatibility -│ │ └── PATCHES.md # Documents why patches are needed and what was changed -│ ├── rust-toolchain.toml # Pinned Rust nightly toolchain -│ ├── rustfmt.toml # Rust formatting config -│ ├── out.py # Results table, charts and README update -│ ├── test_out.py # Unit tests for out.py -│ ├── results.md # Generated results table (committed by CI) -│ ├── src/ -│ │ ├── lib.rs # Links trait, constants (BENCHMARK_LINK_COUNT, BACKGROUND_LINK_COUNT) -│ │ ├── module_bindings/ # spacetimedb-sdk client bindings for the links module -│ │ ├── spacetimedb_impl.rs # SpacetimeDB SDK client (implements Links) -│ │ ├── doublets_impl.rs # Doublets store adapters (implements Links) -│ │ ├── exclusive.rs # Exclusive wrapper for interior mutability -│ │ ├── fork.rs # Fork — benchmark iteration isolation -│ │ └── benched/ -│ │ ├── mod.rs # Benched trait (setup/fork/unfork lifecycle) -│ │ ├── spacetimedb_benched.rs # Benched impl for SpacetimeDB -│ │ └── doublets_benched.rs # Benched impls for Doublets stores -│ └── benches/ -│ └── bench.rs # Criterion benchmark suite (7 operations x 5 backends) -└── .github/ - └── workflows/ - └── rust-benchmark.yml # CI: test on Linux/macOS, benchmark, publish results -``` +- `rust/`: Rust benchmark, adapters, tests, pinned toolchain and patched Doublets. +- `rust/spacetime-module/`: shared server-side WASM module. +- `csharp/SpacetimeDBVSDoublets/`: C# benchmark and generated SDK bindings. +- `csharp/SpacetimeDBVSDoublets.Tests/`: differential, iteration and sampling tests. +- `rust/out.py`, `rust/test_out.py`: shared report generator and regression tests. +- `scripts/`: pinned server installation/startup, metadata and repository checks. +- `docs/benchmarks/`: committed full-run charts for both languages. +- `.github/workflows/rust-benchmark.yml`: tests, isolated backend benchmarks and reporting. ## License diff --git a/changelog.d/20261006_comparison_parity.md b/changelog.d/20261006_comparison_parity.md new file mode 100644 index 0000000..9315622 --- /dev/null +++ b/changelog.d/20261006_comparison_parity.md @@ -0,0 +1,14 @@ +### Added +- C# SpacetimeDB SDK and four Platform.Data.Doublets benchmarks matching the Rust workloads and sizes. +- Differential backend tests in both languages, and C# sampling and iteration reset tests. +- Per-backend, per-language benchmark VMs with recorded CPU, server/module and SDK/library versions. + +### Fixed +- Preserve Rust Doublets create endpoints and classify Split self-references before detaching index trees. +- Use unique query datasets with actual background IDs and consume query results to prevent optimization. +- Report differences within 5% or overlapping standard deviations as approximately the same. +- Format measured durations with three significant digits and human-readable units. +- Pin SpacetimeDB 2.10.1, fail CI on Clippy warnings, and cancel superseded workflow runs. + +### Removed +- Unrelated template crate, tests, example and crate release workflow. diff --git a/examples/basic_usage.rs b/examples/basic_usage.rs deleted file mode 100644 index bdbd1fc..0000000 --- a/examples/basic_usage.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Basic usage example for my-package. -//! -//! This example demonstrates the basic functionality of the package. -//! -//! Run with: `cargo run --example basic_usage` - -use my_package::{add, delay, multiply}; - -#[tokio::main] -async fn main() { - // Example 1: Basic arithmetic - println!("Example 1: Basic arithmetic"); - println!("2 + 3 = {}", add(2, 3)); - println!("2 * 3 = {}", multiply(2, 3)); - println!(); - - // Example 2: Working with larger numbers - println!("Example 2: Working with larger numbers"); - println!("1000 + 2000 = {}", add(1000, 2000)); - println!("100 * 200 = {}", multiply(100, 200)); - println!(); - - // Example 3: Working with negative numbers - println!("Example 3: Working with negative numbers"); - println!("-5 + 10 = {}", add(-5, 10)); - println!("-3 * 4 = {}", multiply(-3, 4)); - println!(); - - // Example 4: Async delay - println!("Example 4: Async delay"); - println!("Waiting for 1 second..."); - delay(1.0).await; - println!("Done!"); -} diff --git a/scripts/benchmark-metadata.py b/scripts/benchmark-metadata.py new file mode 100644 index 0000000..548343a --- /dev/null +++ b/scripts/benchmark-metadata.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Record provenance on the VM performing the measurement, before artifact upload.""" +import argparse +import json +import os +import platform +import re +import subprocess +import tomllib +from datetime import datetime, timezone +from pathlib import Path +from xml.etree import ElementTree + +ROOT = Path(__file__).resolve().parents[1] + + +def package_version(lock, name): + packages = tomllib.loads(lock.read_text())["package"] + return next(package["version"] for package in packages if package["name"] == name) + + +def collect(language, backend): + if language == "rust": + versions = {"Rust SDK": package_version(ROOT / "rust/Cargo.lock", "spacetimedb-sdk"), + "Doublets (patched)": package_version(ROOT / "rust/Cargo.lock", "doublets")} + else: + project = ElementTree.parse(ROOT / "csharp/SpacetimeDBVSDoublets/SpacetimeDBVSDoublets.csproj") + versions = {item.attrib["Include"]: item.attrib["Version"] for item in project.iter("PackageReference")} + versions["SpacetimeDB module"] = package_version(ROOT / "rust/spacetime-module/Cargo.lock", "spacetimedb") + # The configured pin is useful on Doublets VMs, which never start a server. + versions["SpacetimeDB server/CLI"] = (ROOT / "scripts/spacetimedb-version").read_text().strip() + if backend == "spacetimedb": + actual = subprocess.check_output(["spacetime", "--version"], text=True) + if f"tool version {versions['SpacetimeDB server/CLI']};" not in actual: + raise ValueError(f"Unexpected SpacetimeDB version: {actual}") + cpu = platform.processor() or "unknown" + if Path("/proc/cpuinfo").exists(): + match = re.search(r"model name\s*:\s*(.+)", Path("/proc/cpuinfo").read_text()) + if match: + cpu = match[1] + repository, run = os.environ.get("GITHUB_REPOSITORY"), os.environ.get("GITHUB_RUN_ID") + source = f"[GitHub Actions run {run}](https://github.com/{repository}/actions/runs/{run})" if repository and run else "a local benchmark run" + return {"date": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"), + "source": source, "benchmark_links": os.environ.get("BENCHMARK_LINK_COUNT", "1000"), + "background_links": os.environ.get("BACKGROUND_LINK_COUNT", "3000"), + "versions": versions, "cpus": {backend: cpu}} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("language", choices=["rust", "csharp"]) + parser.add_argument("backend", choices=["spacetimedb", "doublets"]) + parser.add_argument("output", type=Path) + args = parser.parse_args() + args.output.write_text(json.dumps(collect(args.language, args.backend), indent=2) + "\n") diff --git a/scripts/install-spacetimedb.sh b/scripts/install-spacetimedb.sh new file mode 100755 index 0000000..a2405fd --- /dev/null +++ b/scripts/install-spacetimedb.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Install the exact release; no mutable install.spacetimedb.com bootstrap. +set -euo pipefail +root=$(cd "$(dirname "$0")/.." && pwd) +version=$(cat "$root/scripts/spacetimedb-version") +case "$(uname -s)/$(uname -m)" in + Linux/x86_64) target=x86_64-unknown-linux-gnu ;; + Darwin/arm64) target=aarch64-apple-darwin ;; + Darwin/x86_64) target=x86_64-apple-darwin ;; + *) echo "Unsupported platform" >&2; exit 1 ;; +esac +install_dir=${SPACETIMEDB_INSTALL_DIR:-$HOME/.local/bin} +mkdir -p "$install_dir" +archive=$(mktemp) +trap 'rm -f "$archive"' EXIT +curl --fail --location --silent --show-error \ + "https://github.com/clockworklabs/SpacetimeDB/releases/download/v${version}/spacetime-${target}.tar.gz" -o "$archive" +tar -xzf "$archive" -C "$install_dir" +ln -sf "$install_dir/spacetimedb-cli" "$install_dir/spacetime" +"$install_dir/spacetime" --version +if [[ -n "${GITHUB_PATH:-}" ]]; then echo "$install_dir" >> "$GITHUB_PATH"; fi diff --git a/scripts/spacetimedb-version b/scripts/spacetimedb-version new file mode 100644 index 0000000..8bbb6e4 --- /dev/null +++ b/scripts/spacetimedb-version @@ -0,0 +1 @@ +2.10.1 diff --git a/scripts/start-spacetimedb.sh b/scripts/start-spacetimedb.sh new file mode 100755 index 0000000..90d1874 --- /dev/null +++ b/scripts/start-spacetimedb.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +mkdir -p ci-logs +nohup spacetime start > ci-logs/spacetimedb-server.log 2>&1 < /dev/null & +server_pid=$! +echo "$server_pid" > ci-logs/spacetimedb-server.pid +for attempt in $(seq 1 60); do + if curl -sf http://localhost:3000/v1/ping >/dev/null; then + spacetime publish --server http://localhost:3000 \ + --bin-path rust/spacetime-module/target/wasm32-unknown-unknown/release/spacetime_module.wasm \ + --yes benchmark-links + exit 0 + fi + if ! kill -0 "$server_pid" 2>/dev/null; then cat ci-logs/spacetimedb-server.log; exit 1; fi + sleep 1 +done +cat ci-logs/spacetimedb-server.log +exit 1 diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index ff6e523..0000000 --- a/src/lib.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! Example module entry point. -//! -//! Replace this with your actual implementation. - -/// Package version (matches Cargo.toml version). -pub const VERSION: &str = env!("CARGO_PKG_VERSION"); - -/// Adds two numbers together. -/// -/// # Arguments -/// -/// * `a` - First number -/// * `b` - Second number -/// -/// # Returns -/// -/// Sum of `a` and `b` -/// -/// # Examples -/// -/// ``` -/// use my_package::add; -/// assert_eq!(add(2, 3), 5); -/// ``` -#[must_use] -pub const fn add(a: i64, b: i64) -> i64 { - a + b -} - -/// Multiplies two numbers together. -/// -/// # Arguments -/// -/// * `a` - First number -/// * `b` - Second number -/// -/// # Returns -/// -/// Product of `a` and `b` -/// -/// # Examples -/// -/// ``` -/// use my_package::multiply; -/// assert_eq!(multiply(2, 3), 6); -/// ``` -#[must_use] -pub const fn multiply(a: i64, b: i64) -> i64 { - a * b -} - -/// Async delay function. -/// -/// # Arguments -/// -/// * `seconds` - Duration to wait in seconds -/// -/// # Examples -/// -/// ``` -/// use my_package::delay; -/// -/// #[tokio::main] -/// async fn main() { -/// delay(0.1).await; -/// } -/// ``` -pub async fn delay(seconds: f64) { - let duration = std::time::Duration::from_secs_f64(seconds); - tokio::time::sleep(duration).await; -} - -#[cfg(test)] -mod tests { - use super::*; - - mod add_tests { - use super::*; - - #[test] - fn test_add_positive_numbers() { - assert_eq!(add(2, 3), 5); - } - - #[test] - fn test_add_negative_numbers() { - assert_eq!(add(-1, -2), -3); - } - - #[test] - fn test_add_zero() { - assert_eq!(add(5, 0), 5); - } - - #[test] - fn test_add_large_numbers() { - assert_eq!(add(1_000_000, 2_000_000), 3_000_000); - } - } - - mod multiply_tests { - use super::*; - - #[test] - fn test_multiply_positive_numbers() { - assert_eq!(multiply(2, 3), 6); - } - - #[test] - fn test_multiply_by_zero() { - assert_eq!(multiply(5, 0), 0); - } - - #[test] - fn test_multiply_negative_numbers() { - assert_eq!(multiply(-2, 3), -6); - } - - #[test] - fn test_multiply_two_negatives() { - assert_eq!(multiply(-2, -3), 6); - } - } - - mod delay_tests { - use super::*; - - #[tokio::test] - async fn test_delay() { - let start = std::time::Instant::now(); - delay(0.1).await; - let elapsed = start.elapsed(); - assert!(elapsed.as_secs_f64() >= 0.1); - } - } -} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index bffc9ba..0000000 --- a/src/main.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Example binary entry point. -//! -//! This is a simple CLI that demonstrates the library functionality. - -use my_package::{add, delay, multiply}; - -#[tokio::main] -async fn main() { - println!("my-package v{}", my_package::VERSION); - println!(); - - // Example 1: Basic arithmetic - println!("Example 1: Basic arithmetic"); - println!("2 + 3 = {}", add(2, 3)); - println!("2 * 3 = {}", multiply(2, 3)); - println!(); - - // Example 2: Working with larger numbers - println!("Example 2: Working with larger numbers"); - println!("1000 + 2000 = {}", add(1000, 2000)); - println!("100 * 200 = {}", multiply(100, 200)); - println!(); - - // Example 3: Async delay - println!("Example 3: Async delay"); - println!("Waiting for 1 second..."); - delay(1.0).await; - println!("Done!"); -} diff --git a/tests/integration_test.rs b/tests/integration_test.rs deleted file mode 100644 index 89fa538..0000000 --- a/tests/integration_test.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Integration tests for my-package. -//! -//! These tests verify the public API works correctly. - -use my_package::{add, delay, multiply}; - -mod add_integration_tests { - use super::*; - - #[test] - fn test_add_returns_correct_sum() { - assert_eq!(add(10, 20), 30); - } - - #[test] - fn test_add_handles_large_numbers() { - assert_eq!(add(1_000_000_000, 2_000_000_000), 3_000_000_000); - } - - #[test] - fn test_add_handles_negative_result() { - assert_eq!(add(-100, 50), -50); - } -} - -mod multiply_integration_tests { - use super::*; - - #[test] - fn test_multiply_returns_correct_product() { - assert_eq!(multiply(10, 20), 200); - } - - #[test] - fn test_multiply_handles_large_numbers() { - assert_eq!(multiply(1_000, 1_000_000), 1_000_000_000); - } - - #[test] - fn test_multiply_handles_negative_numbers() { - assert_eq!(multiply(-10, -20), 200); - } -} - -mod delay_integration_tests { - use super::*; - - #[tokio::test] - async fn test_delay_waits_minimum_time() { - let start = std::time::Instant::now(); - delay(0.05).await; - let elapsed = start.elapsed(); - - assert!( - elapsed.as_secs_f64() >= 0.05, - "Delay should wait at least 0.05 seconds, but waited {:.4}s", - elapsed.as_secs_f64() - ); - } - - #[tokio::test] - async fn test_delay_zero_completes_quickly() { - let start = std::time::Instant::now(); - delay(0.0).await; - let elapsed = start.elapsed(); - - assert!( - elapsed.as_secs_f64() < 0.1, - "Zero delay should complete quickly, but took {:.4}s", - elapsed.as_secs_f64() - ); - } -} - -mod version_tests { - use my_package::VERSION; - - #[test] - fn test_version_is_not_empty() { - assert!(!VERSION.is_empty()); - } - - #[test] - fn test_version_matches_cargo_toml() { - // Version should match the one in Cargo.toml - assert!(VERSION.starts_with("0.")); - } -} From 79b472698aaef55abe726d265e52e8ec4678b922 Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:43:01 +0000 Subject: [PATCH 6/7] Keep Query All results observable in Rust benchmarks --- rust/benches/bench.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/benches/bench.rs b/rust/benches/bench.rs index 27f7e9a..ba7a1cc 100644 --- a/rust/benches/bench.rs +++ b/rust/benches/bench.rs @@ -525,7 +525,7 @@ fn spacetimedb_query_all(c: &mut Criterion) { fork.create_point(); } let start = Instant::now(); - let _ = fork.query_all(); + std::hint::black_box(fork.query_all()); total += start.elapsed(); } total @@ -553,7 +553,7 @@ fn doublets_united_query_all(c: &mut Criterion) { fork.create_point(); } let start = Instant::now(); - let _ = fork.query_all(); + std::hint::black_box(fork.query_all()); total += start.elapsed(); } total @@ -581,7 +581,7 @@ fn doublets_split_query_all(c: &mut Criterion) { fork.create_point(); } let start = Instant::now(); - let _ = fork.query_all(); + std::hint::black_box(fork.query_all()); total += start.elapsed(); } total @@ -611,7 +611,7 @@ fn doublets_united_non_volatile_query_all(c: &mut Criterion) { fork.create_point(); } let start = Instant::now(); - let _ = fork.query_all(); + std::hint::black_box(fork.query_all()); total += start.elapsed(); } total @@ -642,7 +642,7 @@ fn doublets_split_non_volatile_query_all(c: &mut Criterion) { fork.create_point(); } let start = Instant::now(); - let _ = fork.query_all(); + std::hint::black_box(fork.query_all()); total += start.elapsed(); } total From 6845921989dd276dc00d1c1b447b56fe52acfd21 Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:53:23 +0000 Subject: [PATCH 7/7] Revert "Initial commit with task details" This reverts commit 2a3424479fb5ec4464c4f8f85a4ba385512bc8b9. --- .gitkeep | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index 013780a..0000000 --- a/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# .gitkeep file auto-generated at 2026-10-06T23:17:41.707Z for PR creation at branch issue-16-e3181c5664cc for issue https://github.com/linksplatform/Comparisons.SpacetimeDBVSDoublets/issues/16 \ No newline at end of file