From 11d29122623763e00f9ac3d0f20c428b4805b9aa Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:30:30 +0000 Subject: [PATCH 1/6] Initial commit with task details Adding .gitkeep for PR creation (default mode). This file will be removed when the task is complete. Issue: https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/issues/42 --- .gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitkeep diff --git a/.gitkeep b/.gitkeep new file mode 100644 index 0000000..3f99036 --- /dev/null +++ b/.gitkeep @@ -0,0 +1 @@ +# .gitkeep file auto-generated at 2026-10-06T23:30:30.621Z for PR creation at branch issue-42-5c34a9a6fc98 for issue https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/issues/42 \ No newline at end of file From b6451b3ff7fda879ba46d8638b961580f96c8f0f Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:34:24 +0000 Subject: [PATCH 2/6] Generate tested benchmark reports with noise-aware ratios and provenance --- experiments/reproduce_report_ratio.py | 18 ++ rust/out.py | 204 +------------- scripts/benchmark_report.py | 377 ++++++++++++++++++++++++++ scripts/test_benchmark_report.py | 88 ++++++ 4 files changed, 491 insertions(+), 196 deletions(-) create mode 100644 experiments/reproduce_report_ratio.py create mode 100644 scripts/benchmark_report.py create mode 100644 scripts/test_benchmark_report.py diff --git a/experiments/reproduce_report_ratio.py b/experiments/reproduce_report_ratio.py new file mode 100644 index 0000000..3656341 --- /dev/null +++ b/experiments/reproduce_report_ratio.py @@ -0,0 +1,18 @@ +"""Reproduce #42 without plotting dependencies: python3 experiments/reproduce_report_ratio.py.""" +import ast +from pathlib import Path + +path = Path(__file__).resolve().parents[1] / 'rust/out.py' +tree = ast.parse(path.read_text()) +legacy = next((node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == 'annotate'), None) +if legacy is not None: + namespace = {'min_psql': 100} + exec(compile(ast.fix_missing_locations(ast.Module(body=[legacy], type_ignores=[])), str(path), 'exec'), namespace) + result = namespace['annotate'](200) +else: + import sys + sys.path.insert(0, str(path.parents[1] / 'scripts')) + from benchmark_report import comparison + result = comparison((200, 0), (100, 0)) +print(result) +assert 'slower' in result, 'Doublets takes twice as long, so it must be labelled slower' diff --git a/rust/out.py b/rust/out.py index 7241de3..41f0d27 100644 --- a/rust/out.py +++ b/rust/out.py @@ -1,198 +1,10 @@ -import re -import logging -import matplotlib.pyplot as plt -import numpy as np +#!/usr/bin/env python3 +"""Compatibility entry point for the shared Rust and C# report generator.""" +import sys +from pathlib import Path -# Enable detailed tracing. Set to False to disable verbose output. -DEBUG = True -logging.basicConfig(level=logging.INFO if DEBUG else logging.WARNING, - format="%(message)s") +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from benchmark_report import main -# Read the complete file -data = open("out.txt").read() -if DEBUG: - logging.info("Loaded out.txt, length: %d characters", len(data)) - -# Use two regex patterns to extract benchmarks. -# The first captures PostgreSQL tests and the second captures Doublets tests. -patterns = [ - r"test\s+(\w+)/(PSQL)_(\w+)\s+\.\.\.\s+bench:\s+(\d+)\s+ns/iter\s+\(\+/-\s+\d+\)", - r"test\s+(\w+)/(Doublets)_(\w+)_(\w+)\s+\.\.\.\s+bench:\s+(\d+)\s+ns/iter\s+\(\+/-\s+\d+\)" -] - -# Instead of using lists, we use dictionaries mapping operation names to values. -PSQL_Transaction = {} -PSQL_NonTransaction = {} -Doublets_United_Volatile = {} -Doublets_United_NonVolatile = {} -Doublets_Split_Volatile = {} -Doublets_Split_NonVolatile = {} - -# Process each regex pattern -for pattern in patterns: - matches = re.findall(pattern, data) - if DEBUG: - logging.info("Pattern %s matched %d entries", pattern, len(matches)) - for match in matches: - # Normalise name - op = match[0].replace("_", " ") # Create, Each All, … - if match[1] == 'PSQL': - # (operation, 'PSQL', transaction, time) - transaction = match[2] - time_val = int(match[3]) - if DEBUG: - logging.info("PSQL %s - %s: %d ns", op, transaction, time_val) - if transaction == "Transaction": - PSQL_Transaction[op] = time_val - else: - PSQL_NonTransaction[op] = time_val - else: - # (operation, 'Doublets', trees, storage, time) - trees = match[2] - storage = match[3] - time_val = int(match[4]) - if DEBUG: - logging.info("Doublets %s - %s %s: %d ns", op, trees, storage, time_val) - if trees == 'United': - if storage == 'Volatile': - Doublets_United_Volatile[op] = time_val - else: - Doublets_United_NonVolatile[op] = time_val - else: - if storage == 'Volatile': - Doublets_Split_Volatile[op] = time_val - else: - Doublets_Split_NonVolatile[op] = time_val - -# Operation order for table and plots -ordered_ops = [ - "Create", "Update", "Delete", - "Each All", "Each Identity", "Each Concrete", "Each Outgoing", "Each Incoming" -] - -if DEBUG: - logging.info("\nFinal dictionaries (after parsing):") - logging.info("PSQL_Transaction: %s", PSQL_Transaction) - logging.info("PSQL_NonTransaction: %s", PSQL_NonTransaction) - logging.info("Doublets_United_Volatile: %s", Doublets_United_Volatile) - logging.info("Doublets_United_NonVolatile: %s", Doublets_United_NonVolatile) - logging.info("Doublets_Split_Volatile: %s", Doublets_Split_Volatile) - logging.info("Doublets_Split_NonVolatile: %s", Doublets_Split_NonVolatile) - -# Assemble series in the desired order. -def get_series(d): return [d.get(op, 0) for op in ordered_ops] - -du_volatile_arr = get_series(Doublets_United_Volatile) -du_nonvolatile_arr = get_series(Doublets_United_NonVolatile) -ds_volatile_arr = get_series(Doublets_Split_Volatile) -ds_nonvolatile_arr = get_series(Doublets_Split_NonVolatile) -psql_non_arr = get_series(PSQL_NonTransaction) -psql_trans_arr = get_series(PSQL_Transaction) - -# ───────────────────────────────────────────────────────────────────────────── -# Markdown Table -# ───────────────────────────────────────────────────────────────────────────── -def print_results_markdown(): - header = ( - "| Operation | Doublets United Volatile | Doublets United NonVolatile | " - "Doublets Split Volatile | Doublets Split NonVolatile | PSQL NonTransaction | PSQL Transaction |\n" - "|---------------|--------------------------|-----------------------------|-------------------------|----------------------------|---------------------|------------------|" - ) - lines = [header] - - for i, op in enumerate(ordered_ops): - psql_val1 = psql_non_arr[i] if psql_non_arr[i] else float('inf') - psql_val2 = psql_trans_arr[i] if psql_trans_arr[i] else float('inf') - min_psql = min(psql_val1, psql_val2) - - def annotate(v): - if v == 0: return "N/A" - if min_psql == float('inf'): return f"{v}" - return f"{v} ({min_psql / v:.1f}x faster)" - - row = ( - f"| {op:<13} | {annotate(du_volatile_arr[i]):<24} | " - f"{annotate(du_nonvolatile_arr[i]):<27} | " - f"{annotate(ds_volatile_arr[i]):<23} | " - f"{annotate(ds_nonvolatile_arr[i]):<26} | " - f"{psql_non_arr[i] or 'N/A':<19} | {psql_trans_arr[i] or 'N/A':<16} |" - ) - lines.append(row) - - table_md = "\n".join(lines) - print(table_md) - - # Save to file for CI to use - with open("results.md", "w") as f: - f.write(table_md) - - if DEBUG: logging.info("\nGenerated Markdown Table:\n%s", table_md) - -# ───────────────────────────────────────────────────────────────────────────── -# Plots -# ───────────────────────────────────────────────────────────────────────────── -def ensure_min_visible(arr, min_val): - """Ensure non-zero values are at least min_val for visibility on graph.""" - return [max(v, min_val) if v > 0 else 0 for v in arr] - -def bench1(): - """Horizontal bars – raw values (pixel scale).""" - y, w = np.arange(len(ordered_ops)), 0.1 - fig, ax = plt.subplots(figsize=(12, 8)) - - # Calculate maximum value across all data series to determine scale - all_values = (du_volatile_arr + du_nonvolatile_arr + ds_volatile_arr + - ds_nonvolatile_arr + psql_non_arr + psql_trans_arr) - max_val = max(all_values) if all_values else 1 - - # Minimum visible bar width: ~0.5% of max value ensures at least 2 pixels - # on typical 12-inch wide figure at 100 DPI (~900px plot area) - min_visible = max_val * 0.005 - if DEBUG: - logging.info("bench1: max_val=%d, min_visible=%d", max_val, min_visible) - - # Apply minimum visibility to all data series - du_volatile_vis = ensure_min_visible(du_volatile_arr, min_visible) - du_nonvolatile_vis = ensure_min_visible(du_nonvolatile_arr, min_visible) - ds_volatile_vis = ensure_min_visible(ds_volatile_arr, min_visible) - ds_nonvolatile_vis = ensure_min_visible(ds_nonvolatile_arr, min_visible) - psql_non_vis = ensure_min_visible(psql_non_arr, min_visible) - psql_trans_vis = ensure_min_visible(psql_trans_arr, min_visible) - - ax.barh(y - 2*w, du_volatile_vis, w, label='Doublets United Volatile', color='salmon') - ax.barh(y - w, du_nonvolatile_vis, w, label='Doublets United NonVolatile', color='red') - ax.barh(y, ds_volatile_vis, w, label='Doublets Split Volatile', color='lightgreen') - ax.barh(y + w, ds_nonvolatile_vis, w, label='Doublets Split NonVolatile', color='green') - ax.barh(y + 2*w, psql_non_vis, w, label='PSQL NonTransaction', color='lightblue') - ax.barh(y + 3*w, psql_trans_vis, w, label='PSQL Transaction', color='blue') - - ax.set_xlabel('Time (ns)') - ax.set_title ('Benchmark Comparison: PostgreSQL vs Doublets (Rust)') - ax.set_yticks(y); ax.set_yticklabels(ordered_ops); ax.legend() - fig.tight_layout(); plt.savefig("bench_rust.png"); plt.close(fig) - if DEBUG: logging.info("bench_rust.png saved.") - -def bench2(): - """Horizontal bars – raw values on a log scale.""" - y, w = np.arange(len(ordered_ops)), 0.1 - fig, ax = plt.subplots(figsize=(12, 8)) - - ax.barh(y - 2*w, du_volatile_arr, w, label='Doublets United Volatile', color='salmon') - ax.barh(y - w, du_nonvolatile_arr, w, label='Doublets United NonVolatile', color='red') - ax.barh(y, ds_volatile_arr, w, label='Doublets Split Volatile', color='lightgreen') - ax.barh(y + w, ds_nonvolatile_arr, w, label='Doublets Split NonVolatile', color='green') - ax.barh(y + 2*w, psql_non_arr, w, label='PSQL NonTransaction', color='lightblue') - ax.barh(y + 3*w, psql_trans_arr, w, label='PSQL Transaction', color='blue') - - ax.set_xlabel('Time (ns) – log scale') - ax.set_title ('Benchmark Comparison: PostgreSQL vs Doublets (Rust)') - ax.set_yticks(y); ax.set_yticklabels(ordered_ops); ax.set_xscale('log'); ax.legend() - fig.tight_layout(); plt.savefig("bench_rust_log_scale.png"); plt.close(fig) - if DEBUG: logging.info("bench_rust_log_scale.png saved.") - -# ───────────────────────────────────────────────────────────────────────────── -# Run -# ───────────────────────────────────────────────────────────────────────────── -print_results_markdown() -bench1() -bench2() +if __name__ == "__main__": + main() diff --git a/scripts/benchmark_report.py b/scripts/benchmark_report.py new file mode 100644 index 0000000..de8eac0 --- /dev/null +++ b/scripts/benchmark_report.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""Turns the benchmark output into the README results and charts. + +Input: `RESULTS_DIR/--.txt`, the output of +one benchmark run for one language (`rust` or `csharp`), one number of +background links and one backend (`psql` or `doublets`). Both languages print +the bencher format of Criterion, + + test Create/PSQL_Transaction ... bench: 44,055,505 ns/iter (+/- 5,345,991) + +with the median time of one iteration and its standard deviation, after +`# key: value` lines that describe the run (see `scripts/benchmark_header.sh`). + +Output, for every language and number of background links: +- a Markdown table, written into the README between the markers; +- `/bench__.png` (linear scale) and + `/bench__log_scale_.png` (log scale). + +Usage: benchmark_report.py RESULTS_DIR [--readme README.md] [--charts Docs] +""" + +import argparse +import logging +import os +import re +from pathlib import Path + +# Enable detailed tracing. Set to True to see every parsed result. +DEBUG = False +logging.basicConfig(level=logging.INFO if DEBUG else logging.WARNING, format="%(message)s") +log = logging.getLogger("benchmark_report") + +ROOT = Path(__file__).resolve().parent.parent +START_MARKER = "" +END_MARKER = "" +# The wide tables cannot be wrapped, and every language repeats the same headings. +LINT_OFF = "" +LINT_ON = "" + +# Medians that differ by less than 5%, or whose ranges of one standard +# deviation overlap, are not called a difference. +NOISE = 0.05 + +OPERATIONS = [ + "Create", "Update", "Delete", + "Each All", "Each Identity", "Each Concrete", "Each Outgoing", "Each Incoming", +] + +# Benchmark id, column name and chart color of every implementation. +DOUBLETS = [ + ("Doublets_United_Volatile", "Doublets United Volatile", "salmon"), + ("Doublets_United_NonVolatile", "Doublets United NonVolatile", "red"), + ("Doublets_Split_Volatile", "Doublets Split Volatile", "lightgreen"), + ("Doublets_Split_NonVolatile", "Doublets Split NonVolatile", "green"), +] +PSQL = [ + ("PSQL_NonTransaction", "PostgreSQL NonTransaction", "lightblue"), + ("PSQL_Transaction", "PostgreSQL Transaction", "blue"), + ] +IMPLEMENTATIONS = DOUBLETS + PSQL +IMPLEMENTATION_IDS = {implementation for implementation, _, _ in IMPLEMENTATIONS} +DOUBLETS_IDS = {implementation for implementation, _, _ in DOUBLETS} +BACKENDS = {"doublets": DOUBLETS, "psql": PSQL} + +# Each All is one statement already, so it has no batch variant. +NOT_MEASURED = set() + +# Name in the README, and how to find the versions of the PostgreSQL driver and of +# the Doublets library in the repository: (file, regex with the version). +LANGUAGES = { + "rust": ( + "Rust", + [ + ("postgres", "rust/Cargo.lock", r'name = "postgres"\nversion = "([^"]+)"'), + ("doublets", "rust/Cargo.lock", r'name = "doublets"\nversion = "([^"]+)"'), + ], + ), + "csharp": ( + "C#", + [ + ("Npgsql", "csharp/PostgreSQLVSDoublets/PostgreSQLVSDoublets.csproj", + r'Include="Npgsql" Version="([^"]+)"'), + ("Platform.Data.Doublets", "csharp/PostgreSQLVSDoublets/PostgreSQLVSDoublets.csproj", + r'Include="Platform\.Data\.Doublets" Version="([^"]+)"'), + ], + ), +} + +# For example: `test Create/PSQL_Transaction ... bench: 44,055,505 ns/iter (+/- 5,345,991)` +# (Criterion 0.8 separates thousands with commas, older versions did not). +BENCH_LINE = re.compile( + r"test\s+(\w+)/(\w+)\s+\.\.\.\s+bench:\s+([\d,]+)\s+ns/iter\s+\(\+/-\s+([\d,]+)\)" +) +METADATA_LINE = re.compile(r"#\s*([\w ]+?)\s*:\s*(.*?)\s*$") +FILE_NAME = re.compile(r"(\w+)-(\d+)-(\w+)") + + +def parse(text, path=""): + """Returns ({(operation, implementation): (median ns, std dev ns)}, {key: value}).""" + times, metadata = {}, {} + for line in text.splitlines(): + if line.startswith("#"): + match = METADATA_LINE.match(line) + if match: + metadata[match.group(1).lower()] = match.group(2) + continue + if not line.startswith("test "): + continue + # A line without a time means the benchmark reported an error + # instead of its result; such a run must not be published. + match = BENCH_LINE.match(line) + if not match: + raise ValueError(f"{path}: no result in line: {line}") + group, implementation, median, deviation = match.groups() + operation = group.replace("_", " ") + if operation not in OPERATIONS or implementation not in IMPLEMENTATION_IDS: + raise ValueError(f"{path}: unknown benchmark {group}/{implementation}") + median, deviation = int(median.replace(",", "")), int(deviation.replace(",", "")) + if median <= 0: + raise ValueError(f"{path}: {group}/{implementation} took no time") + if (operation, implementation) in times: + raise ValueError(f"{path}: duplicate result {group}/{implementation}") + times[(operation, implementation)] = (median, deviation) + log.info("%s: %s %s: %s ns (+/- %s)", path, operation, implementation, median, deviation) + return times, metadata + + +def load(directory): + """All results in `directory`: {(language, background): {"times", "metadata"}}.""" + results = {} + for path in sorted(Path(directory).glob("*.txt")): + match = FILE_NAME.fullmatch(path.stem) + if not match or match.group(1) not in LANGUAGES or match.group(3) not in BACKENDS: + raise ValueError(f"{path}: expected <{'|'.join(LANGUAGES)}>--<{'|'.join(BACKENDS)}>.txt") + language, background, backend = match.group(1), int(match.group(2)), match.group(3) + times, metadata = parse(path.read_text(encoding="utf-8"), path) + expected = { + (operation, implementation) + for operation in OPERATIONS + for implementation, _, _ in BACKENDS[backend] + } - NOT_MEASURED + if set(times) != expected: + missing = sorted(expected - set(times)) + unexpected = sorted(set(times) - expected) + raise ValueError(f"{path}: missing results {missing}, unexpected results {unexpected}") + result = results.setdefault((language, background), {"times": {}, "metadata": {}}) + result["times"].update(times) + result["metadata"][backend] = metadata + if not results: + raise ValueError(f"no results found in {directory}/") + for (language, background), result in results.items(): + missing = set(BACKENDS) - set(result["metadata"]) + if missing: + raise ValueError(f"{language}, {background} background links: no {', '.join(sorted(missing))} results") + return dict(sorted(results.items(), key=lambda item: (list(LANGUAGES).index(item[0][0]), item[0][1]))) + + +def significant(value): + """Three significant digits without an exponent: 999.6 is 1000, not 1e+03.""" + return f"{float(f'{value:.3g}'):g}" + + +def duration(nanoseconds): + rounded = float(significant(nanoseconds)) + for unit, scale in (("s", 1e9), ("ms", 1e6), ("µs", 1e3)): + if rounded >= scale: + return f"{significant(rounded / scale)} {unit}" + return f"{significant(rounded)} ns" + + +def ratio(value): + """`significant`, with thousands separators: 36,600, not 36600.""" + rounded = float(significant(value)) + return f"{rounded:,.0f}" if rounded >= 1000 else significant(rounded) + + +def comparison(measured, reference): + """How `measured` compares with `reference`, both (median, std dev).""" + (median, deviation), (reference_median, reference_deviation) = measured, reference + overlapping = ( + median - deviation <= reference_median + reference_deviation + and reference_median - reference_deviation <= median + deviation + ) + if overlapping or max(median, reference_median) / min(median, reference_median) <= 1 + NOISE: + return "≈ same" + if median < reference_median: + return f"{ratio(reference_median / median)}× faster" + return f"{ratio(median / reference_median)}× slower" + + +def baseline(times, operation): + """The fastest PostgreSQL implementation of `operation`.""" + measured = [ + implementation for implementation, _, _ in PSQL if (operation, implementation) in times + ] + return min(measured, key=lambda implementation: times[(operation, implementation)][0]) + + +def table(times): + header = "| Operation | " + " | ".join(name for _, name, _ in IMPLEMENTATIONS) + " |" + separator = "| --- |" + " ---: |" * len(IMPLEMENTATIONS) + rows = [header, separator] + for operation in OPERATIONS: + fastest = baseline(times, operation) + cells = [] + for implementation, _, _ in IMPLEMENTATIONS: + measured = times.get((operation, implementation)) + if measured is None: + cells.append("—") + elif implementation == fastest: + cells.append(f"**{duration(measured[0])}**") + elif implementation in DOUBLETS_IDS: + cells.append(f"{duration(measured[0])} ({comparison(measured, times[(operation, fastest)])})") + else: + cells.append(duration(measured[0])) + rows.append(f"| {operation} | " + " | ".join(cells) + " |") + return "\n".join(rows) + + +def versions(language, root=ROOT): + """`name version` of the PostgreSQL driver and the Doublets library of `language`.""" + found = [] + for name, file, pattern in LANGUAGES[language][1]: + path = root / file + match = re.search(pattern, path.read_text(encoding="utf-8")) if path.exists() else None + found.append(f"{name} {match.group(1) if match else 'unknown version'}") + return found + + +def provenance(language, result, links, root=ROOT): + """Where the numbers come from: versions, machines and the run.""" + psql, doublets = result["metadata"]["psql"], result["metadata"]["doublets"] + default_driver, default_library = versions(language, root) + driver = psql.get("driver", default_driver) + library = doublets.get("doublets", default_library) + cpus = {psql.get("cpu"), doublets.get("cpu")} - {None} + if not cpus: + machine = "CPU unknown" + elif len(cpus) == 1: + machine = f"CPU {cpus.pop()}" + else: + machine = f"CPU {psql.get('cpu', 'unknown')} (PostgreSQL) and {doublets.get('cpu', 'unknown')} (Doublets)" + run, date = psql.get("run") or doublets.get("run"), psql.get("date") or doublets.get("date") + source = f"[GitHub Actions run]({run})" if run else "a local run" + if date: + source += f" on {date}" + return ( + f"_Median time of one iteration with {links:,} links. " + f"PostgreSQL {psql.get('postgresql', 'unknown version')} through {driver}; {library}. " + f"{machine}, {source}._" + ) + + +def links_per_iteration(result): + counts = {metadata.get("links") for metadata in result["metadata"].values()} + if len(counts) != 1 or None in counts: + raise ValueError(f"the backends were measured with different or unknown links per iteration: {counts}") + return int(counts.pop()) + + +def chart_names(language, background): + return f"bench_{language}_{background}.png", f"bench_{language}_log_scale_{background}.png" + + +def section(results, charts_link): + """The Markdown between the markers: a heading per language and per number of background links.""" + lines = [LINT_OFF, ""] + for language, (title, _) in LANGUAGES.items(): + lines += [f"### {title}", ""] + backgrounds = [background for (lang, background) in results if lang == language] + if not backgrounds: + lines += ["_No results yet._", ""] + continue + for background in backgrounds: + result = results[(language, background)] + links = links_per_iteration(result) + heading = f"{background:,} background links, {links:,} links per iteration" + lines += [ + f"#### {title}: {heading}", + "", + provenance(language, result, links), + "", + table(result["times"]), + "", + ] + if charts_link is not None: + linear, log = chart_names(language, background) + lines += [ + f"![{title}, {heading}, linear scale]({charts_link}/{linear})", + f"![{title}, {heading}, log scale]({charts_link}/{log})", + "", + ] + lines += [LINT_ON] + return "\n".join(lines) + + +def replace_section(document, generated): + pattern = re.compile(re.escape(START_MARKER) + ".*?" + re.escape(END_MARKER), re.DOTALL) + if document.count(START_MARKER) != 1 or document.count(END_MARKER) != 1 or not pattern.search(document): + raise ValueError(f"{START_MARKER} ... {END_MARKER} markers are missing") + return pattern.sub(lambda _: f"{START_MARKER}\n{generated}\n{END_MARKER}", document) + + +def chart(times, title, path, log_scale): + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.ticker import EngFormatter + + positions, width = range(len(OPERATIONS)), 0.8 / len(IMPLEMENTATIONS) + figure, axis = plt.subplots(figsize=(12, 8)) + # Seconds, so that the axis shows `10 ms` instead of `1e7`. + values = [ + [times.get((operation, implementation), (0, 0))[0] / 1e9 for operation in OPERATIONS] + for implementation, _, _ in IMPLEMENTATIONS + ] + if not log_scale: + # Bars of 0.5% of the longest bar stay visible (about 4 pixels). + min_visible = max(max(series) for series in values) * 0.005 + values = [[max(value, min_visible) if value else 0 for value in series] for series in values] + for index, ((_, name, color), series) in enumerate(zip(IMPLEMENTATIONS, values, strict=True)): + offset = (index - (len(IMPLEMENTATIONS) - 1) / 2) * width + axis.barh([position + offset for position in positions], series, width, label=name, color=color) + axis.set_xlabel("Median time of one iteration" + (", log scale" if log_scale else "")) + if log_scale: + axis.set_xscale("log") + axis.xaxis.set_major_formatter(EngFormatter(unit="s")) + axis.grid(axis="x", alpha=0.3) + axis.set_title(title) + axis.set_yticks(list(positions)) + axis.set_yticklabels(OPERATIONS) + axis.invert_yaxis() + axis.legend(loc="upper left", bbox_to_anchor=(1.01, 1)) + figure.tight_layout() + figure.savefig(path) + plt.close(figure) + log.info("%s saved", path) + + +def charts(results, directory): + directory.mkdir(parents=True, exist_ok=True) + written = [] + for (language, background), result in results.items(): + title = ( + f"{LANGUAGES[language][0]}: {background:,} background links, " + f"{links_per_iteration(result):,} links per iteration" + ) + linear, log = chart_names(language, background) + chart(result["times"], title, directory / linear, log_scale=False) + chart(result["times"], title, directory / log, log_scale=True) + written += [directory / linear, directory / log] + return written + + +def main(arguments=None): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("results", type=Path, help="directory with the benchmark output") + parser.add_argument("--readme", type=Path, help="README to update in place") + parser.add_argument("--charts", type=Path, help="directory for the charts") + options = parser.parse_args(arguments) + results = load(options.results) + if options.charts is not None: + charts(results, options.charts) + link = None + if options.charts is not None: + base = options.readme.resolve().parent if options.readme else Path.cwd() + link = Path(os.path.relpath(options.charts.resolve(), base)).as_posix() + generated = section(results, link) + print(generated) + if options.readme is not None: + document = options.readme.read_text(encoding="utf-8") + options.readme.write_text(replace_section(document, generated), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/test_benchmark_report.py b/scripts/test_benchmark_report.py new file mode 100644 index 0000000..a0658ec --- /dev/null +++ b/scripts/test_benchmark_report.py @@ -0,0 +1,88 @@ +import tempfile +import unittest +from pathlib import Path + +import benchmark_report as report + + +class ReportTests(unittest.TestCase): + def test_comparisons(self): + cases = [((50, 0), (100, 0), '2× faster'), + ((200, 0), (100, 0), '2× slower'), + ((104, 0), (100, 0), '≈ same'), + ((100, 0), (105, 0), '≈ same'), + ((70, 20), (100, 15), '≈ same')] + for measured, baseline, expected in cases: + with self.subTest(measured=measured): + self.assertEqual(report.comparison(measured, baseline), expected) + + def test_parse_deviation_and_commas(self): + times, metadata = report.parse('# cpu: Example CPU\ntest Create/PSQL_Transaction ... bench: 1,234 ns/iter (+/- 56)') + self.assertEqual(times[('Create', 'PSQL_Transaction')], (1234, 56)) + self.assertEqual(metadata['cpu'], 'Example CPU') + + def test_reject_invalid_results(self): + for line in ['test Create/PSQL_Transaction ... ERROR', + 'test Unknown/PSQL_Transaction ... bench: 10 ns/iter (+/- 0)', + 'test Create/Unknown ... bench: 10 ns/iter (+/- 0)', + 'test Create/PSQL_Transaction ... bench: 0 ns/iter (+/- 0)']: + with self.subTest(line=line), self.assertRaises(ValueError): + report.parse(line) + + def test_duplicate_is_rejected(self): + line = 'test Create/PSQL_Transaction ... bench: 10 ns/iter (+/- 0)' + with self.assertRaises(ValueError): + report.parse(line + '\n' + line) + + def test_markers_preserve_surrounding_text(self): + doc = 'before\n\nstale\n\nafter' + self.assertEqual(report.replace_section(doc, 'new'), doc.replace('stale', 'new')) + for invalid in ['no markers', '', doc + doc]: + with self.assertRaises(ValueError): + report.replace_section(invalid, 'new') + + def test_missing_backend_rejected(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'rust-100-doublets.txt' + path.write_text(self.output('doublets')) + with self.assertRaises(ValueError): + report.load(directory) + + @staticmethod + def output(backend, links=10): + lines = [f'# links: {links}', '# background: 100', '# cpu: Test CPU', + '# date: 2026-10-06', '# run: https://github.com/example/actions/runs/1', + '# postgresql: 17.6', '# driver: test-driver 1', '# doublets: test-doublets 2'] + lines += [f'test {op.replace(" ", "_")}/{implementation} ... bench: 100 ns/iter (+/- 10)' + for op in report.OPERATIONS for implementation, _, _ in report.BACKENDS[backend]] + return '\n'.join(lines) + + def test_complete_report_and_mismatched_sizes(self): + with tempfile.TemporaryDirectory() as directory: + for backend in report.BACKENDS: + (Path(directory) / f'rust-100-{backend}.txt').write_text(self.output(backend)) + results = report.load(directory) + generated = report.section(results, None) + for text in ['### Rust', '### C#', '≈ same', 'Test CPU', '17.6', 'GitHub Actions run']: + self.assertIn(text, generated) + (Path(directory) / 'rust-100-psql.txt').write_text(self.output('psql', 20)) + with self.assertRaises(ValueError): + report.section(report.load(directory), None) + + def test_incomplete_table_rejected(self): + with tempfile.TemporaryDirectory() as directory: + for backend in report.BACKENDS: + (Path(directory) / f'rust-100-{backend}.txt').write_text(self.output(backend).rsplit('\n', 1)[0]) + with self.assertRaises(ValueError): + report.load(directory) + + def test_fastest_postgresql_baseline(self): + times = {(op, implementation): (200, 0) + for op in report.OPERATIONS for implementation, _, _ in report.IMPLEMENTATIONS} + times.update({(op, 'PSQL_Transaction'): (100, 0) for op in report.OPERATIONS}) + self.assertIn('2× slower', report.table(times)) + self.assertIn('**100 ns**', report.table(times)) + + +if __name__ == '__main__': + unittest.main() From 823a987c15061b91cb9daf46097b5942e6df6e3a Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:40:05 +0000 Subject: [PATCH 3/6] Test Rust backend parity and isolate configurable PostgreSQL benchmarks --- rust/benches/bench.rs | 39 +++++- rust/benches/benchmarks/doublets/create.rs | 7 +- rust/benches/benchmarks/doublets/delete.rs | 9 +- rust/benches/benchmarks/doublets/each/all.rs | 9 +- .../benchmarks/doublets/each/concrete.rs | 7 +- .../benchmarks/doublets/each/identity.rs | 7 +- .../benchmarks/doublets/each/incoming.rs | 7 +- .../benchmarks/doublets/each/outgoing.rs | 7 +- rust/benches/benchmarks/doublets/update.rs | 7 +- rust/benches/benchmarks/psql/create.rs | 5 +- rust/benches/benchmarks/psql/delete.rs | 10 +- rust/benches/benchmarks/psql/each/all.rs | 9 +- rust/benches/benchmarks/psql/each/concrete.rs | 7 +- rust/benches/benchmarks/psql/each/identity.rs | 7 +- rust/benches/benchmarks/psql/each/incoming.rs | 7 +- rust/benches/benchmarks/psql/each/outgoing.rs | 7 +- rust/benches/benchmarks/psql/update.rs | 8 +- rust/src/benched/doublets_benched.rs | 12 +- rust/src/benched/psql_benched.rs | 4 +- rust/src/client.rs | 125 ++++++++++++------ rust/src/exclusive.rs | 23 ++-- rust/src/fork.rs | 2 +- rust/src/lib.rs | 47 +++++-- rust/src/transaction.rs | 100 +++++++++----- rust/tests/same_behavior.rs | 73 ++++++++++ 25 files changed, 378 insertions(+), 167 deletions(-) create mode 100644 rust/tests/same_behavior.rs diff --git a/rust/benches/bench.rs b/rust/benches/bench.rs index 29e6f96..a663aba 100644 --- a/rust/benches/bench.rs +++ b/rust/benches/bench.rs @@ -6,16 +6,18 @@ use { psql_each_concrete, psql_each_identity, psql_each_incoming, psql_each_outgoing, psql_update_links, }, - criterion::{criterion_group, criterion_main}, + criterion::criterion_group, }; mod benchmarks; macro_rules! tri { ($($body:tt)*) => { - let _ = (|| -> linkspsql::Result<()> { - Ok({ $($body)* }) - })().unwrap(); + let mut run = || -> linkspsql::Result<()> { + { $($body)* }; + Ok(()) + }; + run().unwrap(); }; } @@ -23,7 +25,9 @@ pub(crate) use tri; // PostgreSQL benchmarks criterion_group!( - psql_benches, + name = psql_benches; + config = configuration(); + targets = psql_create_links, psql_delete_links, psql_each_identity, @@ -36,7 +40,9 @@ criterion_group!( // Doublets benchmarks criterion_group!( - doublets_benches, + name = doublets_benches; + config = configuration(); + targets = doublets_create_links, doublets_delete_links, doublets_each_identity, @@ -47,4 +53,23 @@ criterion_group!( doublets_update_links ); -criterion_main!(psql_benches, doublets_benches); +fn configuration() -> criterion::Criterion { + criterion::Criterion::default() + .sample_size(10) + .warm_up_time(std::time::Duration::from_secs(1)) + .measurement_time(std::time::Duration::from_secs(1)) +} + +fn main() { + linkspsql::benchmark_links(); + match std::env::var("BENCHMARK_BACKEND").as_deref().unwrap_or("all") { + "psql" => psql_benches(), + "doublets" => doublets_benches(), + "all" => { + psql_benches(); + doublets_benches(); + } + backend => panic!("unknown BENCHMARK_BACKEND: {backend}"), + } + configuration().configure_from_args().final_summary(); +} diff --git a/rust/benches/benchmarks/doublets/create.rs b/rust/benches/benchmarks/doublets/create.rs index c3535b3..7f278bd 100644 --- a/rust/benches/benchmarks/doublets/create.rs +++ b/rust/benches/benchmarks/doublets/create.rs @@ -12,14 +12,14 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ + Doublets, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; -use linkspsql::{bench, benchmark_links, Benched, Fork}; +use linkspsql::{Benched, Fork, bench, benchmark_links}; use crate::tri; @@ -42,6 +42,7 @@ fn bench>( /// Creates benchmark for Doublets backends on link creation. pub fn create_links(c: &mut Criterion) { let mut group = c.benchmark_group("Create"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench( diff --git a/rust/benches/benchmarks/doublets/delete.rs b/rust/benches/benchmarks/doublets/delete.rs index 3d38fea..aaa427d 100644 --- a/rust/benches/benchmarks/doublets/delete.rs +++ b/rust/benches/benchmarks/doublets/delete.rs @@ -11,14 +11,14 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ + Doublets, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; -use linkspsql::{background_links, bench, benchmark_links, Benched, Fork}; +use linkspsql::{Benched, Fork, background_links, bench, benchmark_links}; use crate::tri; @@ -35,7 +35,7 @@ fn bench>( for _prepare in bg_links..bg_links + links { let _ = fork.create_point(); } - for id in (bg_links..=bg_links + links).rev() { + for id in linkspsql::created_links(bg_links, links).rev() { let _ = elapsed! {fork.delete(id)?}; } })(bencher, &mut benched); @@ -45,6 +45,7 @@ fn bench>( /// Creates benchmark for Doublets backends on link deletion. pub fn delete_links(c: &mut Criterion) { let mut group = c.benchmark_group("Delete"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench( diff --git a/rust/benches/benchmarks/doublets/each/all.rs b/rust/benches/benchmarks/doublets/each/all.rs index dce095d..5b0f96d 100644 --- a/rust/benches/benchmarks/doublets/each/all.rs +++ b/rust/benches/benchmarks/doublets/each/all.rs @@ -10,15 +10,15 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ + Doublets, data::Flow, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; -use linkspsql::{bench, Benched, Fork}; +use linkspsql::{Benched, Fork, bench}; use crate::tri; @@ -31,7 +31,7 @@ fn bench>( let handler = |_| Flow::Continue; group.bench_function(id, |bencher| { bench!(|fork| as B { - let _ = elapsed! { fork.each(handler) }; + elapsed! { fork.each(handler) }; })(bencher, &mut benched); }); } @@ -39,6 +39,7 @@ fn bench>( /// Creates benchmark for Doublets backends on querying all links. pub fn each_all(c: &mut Criterion) { let mut group = c.benchmark_group("Each_All"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench( diff --git a/rust/benches/benchmarks/doublets/each/concrete.rs b/rust/benches/benchmarks/doublets/each/concrete.rs index ea4be54..88709b2 100644 --- a/rust/benches/benchmarks/doublets/each/concrete.rs +++ b/rust/benches/benchmarks/doublets/each/concrete.rs @@ -10,15 +10,15 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ + Doublets, data::{Flow, LinksConstants}, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; -use linkspsql::{background_links, bench, Benched, Fork}; +use linkspsql::{Benched, Fork, background_links, bench}; use crate::tri; @@ -43,6 +43,7 @@ fn bench>( /// Creates benchmark for Doublets backends on querying links by source+target. pub fn each_concrete(c: &mut Criterion) { let mut group = c.benchmark_group("Each_Concrete"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench( diff --git a/rust/benches/benchmarks/doublets/each/identity.rs b/rust/benches/benchmarks/doublets/each/identity.rs index f7a732e..cd44723 100644 --- a/rust/benches/benchmarks/doublets/each/identity.rs +++ b/rust/benches/benchmarks/doublets/each/identity.rs @@ -10,15 +10,15 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ + Doublets, data::{Flow, LinksConstants}, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; -use linkspsql::{background_links, bench, Benched, Fork}; +use linkspsql::{Benched, Fork, background_links, bench}; use crate::tri; @@ -43,6 +43,7 @@ fn bench>( /// Creates benchmark for Doublets backends on querying links by ID. pub fn each_identity(c: &mut Criterion) { let mut group = c.benchmark_group("Each_Identity"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench( diff --git a/rust/benches/benchmarks/doublets/each/incoming.rs b/rust/benches/benchmarks/doublets/each/incoming.rs index 2d8ce36..324a1ff 100644 --- a/rust/benches/benchmarks/doublets/each/incoming.rs +++ b/rust/benches/benchmarks/doublets/each/incoming.rs @@ -10,15 +10,15 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ + Doublets, data::{Flow, LinksConstants}, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; -use linkspsql::{background_links, bench, Benched, Fork}; +use linkspsql::{Benched, Fork, background_links, bench}; use crate::tri; @@ -43,6 +43,7 @@ fn bench>( /// Creates benchmark for Doublets backends on querying links by target. pub fn each_incoming(c: &mut Criterion) { let mut group = c.benchmark_group("Each_Incoming"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench( diff --git a/rust/benches/benchmarks/doublets/each/outgoing.rs b/rust/benches/benchmarks/doublets/each/outgoing.rs index 7e3e235..66cb583 100644 --- a/rust/benches/benchmarks/doublets/each/outgoing.rs +++ b/rust/benches/benchmarks/doublets/each/outgoing.rs @@ -10,15 +10,15 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ + Doublets, data::{Flow, LinksConstants}, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; -use linkspsql::{background_links, bench, Benched, Fork}; +use linkspsql::{Benched, Fork, background_links, bench}; use crate::tri; @@ -43,6 +43,7 @@ fn bench>( /// Creates benchmark for Doublets backends on querying links by source. pub fn each_outgoing(c: &mut Criterion) { let mut group = c.benchmark_group("Each_Outgoing"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench( diff --git a/rust/benches/benchmarks/doublets/update.rs b/rust/benches/benchmarks/doublets/update.rs index deaa137..72eff69 100644 --- a/rust/benches/benchmarks/doublets/update.rs +++ b/rust/benches/benchmarks/doublets/update.rs @@ -11,14 +11,14 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ + Doublets, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; -use linkspsql::{background_links, bench, benchmark_links, Benched, Fork}; +use linkspsql::{Benched, Fork, background_links, bench, benchmark_links}; use crate::tri; @@ -43,6 +43,7 @@ fn bench>( /// Creates benchmark for Doublets backends on link updates. pub fn update_links(c: &mut Criterion) { let mut group = c.benchmark_group("Update"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench( diff --git a/rust/benches/benchmarks/psql/create.rs b/rust/benches/benchmarks/psql/create.rs index 061162a..54d3eaf 100644 --- a/rust/benches/benchmarks/psql/create.rs +++ b/rust/benches/benchmarks/psql/create.rs @@ -15,9 +15,9 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::Doublets; -use linkspsql::{bench, benchmark_links, connect, Benched, Client, Exclusive, Fork, Transaction}; +use linkspsql::{Benched, Client, Exclusive, Fork, Transaction, bench, benchmark_links, connect}; use crate::tri; @@ -40,6 +40,7 @@ fn bench>( /// Creates benchmark for PostgreSQL backends on link creation. pub fn create_links(c: &mut Criterion) { let mut group = c.benchmark_group("Create"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench(&mut group, "PSQL_NonTransaction", Exclusive::>::setup(()).unwrap()); diff --git a/rust/benches/benchmarks/psql/delete.rs b/rust/benches/benchmarks/psql/delete.rs index b3a5174..4bd0cf2 100644 --- a/rust/benches/benchmarks/psql/delete.rs +++ b/rust/benches/benchmarks/psql/delete.rs @@ -15,9 +15,12 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::Doublets; -use linkspsql::{background_links, bench, benchmark_links, connect, Benched, Client, Exclusive, Fork, Transaction}; +use linkspsql::{ + Benched, Client, Exclusive, Fork, Transaction, background_links, bench, benchmark_links, + connect, +}; use crate::tri; @@ -34,7 +37,7 @@ fn bench>( for _prepare in bg_links..bg_links + links { let _ = fork.create_point(); } - for id in (bg_links..=bg_links + links).rev() { + for id in linkspsql::created_links(bg_links, links).rev() { let _ = elapsed! {fork.delete(id)?}; } })(bencher, &mut benched); @@ -44,6 +47,7 @@ fn bench>( /// Creates benchmark for PostgreSQL backends on link deletion. pub fn delete_links(c: &mut Criterion) { let mut group = c.benchmark_group("Delete"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench(&mut group, "PSQL_NonTransaction", Exclusive::>::setup(()).unwrap()); diff --git a/rust/benches/benchmarks/psql/each/all.rs b/rust/benches/benchmarks/psql/each/all.rs index 84a4a34..b2f88c8 100644 --- a/rust/benches/benchmarks/psql/each/all.rs +++ b/rust/benches/benchmarks/psql/each/all.rs @@ -14,9 +14,9 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; -use doublets::{data::Flow, Doublets}; -use linkspsql::{bench, connect, Benched, Client, Exclusive, Fork, Transaction}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; +use doublets::{Doublets, data::Flow}; +use linkspsql::{Benched, Client, Exclusive, Fork, Transaction, bench, connect}; use crate::tri; @@ -29,7 +29,7 @@ fn bench>( let handler = |_| Flow::Continue; group.bench_function(id, |bencher| { bench!(|fork| as B { - let _ = elapsed! { fork.each(handler) }; + elapsed! { fork.each(handler) }; })(bencher, &mut benched); }); } @@ -37,6 +37,7 @@ fn bench>( /// Creates benchmark for PostgreSQL backends on querying all links. pub fn each_all(c: &mut Criterion) { let mut group = c.benchmark_group("Each_All"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench(&mut group, "PSQL_NonTransaction", Exclusive::>::setup(()).unwrap()); diff --git a/rust/benches/benchmarks/psql/each/concrete.rs b/rust/benches/benchmarks/psql/each/concrete.rs index 43b4614..a692059 100644 --- a/rust/benches/benchmarks/psql/each/concrete.rs +++ b/rust/benches/benchmarks/psql/each/concrete.rs @@ -14,12 +14,12 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ - data::{Flow, LinksConstants}, Doublets, + data::{Flow, LinksConstants}, }; -use linkspsql::{background_links, bench, connect, Benched, Client, Exclusive, Fork, Transaction}; +use linkspsql::{Benched, Client, Exclusive, Fork, Transaction, background_links, bench, connect}; use crate::tri; @@ -44,6 +44,7 @@ fn bench>( /// Creates benchmark for PostgreSQL backends on querying links by source+target. pub fn each_concrete(c: &mut Criterion) { let mut group = c.benchmark_group("Each_Concrete"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench(&mut group, "PSQL_NonTransaction", Exclusive::>::setup(()).unwrap()); diff --git a/rust/benches/benchmarks/psql/each/identity.rs b/rust/benches/benchmarks/psql/each/identity.rs index c376df0..396dfdb 100644 --- a/rust/benches/benchmarks/psql/each/identity.rs +++ b/rust/benches/benchmarks/psql/each/identity.rs @@ -14,12 +14,12 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ - data::{Flow, LinksConstants}, Doublets, + data::{Flow, LinksConstants}, }; -use linkspsql::{background_links, bench, connect, Benched, Client, Exclusive, Fork, Transaction}; +use linkspsql::{Benched, Client, Exclusive, Fork, Transaction, background_links, bench, connect}; use crate::tri; @@ -44,6 +44,7 @@ fn bench>( /// Creates benchmark for PostgreSQL backends on querying links by ID. pub fn each_identity(c: &mut Criterion) { let mut group = c.benchmark_group("Each_Identity"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench(&mut group, "PSQL_NonTransaction", Exclusive::>::setup(()).unwrap()); diff --git a/rust/benches/benchmarks/psql/each/incoming.rs b/rust/benches/benchmarks/psql/each/incoming.rs index 02f1b55..5243168 100644 --- a/rust/benches/benchmarks/psql/each/incoming.rs +++ b/rust/benches/benchmarks/psql/each/incoming.rs @@ -14,12 +14,12 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ - data::{Flow, LinksConstants}, Doublets, + data::{Flow, LinksConstants}, }; -use linkspsql::{background_links, bench, connect, Benched, Client, Exclusive, Fork, Transaction}; +use linkspsql::{Benched, Client, Exclusive, Fork, Transaction, background_links, bench, connect}; use crate::tri; @@ -44,6 +44,7 @@ fn bench>( /// Creates benchmark for PostgreSQL backends on querying links by target. pub fn each_incoming(c: &mut Criterion) { let mut group = c.benchmark_group("Each_Incoming"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench(&mut group, "PSQL_NonTransaction", Exclusive::>::setup(()).unwrap()); diff --git a/rust/benches/benchmarks/psql/each/outgoing.rs b/rust/benches/benchmarks/psql/each/outgoing.rs index 60890a3..458e9c5 100644 --- a/rust/benches/benchmarks/psql/each/outgoing.rs +++ b/rust/benches/benchmarks/psql/each/outgoing.rs @@ -14,12 +14,12 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::{ - data::{Flow, LinksConstants}, Doublets, + data::{Flow, LinksConstants}, }; -use linkspsql::{background_links, bench, connect, Benched, Client, Exclusive, Fork, Transaction}; +use linkspsql::{Benched, Client, Exclusive, Fork, Transaction, background_links, bench, connect}; use crate::tri; @@ -44,6 +44,7 @@ fn bench>( /// Creates benchmark for PostgreSQL backends on querying links by source. pub fn each_outgoing(c: &mut Criterion) { let mut group = c.benchmark_group("Each_Outgoing"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench(&mut group, "PSQL_NonTransaction", Exclusive::>::setup(()).unwrap()); diff --git a/rust/benches/benchmarks/psql/update.rs b/rust/benches/benchmarks/psql/update.rs index 81e0208..d429222 100644 --- a/rust/benches/benchmarks/psql/update.rs +++ b/rust/benches/benchmarks/psql/update.rs @@ -15,9 +15,12 @@ use std::time::{Duration, Instant}; -use criterion::{measurement::WallTime, BenchmarkGroup, Criterion}; +use criterion::{BenchmarkGroup, Criterion, measurement::WallTime}; use doublets::Doublets; -use linkspsql::{background_links, bench, benchmark_links, connect, Benched, Client, Exclusive, Fork, Transaction}; +use linkspsql::{ + Benched, Client, Exclusive, Fork, Transaction, background_links, bench, benchmark_links, + connect, +}; use crate::tri; @@ -42,6 +45,7 @@ fn bench>( /// Creates benchmark for PostgreSQL backends on link updates. pub fn update_links(c: &mut Criterion) { let mut group = c.benchmark_group("Update"); + group.sampling_mode(criterion::SamplingMode::Flat); tri! { bench(&mut group, "PSQL_NonTransaction", Exclusive::>::setup(()).unwrap()); diff --git a/rust/src/benched/doublets_benched.rs b/rust/src/benched/doublets_benched.rs index 5e586a8..d3e2a94 100644 --- a/rust/src/benched/doublets_benched.rs +++ b/rust/src/benched/doublets_benched.rs @@ -17,13 +17,13 @@ //! Each backend's `unfork()` implementation calls `delete_all()` to clean up //! all links created during the benchmark iteration. -use crate::{map_file, Result}; +use crate::{Result, map_file}; use doublets::{ + Doublets, data::LinkReference, mem::{FileMapped, Global}, split::{self, DataPart, IndexPart}, unit::{self, LinkPart}, - Doublets, }; use super::Benched; @@ -55,7 +55,9 @@ impl Benched for unit::Store>> { } /// Benched implementation for file-mapped split storage. -impl Benched for split::Store>, FileMapped>> { +impl Benched + for split::Store>, FileMapped>> +{ type Builder<'a> = (&'a str, &'a str); fn setup((data, index): Self::Builder<'_>) -> Result { @@ -68,9 +70,7 @@ impl Benched for split::Store>, File } /// Benched implementation for in-memory split storage. -impl Benched - for split::Store>, Global>> -{ +impl Benched for split::Store>, Global>> { type Builder<'a> = (); fn setup(_: Self::Builder<'_>) -> Result { diff --git a/rust/src/benched/psql_benched.rs b/rust/src/benched/psql_benched.rs index cabbf1c..ba6bc21 100644 --- a/rust/src/benched/psql_benched.rs +++ b/rust/src/benched/psql_benched.rs @@ -38,7 +38,7 @@ impl Benched for Exclusive> { fn setup(_: Self::Builder<'_>) -> Result { // Safety: Exclusive wrapper ensures single-threaded access - unsafe { Ok(Exclusive::new(crate::connect()?)) } + Ok(Exclusive::new(crate::connect()?)) } fn fork(&mut self) -> Fork<'_, Self> { @@ -59,7 +59,7 @@ impl<'a, T: LinkReference> Benched for Exclusive> { let mut transaction = builder.transaction()?; transaction.create_table()?; // Safety: Exclusive wrapper ensures single-threaded access - unsafe { Ok(Exclusive::new(transaction)) } + Ok(Exclusive::new(transaction)) } fn fork(&mut self) -> Fork<'_, Self> { diff --git a/rust/src/client.rs b/rust/src/client.rs index c29827e..62f323c 100644 --- a/rust/src/client.rs +++ b/rust/src/client.rs @@ -1,13 +1,13 @@ use { - crate::{as_i64, transaction::Transaction, Exclusive, Result, Sql}, + crate::{Exclusive, Result, Sql, as_i64, transaction::Transaction}, doublets::{ - data::{Error, Flow, LinkReference, LinksConstants, ReadHandler, WriteHandler}, Doublets, Link, Links, + data::{Error, Flow, LinkReference, LinksConstants, ReadHandler, WriteHandler}, }, }; pub struct Client { - client: postgres::Client, + client: std::sync::Mutex, constants: LinksConstants, } @@ -19,33 +19,38 @@ impl Client { )?; client.query("CREATE INDEX IF NOT EXISTS source ON Links USING btree(from_id);", &[])?; client.query("CREATE INDEX IF NOT EXISTS target ON Links USING btree(to_id);", &[])?; - Ok(Self { client, constants: LinksConstants::new() }) + Ok(Self { client: std::sync::Mutex::new(client), constants: LinksConstants::new() }) } pub fn transaction(&mut self) -> Result> { - Ok(Transaction::new(self.client.transaction()?)) + Ok(Transaction::new(self.client.get_mut().unwrap().transaction()?)) } } impl Sql for Client { fn create_table(&mut self) -> Result<()> { - self.client.query( + self.client.lock().unwrap().query( "CREATE TABLE IF NOT EXISTS Links (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, from_id bigint, to_id bigint);", &[], )?; self.client + .lock() + .unwrap() .query("CREATE INDEX IF NOT EXISTS source ON Links USING btree(from_id);", &[])?; - self.client.query("CREATE INDEX IF NOT EXISTS target ON Links USING btree(to_id);", &[])?; + self.client + .lock() + .unwrap() + .query("CREATE INDEX IF NOT EXISTS target ON Links USING btree(to_id);", &[])?; Ok(()) } fn drop_table(&mut self) -> Result<()> { - self.client.query("DROP TABLE Links;", &[])?; + self.client.lock().unwrap().query("DROP TABLE Links;", &[])?; Ok(()) } } -impl<'c, T: LinkReference> Links for Exclusive> { +impl Links for Exclusive> { fn constants(&self) -> &LinksConstants { &self.constants } @@ -53,7 +58,13 @@ impl<'c, T: LinkReference> Links for Exclusive> { fn count_links(&self, query: &[T]) -> T { let any = self.constants.any; if query.is_empty() { - let result = self.get().client.query("SELECT COUNT(*) FROM Links;", &[]).unwrap(); + let result = self + .get() + .client + .lock() + .unwrap() + .query("SELECT COUNT(*) FROM Links;", &[]) + .unwrap(); let row = &result[0]; row.get::<_, i64>(0).try_into().unwrap() } else if query.len() == 1 { @@ -63,6 +74,8 @@ impl<'c, T: LinkReference> Links for Exclusive> { let result = self .get() .client + .lock() + .unwrap() .query("SELECT COUNT(*) FROM Links WHERE id = $1;", &[&as_i64(query[0])]) .unwrap(); result[0].get::<_, i64>(0).try_into().unwrap() @@ -70,15 +83,18 @@ impl<'c, T: LinkReference> Links for Exclusive> { } else if query.len() == 3 { let id = if query[0] == any { String::new() } else { format!("id = {} AND ", query[0]) }; - let source = - if query[1] == any { String::new() } else { format!("from_id = {} AND", query[1]) }; + let source = if query[1] == any { + String::new() + } else { + format!("from_id = {} AND ", query[1]) + }; let target = if query[2] == any { String::from("true;") } else { format!("to_id = {};", query[2]) }; let statement = format!("SELECT COUNT(*) FROM Links WHERE {}{}{}", id, source, target); - let result = self.get().client.query(&statement, &[]).unwrap(); + let result = self.get().client.lock().unwrap().query(&statement, &[]).unwrap(); result[0].get::<_, i64>(0).try_into().unwrap() } else { panic!("Constraints violation: size of query neither 1 nor 3") @@ -88,24 +104,33 @@ impl<'c, T: LinkReference> Links for Exclusive> { fn create_links(&mut self, _query: &[T], handler: WriteHandler) -> Result> { let result = self .client + .lock() + .unwrap() .query("INSERT INTO Links(to_id, from_id) VALUES (0, 0) RETURNING id;", &[]) .unwrap(); Ok(handler( Link::nothing(), - Link::new(result[0].get::<_, i64>(0).try_into().unwrap(), T::from_byte(0), T::from_byte(0)), + Link::new( + result[0].get::<_, i64>(0).try_into().unwrap(), + T::from_byte(0), + T::from_byte(0), + ), )) } fn each_links(&self, query: &[T], handler: ReadHandler) -> Flow { let any = self.constants.any; if query.is_empty() { - let result = self.get().client.query("SELECT * FROM Links;", &[]).unwrap(); + let result = + self.get().client.lock().unwrap().query("SELECT * FROM Links;", &[]).unwrap(); for row in result { if handler(Link::new( row.get::<_, i64>(0).try_into().unwrap(), row.get::<_, i64>(1).try_into().unwrap(), row.get::<_, i64>(2).try_into().unwrap(), - )).is_break() { + )) + .is_break() + { return Flow::Break; } } @@ -117,14 +142,18 @@ impl<'c, T: LinkReference> Links for Exclusive> { let result = self .get() .client - .query("SELECT * FROM Links WHERE id = %1;", &[&as_i64(query[0])]) + .lock() + .unwrap() + .query("SELECT * FROM Links WHERE id = $1;", &[&as_i64(query[0])]) .unwrap(); for row in result { if handler(Link::new( row.get::<_, i64>(0).try_into().unwrap(), row.get::<_, i64>(1).try_into().unwrap(), row.get::<_, i64>(2).try_into().unwrap(), - )).is_break() { + )) + .is_break() + { return Flow::Break; } } @@ -144,13 +173,15 @@ impl<'c, T: LinkReference> Links for Exclusive> { format!("to_id = {};", query[2]) }; let statement = &format!("SELECT * FROM Links WHERE {id}{source}{target}"); - let result = self.get().client.query(statement, &[]).unwrap(); + let result = self.get().client.lock().unwrap().query(statement, &[]).unwrap(); for row in result { if handler(Link::new( row.get::<_, i64>(0).try_into().unwrap(), row.get::<_, i64>(1).try_into().unwrap(), row.get::<_, i64>(2).try_into().unwrap(), - )).is_break() { + )) + .is_break() + { return Flow::Break; } } @@ -169,16 +200,24 @@ impl<'c, T: LinkReference> Links for Exclusive> { let id = query[0]; let source = change[1]; let target = change[2]; - let old_links = - self.client.query("SELECT * FROM Links WHERE id = $1;", &[&as_i64(id)]).unwrap(); + let old_links = self + .client + .lock() + .unwrap() + .query("SELECT * FROM Links WHERE id = $1;", &[&as_i64(id)]) + .unwrap(); let (old_source, old_target) = ( old_links[0].get::<_, i64>(1).try_into().unwrap(), old_links[0].get::<_, i64>(2).try_into().unwrap(), ); - let _ = self.client.query( - "UPDATE Links SET from_id = $1, to_id = $2 WHERE id = $3;", - &[&as_i64(source), &as_i64(target), &as_i64(id)], - ); + self.client + .lock() + .unwrap() + .query( + "UPDATE Links SET from_id = $1, to_id = $2 WHERE id = $3;", + &[&as_i64(source), &as_i64(target), &as_i64(id)], + ) + .unwrap(); Ok(handler(Link::new(id, old_source, old_target), Link::new(id, source, target))) } @@ -186,6 +225,8 @@ impl<'c, T: LinkReference> Links for Exclusive> { let id = query[0]; let result = self .client + .lock() + .unwrap() .query("DELETE FROM Links WHERE id = $1 RETURNING from_id, to_id", &[&as_i64(id)]) .unwrap(); let row = if result.is_empty() { @@ -204,24 +245,20 @@ impl<'c, T: LinkReference> Links for Exclusive> { } } -impl<'c, T: LinkReference> Doublets for Exclusive> { +impl Doublets for Exclusive> { fn get_link(&self, index: T) -> Option> { - let result = - self.get().client.query("SELECT * FROM Links WHERE id = $1", &[&as_i64(index)]); - match result { - Ok(rows) => { - let row = &rows[0]; - if row.is_empty() { - None - } else { - Some(Link::new( - row.get::<_, i64>(0).try_into().unwrap(), - row.get::<_, i64>(1).try_into().unwrap(), - row.get::<_, i64>(2).try_into().unwrap(), - )) - } - } - Err(_) => None, - } + let result = self + .get() + .client + .lock() + .unwrap() + .query("SELECT * FROM Links WHERE id = $1", &[&as_i64(index)]); + let rows = result.ok()?; + let row = rows.first()?; + Some(Link::new( + row.get::<_, i64>(0).try_into().unwrap(), + row.get::<_, i64>(1).try_into().unwrap(), + row.get::<_, i64>(2).try_into().unwrap(), + )) } } diff --git a/rust/src/exclusive.rs b/rust/src/exclusive.rs index 99c3961..7b49fcc 100644 --- a/rust/src/exclusive.rs +++ b/rust/src/exclusive.rs @@ -1,17 +1,16 @@ -use std::{ - cell::UnsafeCell, - ops::{Deref, DerefMut}, -}; +use std::ops::{Deref, DerefMut}; -pub struct Exclusive(UnsafeCell); +/// Compatibility wrapper around backends whose SQL clients use a mutex. +/// Shared reads lock the client, as required by the Doublets `Send + Sync` API. +pub struct Exclusive(T); impl Exclusive { - pub unsafe fn new(t: T) -> Self { - Exclusive(UnsafeCell::new(t)) + pub fn new(value: T) -> Self { + Self(value) } - pub fn get(&self) -> &mut T { - unsafe { &mut *self.0.get() } + pub fn get(&self) -> &T { + &self.0 } } @@ -19,14 +18,12 @@ impl Deref for Exclusive { type Target = T; fn deref(&self) -> &Self::Target { - self.get() + &self.0 } } impl DerefMut for Exclusive { fn deref_mut(&mut self) -> &mut Self::Target { - self.get() + &mut self.0 } } - -unsafe impl Sync for Exclusive {} diff --git a/rust/src/fork.rs b/rust/src/fork.rs index e9875ef..1764257 100644 --- a/rust/src/fork.rs +++ b/rust/src/fork.rs @@ -21,6 +21,6 @@ impl DerefMut for Fork<'_, B> { impl Drop for Fork<'_, B> { fn drop(&mut self) { - let _ = unsafe { self.unfork() }; + unsafe { self.unfork() }; } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index c65dd42..0d4effa 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -12,7 +12,7 @@ macro_rules! bench { __ret }}; } - crate::tri! { + tri! { let background_links = linkspsql::background_links(); for _iter in 0..iters { let mut $fork: Fork<$B> = Benched::fork(&mut *benched); @@ -50,25 +50,43 @@ pub type Result> = result::Result usize { - env::var("BENCHMARK_BACKGROUND_LINKS") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or(1_000) + size("BENCHMARK_BACKGROUND_LINKS", 1_000) } -/// Number of links to create/update/delete in each benchmark operation. -/// Configurable via BENCHMARK_LINKS environment variable. -/// Default: 100 (for local testing), CI uses 10 for PRs and 100 for main branch. +/// Number of active links; must fit in the background for update benchmarks. pub fn benchmark_links() -> usize { - env::var("BENCHMARK_LINKS") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or(100) + let links = size("BENCHMARK_LINKS", 100); + assert!( + links <= background_links(), + "BENCHMARK_LINKS must not exceed BENCHMARK_BACKGROUND_LINKS" + ); + links } + +fn size(key: &str, default: usize) -> usize { + match env::var(key) { + Ok(value) => value + .parse() + .ok() + .filter(|value| *value > 0) + .unwrap_or_else(|| panic!("{key} must be a positive integer")), + Err(env::VarError::NotPresent) => default, + Err(_) => panic!("{key} must be a positive integer"), + } +} + +/// IDs created after the background links; used by both delete benchmarks. +pub fn created_links(background: usize, links: usize) -> std::ops::RangeInclusive { + background + 1..=background + links +} + const PARAMS: &str = "user=postgres dbname=postgres password=postgres host=localhost port=5432"; pub fn connect() -> Result> { - Client::new(postgres::Client::connect(PARAMS, NoTls)?).map_err(Into::into) + Client::new(postgres::Client::connect( + &env::var("POSTGRES_CONNECTION").unwrap_or_else(|_| PARAMS.to_owned()), + NoTls, + )?) } /// Converts a link value into the `i64` used by PostgreSQL `bigint` columns. @@ -81,7 +99,8 @@ pub fn as_i64(value: T) -> i64 { } pub fn map_file(filename: &str) -> io::Result> { - let file = File::options().create(true).write(true).read(true).open(filename)?; + let file = + File::options().create(true).truncate(false).write(true).read(true).open(filename)?; FileMapped::new(file) } diff --git a/rust/src/transaction.rs b/rust/src/transaction.rs index e2e1764..abc8682 100644 --- a/rust/src/transaction.rs +++ b/rust/src/transaction.rs @@ -1,43 +1,50 @@ use { - crate::{as_i64, Exclusive, Result, Sql}, + crate::{Exclusive, Result, Sql, as_i64}, doublets::{ - data::{Flow, LinkReference, LinksConstants, ReadHandler, WriteHandler}, Doublets, Error, Link, Links, + data::{Flow, LinkReference, LinksConstants, ReadHandler, WriteHandler}, }, }; pub struct Transaction<'a, T: LinkReference> { - transaction: postgres::Transaction<'a>, + transaction: std::sync::Mutex>, constants: LinksConstants, } impl<'a, T: LinkReference> Transaction<'a, T> { pub fn new(transaction: postgres::Transaction<'a>) -> Self { - Self { transaction, constants: LinksConstants::::new() } + Self { + transaction: std::sync::Mutex::new(transaction), + constants: LinksConstants::::new(), + } } pub fn commit(&mut self) -> Result<()> { - self.transaction.execute("COMMIT;", &[])?; - self.transaction.execute("BEGIN;", &[])?; + self.transaction.lock().unwrap().execute("COMMIT;", &[])?; + self.transaction.lock().unwrap().execute("BEGIN;", &[])?; Ok(()) } } impl<'a, T: LinkReference> Sql for Transaction<'a, T> { fn create_table(&mut self) -> Result<()> { - self.transaction.query( + self.transaction.lock().unwrap().query( "CREATE TABLE IF NOT EXISTS Links (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, from_id bigint, to_id bigint);", &[], )?; self.transaction + .lock() + .unwrap() .query("CREATE INDEX IF NOT EXISTS source ON Links USING btree(from_id);", &[])?; self.transaction + .lock() + .unwrap() .query("CREATE INDEX IF NOT EXISTS target ON Links USING btree(to_id);", &[])?; Ok(()) } fn drop_table(&mut self) -> Result<()> { - self.transaction.query("DROP TABLE Links;", &[])?; + self.transaction.lock().unwrap().query("DROP TABLE Links;", &[])?; self.commit() } } @@ -50,7 +57,13 @@ impl Links for Exclusive> { fn count_links(&self, query: &[T]) -> T { let any = self.constants.any; if query.is_empty() { - let result = self.get().transaction.query("SELECT COUNT(*) FROM Links;", &[]).unwrap(); + let result = self + .get() + .transaction + .lock() + .unwrap() + .query("SELECT COUNT(*) FROM Links;", &[]) + .unwrap(); let row = &result[0]; row.get::<_, i64>(0).try_into().unwrap() } else if query.len() == 1 { @@ -60,6 +73,8 @@ impl Links for Exclusive> { let result = self .get() .transaction + .lock() + .unwrap() .query("SELECT COUNT(*) FROM Links WHERE id = $1;", &[&as_i64(query[0])]) .unwrap(); result[0].get::<_, i64>(0).try_into().unwrap() @@ -78,7 +93,7 @@ impl Links for Exclusive> { format!("to_id = {};", query[2]) }; let statement = format!("SELECT COUNT(*) FROM Links WHERE {}{}{}", id, source, target); - let result = self.get().transaction.query(&statement, &[]).unwrap(); + let result = self.get().transaction.lock().unwrap().query(&statement, &[]).unwrap(); result[0].get::<_, i64>(0).try_into().unwrap() } else { panic!("Constraints violation: size of query neither 1 nor 3") @@ -88,24 +103,33 @@ impl Links for Exclusive> { fn create_links(&mut self, _query: &[T], handler: WriteHandler) -> Result> { let result = self .transaction + .lock() + .unwrap() .query("INSERT INTO Links(to_id, from_id) VALUES (0, 0) RETURNING id;", &[]) .unwrap(); Ok(handler( Link::nothing(), - Link::new(result[0].get::<_, i64>(0).try_into().unwrap(), T::from_byte(0), T::from_byte(0)), + Link::new( + result[0].get::<_, i64>(0).try_into().unwrap(), + T::from_byte(0), + T::from_byte(0), + ), )) } fn each_links(&self, query: &[T], handler: ReadHandler) -> Flow { let any = self.constants.any; if query.is_empty() { - let result = self.get().transaction.query("SELECT * FROM Links;", &[]).unwrap(); + let result = + self.get().transaction.lock().unwrap().query("SELECT * FROM Links;", &[]).unwrap(); for row in result { if handler(Link::new( row.get::<_, i64>(0).try_into().unwrap(), row.get::<_, i64>(1).try_into().unwrap(), row.get::<_, i64>(2).try_into().unwrap(), - )).is_break() { + )) + .is_break() + { return Flow::Break; } } @@ -117,6 +141,8 @@ impl Links for Exclusive> { let result = self .get() .transaction + .lock() + .unwrap() .query("SELECT * FROM Links WHERE id = $1;", &[&as_i64(query[0])]) .unwrap(); for row in result { @@ -124,7 +150,9 @@ impl Links for Exclusive> { row.get::<_, i64>(0).try_into().unwrap(), row.get::<_, i64>(1).try_into().unwrap(), row.get::<_, i64>(2).try_into().unwrap(), - )).is_break() { + )) + .is_break() + { return Flow::Break; } } @@ -144,13 +172,15 @@ impl Links for Exclusive> { format!("to_id = {};", query[2]) }; let statement = &format!("SELECT * FROM Links WHERE {id}{source}{target}"); - let result = self.get().transaction.query(statement, &[]).unwrap(); + let result = self.get().transaction.lock().unwrap().query(statement, &[]).unwrap(); for row in result { if handler(Link::new( row.get::<_, i64>(0).try_into().unwrap(), row.get::<_, i64>(1).try_into().unwrap(), row.get::<_, i64>(2).try_into().unwrap(), - )).is_break() { + )) + .is_break() + { return Flow::Break; } } @@ -169,13 +199,19 @@ impl Links for Exclusive> { let id = query[0]; let source = change[1]; let target = change[2]; - let old_links = - self.transaction.query("SELECT * FROM Links WHERE id = $1;", &[&as_i64(id)]).unwrap(); + let old_links = self + .transaction + .lock() + .unwrap() + .query("SELECT * FROM Links WHERE id = $1;", &[&as_i64(id)]) + .unwrap(); let (old_source, old_target) = ( old_links[0].get::<_, i64>(1).try_into().unwrap(), old_links[0].get::<_, i64>(2).try_into().unwrap(), ); self.transaction + .lock() + .unwrap() .query( "UPDATE Links SET from_id = $1, to_id = $2 WHERE id = $3;", &[&as_i64(source), &as_i64(target), &as_i64(id)], @@ -188,6 +224,8 @@ impl Links for Exclusive> { let id = query[0]; let result = self .transaction + .lock() + .unwrap() .query("DELETE FROM Links WHERE id = $1 RETURNING from_id, to_id", &[&as_i64(id)]) .unwrap(); let row = if result.is_empty() { @@ -208,18 +246,18 @@ impl Links for Exclusive> { impl Doublets for Exclusive> { fn get_link(&self, index: T) -> Option> { - let result = - self.get().transaction.query("SELECT * FROM Links WHERE id = $1", &[&as_i64(index)]); - match result { - Ok(rows) => { - let ref row = rows[0]; - Some(Link::new( - row.get::<_, i64>(0).try_into().unwrap(), - row.get::<_, i64>(1).try_into().unwrap(), - row.get::<_, i64>(2).try_into().unwrap(), - )) - } - Err(_) => None, - } + let result = self + .get() + .transaction + .lock() + .unwrap() + .query("SELECT * FROM Links WHERE id = $1", &[&as_i64(index)]); + let rows = result.ok()?; + let row = rows.first()?; + Some(Link::new( + row.get::<_, i64>(0).try_into().unwrap(), + row.get::<_, i64>(1).try_into().unwrap(), + row.get::<_, i64>(2).try_into().unwrap(), + )) } } diff --git a/rust/tests/same_behavior.rs b/rust/tests/same_behavior.rs new file mode 100644 index 0000000..d10127a --- /dev/null +++ b/rust/tests/same_behavior.rs @@ -0,0 +1,73 @@ +use doublets::{Doublets, data::Flow, mem::Global, split, unit}; +use linkspsql::{Benched, Client, Exclusive, Sql, Transaction}; + +fn rows(store: &impl Doublets, query: &[usize]) -> Vec<(usize, usize, usize)> { + let mut rows = Vec::new(); + store.each_links(query, &mut |link| { + rows.push((link.index, link.source, link.target)); + Flow::Continue + }); + rows.sort_unstable(); + rows +} + +fn exercise(store: &mut impl Doublets) { + let any = store.constants().any; + let first = store.create_point().unwrap(); + let second = store.create_point().unwrap(); + assert_eq!((first, second), (1, 2)); + assert_eq!(store.count(), 2); + assert_eq!(rows(store, &[]), vec![(1, 1, 1), (2, 2, 2)]); + assert_eq!(rows(store, &[first]), vec![(1, 1, 1)]); + assert!(store.get_link(999).is_none()); + for query in [[any, second, second], [any, second, any], [any, any, second]] { + assert_eq!(rows(store, &query), vec![(2, 2, 2)]); + assert_eq!(store.count_by(query), 1); + } + store.update(second, 0, 0).unwrap(); + assert_eq!(rows(store, &[second, any, any]), vec![(2, 0, 0)]); + store.update(second, second, second).unwrap(); + let third = store.create_point().unwrap(); + let fourth = store.create_point().unwrap(); + assert_eq!((third, fourth), (3, 4)); + for id in linkspsql::created_links(2, 2).rev() { + store.delete(id).unwrap(); + } + assert_eq!(rows(store, &[]), vec![(1, 1, 1), (2, 2, 2)]); + store.delete(second).unwrap(); + assert_eq!(rows(store, &[]), vec![(1, 1, 1)]); + store.delete(first).unwrap(); + assert_eq!(store.count(), 0); +} + +#[test] +fn doublets_same_behavior() { + exercise(&mut unit::Store::>::setup(()).unwrap()); + exercise(&mut split::Store::, Global<_>>::setup(()).unwrap()); + let directory = std::env::temp_dir().join(format!("linkspsql-test-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let united = directory.join("united.links"); + exercise( + &mut unit::Store::>::setup(united.to_str().unwrap()) + .unwrap(), + ); + let data = directory.join("data.links"); + let index = directory.join("index.links"); + exercise(&mut split::Store::, doublets::mem::FileMapped<_>>::setup((data.to_str().unwrap(), index.to_str().unwrap())).unwrap()); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +#[ignore = "requires a disposable PostgreSQL database; set POSTGRES_CONNECTION"] +fn postgresql_same_behavior() { + let mut client = Exclusive::>::new(linkspsql::connect().unwrap()); + client.drop_table().unwrap(); + client.create_table().unwrap(); + exercise(&mut client); + client.drop_table().unwrap(); + client.create_table().unwrap(); + let mut connection = linkspsql::connect().unwrap(); + let mut transaction = Exclusive::>::setup(&mut connection).unwrap(); + exercise(&mut transaction); + transaction.drop_table().unwrap(); +} From 03e31531cc09df212c1ceb6a1eaf598b5de75cf4 Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:42:38 +0000 Subject: [PATCH 4/6] Add Npgsql and Doublets C# benchmarks with parity and harness tests --- .../PostgreSQLVSDoublets.Tests.csproj | 14 + .../SameBehaviorTests.cs | 108 +++++++ .../packages.lock.json | 296 ++++++++++++++++++ csharp/PostgreSQLVSDoublets.slnx | 4 + csharp/PostgreSQLVSDoublets/Benchmarks.cs | 74 +++++ csharp/PostgreSQLVSDoublets/DoubletsLinks.cs | 95 ++++++ csharp/PostgreSQLVSDoublets/Harness.cs | 93 ++++++ csharp/PostgreSQLVSDoublets/IBenchedLinks.cs | 15 + .../PostgreSQLVSDoublets.csproj | 12 + csharp/PostgreSQLVSDoublets/PostgresLinks.cs | 124 ++++++++ csharp/PostgreSQLVSDoublets/Program.cs | 55 ++++ .../PostgreSQLVSDoublets/packages.lock.json | 200 ++++++++++++ csharp/global.json | 1 + 13 files changed, 1091 insertions(+) create mode 100644 csharp/PostgreSQLVSDoublets.Tests/PostgreSQLVSDoublets.Tests.csproj create mode 100644 csharp/PostgreSQLVSDoublets.Tests/SameBehaviorTests.cs create mode 100644 csharp/PostgreSQLVSDoublets.Tests/packages.lock.json create mode 100644 csharp/PostgreSQLVSDoublets.slnx create mode 100644 csharp/PostgreSQLVSDoublets/Benchmarks.cs create mode 100644 csharp/PostgreSQLVSDoublets/DoubletsLinks.cs create mode 100644 csharp/PostgreSQLVSDoublets/Harness.cs create mode 100644 csharp/PostgreSQLVSDoublets/IBenchedLinks.cs create mode 100644 csharp/PostgreSQLVSDoublets/PostgreSQLVSDoublets.csproj create mode 100644 csharp/PostgreSQLVSDoublets/PostgresLinks.cs create mode 100644 csharp/PostgreSQLVSDoublets/Program.cs create mode 100644 csharp/PostgreSQLVSDoublets/packages.lock.json create mode 100644 csharp/global.json diff --git a/csharp/PostgreSQLVSDoublets.Tests/PostgreSQLVSDoublets.Tests.csproj b/csharp/PostgreSQLVSDoublets.Tests/PostgreSQLVSDoublets.Tests.csproj new file mode 100644 index 0000000..38cf3b0 --- /dev/null +++ b/csharp/PostgreSQLVSDoublets.Tests/PostgreSQLVSDoublets.Tests.csproj @@ -0,0 +1,14 @@ + + + net10.0 + enable + enable + true + + + + + + + + diff --git a/csharp/PostgreSQLVSDoublets.Tests/SameBehaviorTests.cs b/csharp/PostgreSQLVSDoublets.Tests/SameBehaviorTests.cs new file mode 100644 index 0000000..03466ca --- /dev/null +++ b/csharp/PostgreSQLVSDoublets.Tests/SameBehaviorTests.cs @@ -0,0 +1,108 @@ +using Comparisons.PostgreSQLVSDoublets; +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] + +public class SameBehaviorTests +{ + private static List Rows(IBenchedLinks store, ulong id, ulong source, ulong target) + { + var rows = new List(); + store.Each(id, source, target, rows.Add); + return rows.OrderBy(link => link.Id).ToList(); + } + + private static void Exercise(IBenchedLinks store) + { + store.Fork(2); + try + { + var any = store.Any; + Assert.Equal(2UL, store.Count()); + Assert.Equal([new Link(1, 1, 1), new Link(2, 2, 2)], Rows(store, any, any, any)); + Assert.Equal([new Link(1, 1, 1)], Rows(store, 1, any, any)); + Assert.Empty(Rows(store, 999, any, any)); + foreach (var query in new[] { (any, 2UL, 2UL), (any, 2UL, any), (any, any, 2UL) }) + { + Assert.Equal([new Link(2, 2, 2)], Rows(store, query.Item1, query.Item2, query.Item3)); + } + store.Update(2, 0, 0); + Assert.Equal([new Link(2, 0, 0)], Rows(store, 2, any, any)); + store.Update(2, 2, 2); + var created = store.CreatePoint(); + Assert.Equal(3UL, created); + store.Delete(created); + Assert.Equal(2UL, store.Count()); + } + finally + { + store.Unfork(); + } + store.Fork(1); + Assert.Equal([new Link(1, 1, 1)], Rows(store, store.Any, store.Any, store.Any)); + store.Unfork(); + } + + [Theory] + [InlineData("Doublets_United_Volatile")] + [InlineData("Doublets_United_NonVolatile")] + [InlineData("Doublets_Split_Volatile")] + [InlineData("Doublets_Split_NonVolatile")] + public void Doublets(string variant) + { + var directory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(directory); + try + { + using var store = DoubletsLinks.Open(variant, directory); + Exercise(store); + foreach (var operation in Benchmarks.Operations) + { + Benchmarks.Iteration(store, operation, 10, 2); + Assert.Equal(0UL, store.Count()); + } + } + finally + { + Directory.Delete(directory, true); + } + } + + [PostgresTheory] + [InlineData(false)] + [InlineData(true)] + public void Postgres(bool transactional) + { + var connection = Environment.GetEnvironmentVariable("NPGSQL_CONNECTION"); + using var store = new PostgresLinks(connection!, transactional); + Exercise(store); + foreach (var operation in Benchmarks.Operations) + { + Benchmarks.Iteration(store, operation, 10, 2); + } + } + + [Fact] + public void StatisticsAndSampling() + { + var estimate = Estimate.Of([1, 2, 3, 4]); + Assert.Equal(2.5, estimate.Median); + Assert.Equal(Math.Sqrt(5.0 / 3), estimate.StandardDeviation, 8); + Assert.Equal(Enumerable.Repeat(10L, 10), new Sampling(10, true, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)).IterationCounts(10_000_000)); + var measured = Harness.Measure(new Sampling(10, true, TimeSpan.FromTicks(1), TimeSpan.FromTicks(1)), () => TimeSpan.FromTicks(2)); + Assert.Equal(200, measured.Median); + Assert.Equal(0, measured.StandardDeviation); + Assert.Contains("bench: 3 ns/iter", new Estimate(3, 0).Bencher("Create", "PSQL_Transaction")); + } +} + +public sealed class PostgresTheoryAttribute : TheoryAttribute +{ + public PostgresTheoryAttribute() + { + if (Environment.GetEnvironmentVariable("NPGSQL_CONNECTION") is null) + { + Skip = "Set NPGSQL_CONNECTION to a disposable PostgreSQL database"; + } + } +} diff --git a/csharp/PostgreSQLVSDoublets.Tests/packages.lock.json b/csharp/PostgreSQLVSDoublets.Tests/packages.lock.json new file mode 100644 index 0000000..89bb1ce --- /dev/null +++ b/csharp/PostgreSQLVSDoublets.Tests/packages.lock.json @@ -0,0 +1,296 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Microsoft.NET.Test.Sdk": { + "type": "Direct", + "requested": "[17.14.1, )", + "resolved": "17.14.1", + "contentHash": "HJKqKOE+vshXra2aEHpi2TlxYX7Z9VFYkr+E5rwEvHC8eIXiyO+K9kNm8vmNom3e2rA56WqxU+/N9NJlLGXsJQ==", + "dependencies": { + "Microsoft.CodeCoverage": "17.14.1", + "Microsoft.TestPlatform.TestHost": "17.14.1" + } + }, + "xunit": { + "type": "Direct", + "requested": "[2.9.3, )", + "resolved": "2.9.3", + "contentHash": "TlXQBinK35LpOPKHAqbLY4xlEen9TBafjs0V5KnA4wZsoQLQJiirCR4CbIXvOH8NzkW4YeJKP5P/Bnrodm0h9Q==", + "dependencies": { + "xunit.analyzers": "1.18.0", + "xunit.assert": "2.9.3", + "xunit.core": "[2.9.3]" + } + }, + "xunit.runner.visualstudio": { + "type": "Direct", + "requested": "[3.1.4, )", + "resolved": "3.1.4", + "contentHash": "5mj99LvCqrq3CNi06xYdyIAXOEh+5b33F2nErCzI5zWiDdLHXiPXEWFSUAF8zlIv0ZWqjZNCwHTQeAPYbF3pCg==" + }, + "Microsoft.CodeCoverage": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "pmTrhfFIoplzFVbhVwUquT+77CbGH+h4/3mBpdmIlYtBi9nAB+kKI6dN3A/nV4DFi3wLLx/BlHIPK+MkbQ6Tpg==" + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "L3AdmZ1WOK4XXT5YFPEwyt0ep6l8lGIPs7F5OOBZc77Zqeo01Of7XXICy47628sdVl0v/owxYJTe86DTgFwKCA==" + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "FU/IfjDfwaMuKr414SSQNTIti/69bHEMb+QKrskRb26oVqpx3lNFXMjs/RC9ZUuhBhcwDM2BwOgoMw+PZ+beqQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0" + } + }, + "Microsoft.TestPlatform.ObjectModel": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "xTP1W6Mi6SWmuxd3a+jj9G9UoC850WGwZUps1Wah9r1ZxgXhdJfj1QqDLJkFjHDCvN42qDL2Ps5KjQYWUU0zcQ==" + }, + "Microsoft.TestPlatform.TestHost": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "d78LPzGKkJwsJXAQwsbJJ7LE7D1wB+rAyhHHAaODF+RDSQ0NgMjDFkSA1Djw18VrxO76GlKAjRUhl+H8NL8Z+Q==", + "dependencies": { + "Microsoft.TestPlatform.ObjectModel": "17.14.1", + "Newtonsoft.Json": "13.0.3" + } + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.3", + "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" + }, + "Npgsql": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "xZAYhPOU2rUIFpV48xsqhCx9vXs6Y+0jX2LCoSEfDFYMw9jtAOUk3iQsCnDLrFIv9NT3JGMihn7nnuZsPKqJmA==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0" + } + }, + "Platform.Collections": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "ofU+XTf0CJry3dnpSq5pdiz8cArZDlrd3IaGtYoV8L6e+OhTdx2n544s+XLOZK90KJUsNv/docu2Y97Asdbvuw==", + "dependencies": { + "Platform.Disposables": "0.4.0", + "Platform.Interfaces": "0.5.0", + "Platform.Random": "0.3.0", + "Platform.Ranges": "0.5.0" + } + }, + "Platform.Collections.Methods": { + "type": "Transitive", + "resolved": "0.7.3", + "contentHash": "GG+XkIiZWlg96w9HxBeCeGrWdXr6+ST8L8piPjvC4lVrB0XaJNWWj+FYSnQOcX7MZOXs70Y6+e5eD0t1GygB5g==", + "dependencies": { + "Platform.Numbers": "0.9.0" + } + }, + "Platform.Converters": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "Rz4gLL2zPB/KsqORljmhMaQR8EVcTBnRRO2rz91F/nu+qJXuzpurx4VgtnL0lyso0NfkmNJffTiLucxwYX9pWQ==", + "dependencies": { + "Platform.Reflection": "0.7.0" + } + }, + "Platform.Data": { + "type": "Transitive", + "resolved": "0.16.1", + "contentHash": "1JdpgdWbowNgVx+aJatRyTEL5CS+hB5mjOJD57+R26XfbCpIsscJwL+Kcmk/ivaop6OABd3SX7M2DaAhYHXKtw==", + "dependencies": { + "Platform.Delegates": "0.1.1", + "Platform.Numbers": "0.9.0", + "Platform.Setters": "0.5.0", + "Platform.Threading": "0.4.0" + } + }, + "Platform.Data.Doublets": { + "type": "Transitive", + "resolved": "0.18.1", + "contentHash": "b6p7GgLDonEnRdhuy3Xsn/N5g5Qq+aivapAlkFH/Mk/MDTDiSJ0C1wnViC4Ff9/d6IXfmh2Sv/TeGIkyKbTOTw==", + "dependencies": { + "Platform.Collections.Methods": "0.7.3", + "Platform.Data": "0.16.1", + "Platform.Data.Doublets.FFI": "0.9.0", + "Platform.Memory": "0.4.1", + "Platform.Random": "0.3.0", + "Platform.Singletons": "0.3.0", + "Platform.Timestamps": "0.3.0" + } + }, + "Platform.Data.Doublets.FFI": { + "type": "Transitive", + "resolved": "0.9.0", + "contentHash": "NRgGyihV3sdmvNW9SLnwBV14zkq89GJlTgyeiB+EKT2cMOkCjZtd55C/M0Sy5tLkZ076rBcup4jLFKmejnS2tg==" + }, + "Platform.Delegates": { + "type": "Transitive", + "resolved": "0.1.1", + "contentHash": "+182klZgGEJ+yWtxz3BMrTCYcpSXjvJzrYzpK+W2UecAaDKLvLyehjPlMfta+dM/XlKbPt53Wf20vhI1kul0SQ==" + }, + "Platform.Disposables": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "ko0W4cw5XhNxJANdO1T8srpWRVqpPiXtCohumlbTu33kivRBEmk1ivDsySaNhKTyuYjJnrjRYrgA1iEb2sS+/Q==", + "dependencies": { + "Platform.Exceptions": "0.5.0" + } + }, + "Platform.Exceptions": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "htu3iyPVJinbLxU1I1cpc2qFlur7IF2fTc9u2jqQ+3X1Lyzd3aeK9Y7gJu9ZkdnIO3tbdO6wxr1HZG4UhaIG8A==" + }, + "Platform.Interfaces": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "OBViyjgCYWaXH8xhibYL/fbwHfPP/+qVUDxcFGH22IaYzbrFbfNR5TxW6gsk3LR9QXLANnrLsuI7phZueoj/Tw==" + }, + "Platform.IO": { + "type": "Transitive", + "resolved": "0.5.1", + "contentHash": "MZ+uZIrDZuNZZu8J8gv8RcXwGihSTEo9UvFoRHCGIXQnaQxnJ0gBIF5s/zSjJ1+h8c78lv/wmlSvh4zMzwyJIQ==", + "dependencies": { + "Platform.Threading": "0.5.0", + "Platform.Unsafe": "0.6.1" + } + }, + "Platform.Memory": { + "type": "Transitive", + "resolved": "0.4.1", + "contentHash": "Jz4Xb0B8pmh6BU9hhvqMgK2L+R0fCxSk1bGWL7WK/WYKAYSuGaJUAvzWdJLTECyfs+w6PVUxXK53q1OXxHRmMQ==", + "dependencies": { + "Platform.IO": "0.5.1" + } + }, + "Platform.Numbers": { + "type": "Transitive", + "resolved": "0.9.0", + "contentHash": "0GgZSH0p+jP933DKtLzCfOAdkJ1cNKrSrisXgMKy0zTMBZOxMxzs83IVXwBju66vm5N/w6lFq8HwCLW0gQsYVA==", + "dependencies": { + "Platform.Converters": "0.4.0" + } + }, + "Platform.Random": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "GX24Xx45cUIr9EZl55evdwb7oYN/nimpoNqDG4zoMDDLqeEpL35zX8IiHHDYITV9F4bjrCrSEiRCVsRFAsi7SQ==", + "dependencies": { + "Platform.Ranges": "0.5.0" + } + }, + "Platform.Ranges": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "CUhZM/4F714NAyrtWtkBx78L4xJpFZ1wp+6ivFvpAXx6DsaBcBsyDxjn5eoLB9LSUYxB11gFkwwmXmJhMgZ3qw==", + "dependencies": { + "Platform.Exceptions": "0.5.0" + } + }, + "Platform.Reflection": { + "type": "Transitive", + "resolved": "0.7.0", + "contentHash": "CvpRnDWH5YArcXlY7lcY0mDYQCtnJAeZsqZ4/IdllJoj7z8JwTufeo/zK62Vn6K0ulTyHNhb+/T3HFusxnrfxA==", + "dependencies": { + "Platform.Collections": "0.4.0" + } + }, + "Platform.Setters": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "vZG+VnYc4u2x5vnBY7rqaFHtOS2AeoLwGV+02JBFa9pxluHwVTu4nVcVBTJTlenZSfAoZUNVsV1StnUWvINkCQ==", + "dependencies": { + "Platform.Delegates": "0.1.1", + "Platform.Interfaces": "0.5.0" + } + }, + "Platform.Singletons": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "fCfNw4CqPL6HzsUJVVYI/OLYXGkjmdDCe+QWThW8hPoNNlqMZ+FWbfBXbXpCuDYw9EYhLDlhuTAQMbqNt7kCzg==", + "dependencies": { + "Platform.Reflection": "0.7.0" + } + }, + "Platform.Threading": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "Q0p5oVDbhQ76IGHi/vYIX+de9T270XZmNgjNbHtobc+AoJbIWGf+kZXTzq+MmEfXS2ANtGNe9uWN1a7Nzp7olQ==", + "dependencies": { + "Platform.Collections": "0.4.0" + } + }, + "Platform.Timestamps": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "Lmhm8Vp0DobTlxSVTPZZE+3ABTrvuofcmTQOWZv6TyiZ5NyFBH6m61vZdnIkMa/3lQguKvGfgMsUT97p0t5x7w==", + "dependencies": { + "Platform.Interfaces": "0.5.0" + } + }, + "Platform.Unsafe": { + "type": "Transitive", + "resolved": "0.6.1", + "contentHash": "GbKgPC7lfw+q2GqQEch0NSxHOZXxoLTj3ChjdRZIQBvZnGWzYH6nat8glfTJseOmgdabIxlwogFECoai0O3akA==", + "dependencies": { + "Platform.Numbers": "0.7.0" + } + }, + "xunit.abstractions": { + "type": "Transitive", + "resolved": "2.0.3", + "contentHash": "pot1I4YOxlWjIb5jmwvvQNbTrZ3lJQ+jUGkGjWE3hEFM0l5gOnBWS+H3qsex68s5cO52g+44vpGzhAt+42vwKg==" + }, + "xunit.analyzers": { + "type": "Transitive", + "resolved": "1.18.0", + "contentHash": "OtFMHN8yqIcYP9wcVIgJrq01AfTxijjAqVDy/WeQVSyrDC1RzBWeQPztL49DN2syXRah8TYnfvk035s7L95EZQ==" + }, + "xunit.assert": { + "type": "Transitive", + "resolved": "2.9.3", + "contentHash": "/Kq28fCE7MjOV42YLVRAJzRF0WmEqsmflm0cfpMjGtzQ2lR5mYVj1/i0Y8uDAOLczkL3/jArrwehfMD0YogMAA==" + }, + "xunit.core": { + "type": "Transitive", + "resolved": "2.9.3", + "contentHash": "BiAEvqGvyme19wE0wTKdADH+NloYqikiU0mcnmiNyXaF9HyHmE6sr/3DC5vnBkgsWaE6yPyWszKSPSApWdRVeQ==", + "dependencies": { + "xunit.extensibility.core": "[2.9.3]", + "xunit.extensibility.execution": "[2.9.3]" + } + }, + "xunit.extensibility.core": { + "type": "Transitive", + "resolved": "2.9.3", + "contentHash": "kf3si0YTn2a8J8eZNb+zFpwfoyvIrQ7ivNk5ZYA5yuYk1bEtMe4DxJ2CF/qsRgmEnDr7MnW1mxylBaHTZ4qErA==", + "dependencies": { + "xunit.abstractions": "2.0.3" + } + }, + "xunit.extensibility.execution": { + "type": "Transitive", + "resolved": "2.9.3", + "contentHash": "yMb6vMESlSrE3Wfj7V6cjQ3S4TXdXpRqYeNEI3zsX31uTsGMJjEw6oD5F5u1cHnMptjhEECnmZSsPxB6ChZHDQ==", + "dependencies": { + "xunit.extensibility.core": "[2.9.3]" + } + }, + "postgresqlvsdoublets": { + "type": "Project", + "dependencies": { + "Npgsql": "[10.0.0, )", + "Platform.Data.Doublets": "[0.18.1, )" + } + } + } + } +} \ No newline at end of file diff --git a/csharp/PostgreSQLVSDoublets.slnx b/csharp/PostgreSQLVSDoublets.slnx new file mode 100644 index 0000000..8382062 --- /dev/null +++ b/csharp/PostgreSQLVSDoublets.slnx @@ -0,0 +1,4 @@ + + + + diff --git a/csharp/PostgreSQLVSDoublets/Benchmarks.cs b/csharp/PostgreSQLVSDoublets/Benchmarks.cs new file mode 100644 index 0000000..473b006 --- /dev/null +++ b/csharp/PostgreSQLVSDoublets/Benchmarks.cs @@ -0,0 +1,74 @@ +using System.Diagnostics; + +namespace Comparisons.PostgreSQLVSDoublets; + +/// Matches the operations and timed boundaries in rust/benches/benchmarks. +public static class Benchmarks +{ + public static readonly string[] Operations = + ["Create", "Update", "Delete", "Each_All", "Each_Identity", "Each_Concrete", "Each_Outgoing", "Each_Incoming"]; + + public static TimeSpan Iteration(IBenchedLinks store, string operation, int background, int links) + { + var elapsed = TimeSpan.Zero; + void Timed(Action action) + { + var started = Stopwatch.GetTimestamp(); + action(); + elapsed += Stopwatch.GetElapsedTime(started); + } + static void Visit(Link link) { } + store.Fork(background); + try + { + switch (operation) + { + case "Create": + for (var i = 0; i < links; i++) + { + Timed(() => store.CreatePoint()); + } + break; + case "Update": + for (var id = (ulong)(background - links + 1); id <= (ulong)background; id++) + { + Timed(() => store.Update(id, 0, 0)); + Timed(() => store.Update(id, id, id)); + } + break; + case "Delete": + for (var i = 0; i < links; i++) + { + store.CreatePoint(); + } + for (var id = (ulong)(background + links); id > (ulong)background; id--) + { + Timed(() => store.Delete(id)); + } + break; + case "Each_All": + Timed(() => store.Each(store.Any, store.Any, store.Any, Visit)); + break; + default: + for (ulong id = 1; id <= (ulong)background; id++) + { + var query = operation switch + { + "Each_Identity" => (id, store.Any, store.Any), + "Each_Concrete" => (store.Any, id, id), + "Each_Outgoing" => (store.Any, id, store.Any), + "Each_Incoming" => (store.Any, store.Any, id), + _ => throw new ArgumentException($"Unknown operation {operation}"), + }; + Timed(() => store.Each(query.Item1, query.Item2, query.Item3, Visit)); + } + break; + } + return elapsed; + } + finally + { + store.Unfork(); + } + } +} diff --git a/csharp/PostgreSQLVSDoublets/DoubletsLinks.cs b/csharp/PostgreSQLVSDoublets/DoubletsLinks.cs new file mode 100644 index 0000000..e06daff --- /dev/null +++ b/csharp/PostgreSQLVSDoublets/DoubletsLinks.cs @@ -0,0 +1,95 @@ +using Platform.Data; +using Platform.Data.Doublets; +using Platform.Data.Doublets.Memory.Split.Generic; +using Platform.Data.Doublets.Memory.United.Generic; +using Platform.Memory; + +namespace Comparisons.PostgreSQLVSDoublets; + +/// +/// A Doublets store of Platform.Data.Doublets. +/// The store runs inside the benchmark process: every operation reads and writes the store's memory directly, +/// without a server, a network connection or a query language. +/// +/// +/// +/// Doublets_United_VolatileUnitedMemoryLinks in RAM +/// Doublets_United_NonVolatileUnitedMemoryLinks in a memory-mapped file +/// Doublets_Split_VolatileSplitMemoryLinks in RAM +/// Doublets_Split_NonVolatileSplitMemoryLinks in two memory-mapped files +/// +/// Writes to a memory-mapped file go to the OS page cache; the benchmarks do not flush them to the disk. +/// +public sealed class DoubletsLinks(ILinks links) : IBenchedLinks +{ + public static readonly string[] Variants = + [ + "Doublets_United_Volatile", + "Doublets_United_NonVolatile", + "Doublets_Split_Volatile", + "Doublets_Split_NonVolatile", + ]; + + private readonly ulong _continue = links.Constants.Continue; + + public ulong Any { get; } = links.Constants.Any; + + /// Opens the store of ; the non-volatile stores are kept in . + public static DoubletsLinks Open(string variant, string directory) + { + static FileMappedResizableDirectMemory Mapped(string directory, string name) + { + var path = Path.Combine(directory, name); + File.Delete(path); + return new FileMappedResizableDirectMemory(path); + } + + ILinks links = variant switch + { + "Doublets_United_Volatile" => new UnitedMemoryLinks(new HeapResizableDirectMemory()), + "Doublets_United_NonVolatile" => new UnitedMemoryLinks(Mapped(directory, "united.links")), + "Doublets_Split_Volatile" => new SplitMemoryLinks(new HeapResizableDirectMemory(), new HeapResizableDirectMemory()), + "Doublets_Split_NonVolatile" => new SplitMemoryLinks(Mapped(directory, "split_data.links"), Mapped(directory, "split_index.links")), + _ => throw new ArgumentException($"unknown Doublets store {variant}", nameof(variant)), + }; + return new DoubletsLinks(links); + } + + public void Fork(int backgroundLinks) + { + for (var i = 0; i < backgroundLinks; i++) + { + links.CreatePoint(); + } + } + + /// Deletes the links from the highest id down, so the next created link gets id 1 again. + public void Unfork() + { + var ids = new List(); + Each(Any, Any, Any, link => ids.Add(link.Id)); + foreach (var id in ids.OrderDescending()) + { + Delete(id); + } + } + + public ulong CreatePoint() => links.CreatePoint(); + + public void Update(ulong id, ulong source, ulong target) => links.Update(id, source, target); + + /// + /// Resets the link to (0, 0), which removes it from the index trees, and then frees it. + /// + public void Delete(ulong id) => links.Delete(id, handler: null); + + public void Each(ulong id, ulong source, ulong target, Action visit) => links.Each(link => + { + visit(new Link(link![0], link[1], link[2])); + return _continue; + }, id, source, target); + + public ulong Count() => links.Count(); + + public void Dispose() => (links as IDisposable)?.Dispose(); +} diff --git a/csharp/PostgreSQLVSDoublets/Harness.cs b/csharp/PostgreSQLVSDoublets/Harness.cs new file mode 100644 index 0000000..bb225a8 --- /dev/null +++ b/csharp/PostgreSQLVSDoublets/Harness.cs @@ -0,0 +1,93 @@ +using System.Diagnostics; +using System.Globalization; + +namespace Comparisons.PostgreSQLVSDoublets; + +/// +/// How a benchmark is sampled. The values are the ones of the Rust benchmarks (rust/benches/bench.rs), +/// and the iteration counts are chosen as Criterion 0.4 chooses them, so both languages measure the same way. +/// +/// Number of samples. +/// Every sample has the same number of iterations (Criterion's flat sampling); +/// otherwise sample i has i × d iterations (linear sampling). +/// How long the benchmark runs before it is measured. +/// The measured time that all samples take together. +public sealed record Sampling(int SampleSize, bool Flat, TimeSpan WarmUp, TimeSpan MeasurementTime) +{ + /// One PostgreSQL iteration takes milliseconds, so 10 samples of the same number of iterations already give stable results. + public static Sampling PostgreSQL { get; } = new(10, true, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); + + /// One Doublets iteration takes microseconds, so the same flat sampling is used. + public static Sampling Doublets { get; } = new(10, true, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); + + /// The number of iterations of every sample, for iterations that take each. + public long[] IterationCounts(double nanoseconds) + { + var measurement = MeasurementTime.TotalNanoseconds; + if (Flat) + { + var iterations = Math.Max(1, (long)Math.Ceiling(measurement / (nanoseconds * SampleSize))); + return Enumerable.Repeat(iterations, SampleSize).ToArray(); + } + var totalRuns = (double)SampleSize * (SampleSize + 1) / 2; + var step = Math.Max(1, (long)Math.Ceiling(measurement / nanoseconds / totalRuns)); + return Enumerable.Range(1, SampleSize).Select(sample => sample * step).ToArray(); + } +} + +/// The median time of one iteration and the standard deviation of the samples, in nanoseconds. +public readonly record struct Estimate(double Median, double StandardDeviation) +{ + /// Estimates the time of one iteration from the average iteration time of every sample. + public static Estimate Of(IReadOnlyCollection samples) + { + var sorted = samples.Order().ToArray(); + var middle = sorted.Length / 2; + var median = sorted.Length % 2 == 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; + var mean = sorted.Average(); + var variance = sorted.Length > 1 ? sorted.Sum(sample => (sample - mean) * (sample - mean)) / (sorted.Length - 1) : 0; + return new Estimate(median, Math.Sqrt(variance)); + } + + /// + /// The line that Criterion prints with --output-format bencher, for example + /// test Create/PSQL_Transaction ... bench: 44,055,505 ns/iter (+/- 5,345,991). + /// + public string Bencher(string group, string id) => + string.Create(CultureInfo.InvariantCulture, + $"test {group}/{id} ... bench: {Math.Round(Median),11:N0} ns/iter (+/- {Math.Round(StandardDeviation):N0})"); +} + +public static class Harness +{ + /// + /// Measures , which runs one iteration and returns its measured time: + /// first warms up for , doubling the number of iterations each round, + /// then takes samples. + /// + public static Estimate Measure(Sampling sampling, Func iteration) + { + TimeSpan Run(long iterations) + { + var elapsed = TimeSpan.Zero; + for (long i = 0; i < iterations; i++) + { + elapsed += iteration(); + } + return elapsed; + } + + var warmUp = TimeSpan.Zero; + long warmUpIterations = 0; + for (long iterations = 1; warmUp <= sampling.WarmUp; iterations *= 2) + { + var started = Stopwatch.GetTimestamp(); + Run(iterations); + warmUp += Stopwatch.GetElapsedTime(started); + warmUpIterations += iterations; + } + var counts = sampling.IterationCounts(Math.Max(1, warmUp.TotalNanoseconds / warmUpIterations)); + return Estimate.Of(counts.Select(count => Run(count).TotalNanoseconds / count).ToArray()); + } + +} diff --git a/csharp/PostgreSQLVSDoublets/IBenchedLinks.cs b/csharp/PostgreSQLVSDoublets/IBenchedLinks.cs new file mode 100644 index 0000000..7ca1497 --- /dev/null +++ b/csharp/PostgreSQLVSDoublets/IBenchedLinks.cs @@ -0,0 +1,15 @@ +namespace Comparisons.PostgreSQLVSDoublets; + +public readonly record struct Link(ulong Id, ulong Source, ulong Target); + +public interface IBenchedLinks : IDisposable +{ + ulong Any { get; } + void Fork(int backgroundLinks); + void Unfork(); + ulong CreatePoint(); + void Update(ulong id, ulong source, ulong target); + void Delete(ulong id); + void Each(ulong id, ulong source, ulong target, Action visit); + ulong Count(); +} diff --git a/csharp/PostgreSQLVSDoublets/PostgreSQLVSDoublets.csproj b/csharp/PostgreSQLVSDoublets/PostgreSQLVSDoublets.csproj new file mode 100644 index 0000000..66a8d90 --- /dev/null +++ b/csharp/PostgreSQLVSDoublets/PostgreSQLVSDoublets.csproj @@ -0,0 +1,12 @@ + + + Exe + net10.0 + enable + enable + + + + + + diff --git a/csharp/PostgreSQLVSDoublets/PostgresLinks.cs b/csharp/PostgreSQLVSDoublets/PostgresLinks.cs new file mode 100644 index 0000000..bf47c47 --- /dev/null +++ b/csharp/PostgreSQLVSDoublets/PostgresLinks.cs @@ -0,0 +1,124 @@ +using Npgsql; + +namespace Comparisons.PostgreSQLVSDoublets; + +/// The schema and statement sequence used by the Rust PostgreSQL adapter. +public sealed class PostgresLinks : IBenchedLinks +{ + private readonly NpgsqlDataSource _source; + private readonly NpgsqlConnection _connection; + private readonly bool _transactional; + private NpgsqlTransaction? _transaction; + public ulong Any => ulong.MaxValue; + + public PostgresLinks(string connection, bool transactional) + { + _source = NpgsqlDataSource.Create(connection); + _connection = _source.OpenConnection(); + _transactional = transactional; + } + + private NpgsqlCommand Command(string sql, params ulong[] parameters) + { + var command = new NpgsqlCommand(sql, _connection, _transaction); + foreach (var value in parameters) + { + command.Parameters.Add(new NpgsqlParameter { TypedValue = checked((long)value) }); + } + return command; + } + + private void Execute(string sql, params ulong[] parameters) + { + using var command = Command(sql, parameters); + command.ExecuteNonQuery(); + } + + public void Fork(int backgroundLinks) + { + if (_transactional) + { + _transaction = _connection.BeginTransaction(); + } + Execute("CREATE TABLE Links (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, from_id bigint, to_id bigint)"); + Execute("CREATE INDEX source ON Links USING btree(from_id)"); + Execute("CREATE INDEX target ON Links USING btree(to_id)"); + for (var i = 0; i < backgroundLinks; i++) + { + CreatePoint(); + } + } + + public void Unfork() + { + Execute("DROP TABLE Links"); + _transaction?.Commit(); + _transaction?.Dispose(); + _transaction = null; + } + + public ulong CreatePoint() + { + using var command = Command("INSERT INTO Links(to_id, from_id) VALUES (0, 0) RETURNING id"); + var id = checked((ulong)(long)command.ExecuteScalar()!); + Update(id, id, id); + return id; + } + + public void Update(ulong id, ulong source, ulong target) + { + using (var previous = Command("SELECT * FROM Links WHERE id = $1", id)) + using (var reader = previous.ExecuteReader()) + { + if (!reader.Read()) + { + throw new InvalidOperationException($"Link {id} does not exist"); + } + } + Execute("UPDATE Links SET from_id = $1, to_id = $2 WHERE id = $3", source, target, id); + } + + public void Delete(ulong id) + { + using var command = Command("DELETE FROM Links WHERE id = $1 RETURNING from_id, to_id", id); + using var reader = command.ExecuteReader(); + if (!reader.Read()) + { + throw new InvalidOperationException($"Link {id} does not exist"); + } + } + + public void Each(ulong id, ulong source, ulong target, Action visit) + { + var conditions = new List(); + var parameters = new List(); + foreach (var (column, value) in new[] { ("id", id), ("from_id", source), ("to_id", target) }) + { + if (value != Any) + { + parameters.Add(value); + conditions.Add($"{column} = ${parameters.Count}"); + } + } + var where = conditions.Count == 0 ? "" : " WHERE " + string.Join(" AND ", conditions); + using var command = Command("SELECT * FROM Links" + where, parameters.ToArray()); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + visit(new Link((ulong)reader.GetInt64(0), (ulong)reader.GetInt64(1), (ulong)reader.GetInt64(2))); + } + } + + public ulong Count() + { + using var command = Command("SELECT COUNT(*) FROM Links"); + return (ulong)(long)command.ExecuteScalar()!; + } + + public void Dispose() + { + _transaction?.Dispose(); + _connection.Dispose(); + _source.Dispose(); + } +} diff --git a/csharp/PostgreSQLVSDoublets/Program.cs b/csharp/PostgreSQLVSDoublets/Program.cs new file mode 100644 index 0000000..cef787a --- /dev/null +++ b/csharp/PostgreSQLVSDoublets/Program.cs @@ -0,0 +1,55 @@ +using Comparisons.PostgreSQLVSDoublets; + +var background = ReadSize("BENCHMARK_BACKGROUND_LINKS", 1000); +var links = ReadSize("BENCHMARK_LINKS", 100); +if (links > background) +{ + throw new ArgumentException("BENCHMARK_LINKS must not exceed BENCHMARK_BACKGROUND_LINKS"); +} +var backend = Environment.GetEnvironmentVariable("BENCHMARK_BACKEND") ?? "all"; +if (backend is not ("all" or "psql" or "doublets")) +{ + throw new ArgumentException($"Unknown BENCHMARK_BACKEND: {backend}"); +} +var connection = Environment.GetEnvironmentVariable("NPGSQL_CONNECTION") ?? + "Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=postgres"; +var variants = new List(); +if (backend is "all" or "psql") +{ + variants.AddRange(["PSQL_NonTransaction", "PSQL_Transaction"]); +} +if (backend is "all" or "doublets") +{ + variants.AddRange(DoubletsLinks.Variants); +} +var directory = Path.Combine(Path.GetTempPath(), "postgres-doublets-" + Guid.NewGuid()); +Directory.CreateDirectory(directory); +try +{ + foreach (var operation in Benchmarks.Operations) + { + foreach (var variant in variants) + { + using IBenchedLinks store = variant.StartsWith("PSQL", StringComparison.Ordinal) + ? new PostgresLinks(connection, variant == "PSQL_Transaction") + : DoubletsLinks.Open(variant, directory); + var estimate = Harness.Measure(variant.StartsWith("PSQL", StringComparison.Ordinal) ? Sampling.PostgreSQL : Sampling.Doublets, () => Benchmarks.Iteration(store, operation, background, links)); + Console.WriteLine(estimate.Bencher(operation, variant)); + } + } +} +finally +{ + Directory.Delete(directory, true); +} + +static int ReadSize(string key, int fallback) +{ + var text = Environment.GetEnvironmentVariable(key); + if (text is null) return fallback; + if (!int.TryParse(text, out var value) || value <= 0) + { + throw new ArgumentException($"{key} must be a positive integer"); + } + return value; +} diff --git a/csharp/PostgreSQLVSDoublets/packages.lock.json b/csharp/PostgreSQLVSDoublets/packages.lock.json new file mode 100644 index 0000000..8ff4d4d --- /dev/null +++ b/csharp/PostgreSQLVSDoublets/packages.lock.json @@ -0,0 +1,200 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Npgsql": { + "type": "Direct", + "requested": "[10.0.0, )", + "resolved": "10.0.0", + "contentHash": "xZAYhPOU2rUIFpV48xsqhCx9vXs6Y+0jX2LCoSEfDFYMw9jtAOUk3iQsCnDLrFIv9NT3JGMihn7nnuZsPKqJmA==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0" + } + }, + "Platform.Data.Doublets": { + "type": "Direct", + "requested": "[0.18.1, )", + "resolved": "0.18.1", + "contentHash": "b6p7GgLDonEnRdhuy3Xsn/N5g5Qq+aivapAlkFH/Mk/MDTDiSJ0C1wnViC4Ff9/d6IXfmh2Sv/TeGIkyKbTOTw==", + "dependencies": { + "Platform.Collections.Methods": "0.7.3", + "Platform.Data": "0.16.1", + "Platform.Data.Doublets.FFI": "0.9.0", + "Platform.Memory": "0.4.1", + "Platform.Random": "0.3.0", + "Platform.Singletons": "0.3.0", + "Platform.Timestamps": "0.3.0" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "L3AdmZ1WOK4XXT5YFPEwyt0ep6l8lGIPs7F5OOBZc77Zqeo01Of7XXICy47628sdVl0v/owxYJTe86DTgFwKCA==" + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "FU/IfjDfwaMuKr414SSQNTIti/69bHEMb+QKrskRb26oVqpx3lNFXMjs/RC9ZUuhBhcwDM2BwOgoMw+PZ+beqQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0" + } + }, + "Platform.Collections": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "ofU+XTf0CJry3dnpSq5pdiz8cArZDlrd3IaGtYoV8L6e+OhTdx2n544s+XLOZK90KJUsNv/docu2Y97Asdbvuw==", + "dependencies": { + "Platform.Disposables": "0.4.0", + "Platform.Interfaces": "0.5.0", + "Platform.Random": "0.3.0", + "Platform.Ranges": "0.5.0" + } + }, + "Platform.Collections.Methods": { + "type": "Transitive", + "resolved": "0.7.3", + "contentHash": "GG+XkIiZWlg96w9HxBeCeGrWdXr6+ST8L8piPjvC4lVrB0XaJNWWj+FYSnQOcX7MZOXs70Y6+e5eD0t1GygB5g==", + "dependencies": { + "Platform.Numbers": "0.9.0" + } + }, + "Platform.Converters": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "Rz4gLL2zPB/KsqORljmhMaQR8EVcTBnRRO2rz91F/nu+qJXuzpurx4VgtnL0lyso0NfkmNJffTiLucxwYX9pWQ==", + "dependencies": { + "Platform.Reflection": "0.7.0" + } + }, + "Platform.Data": { + "type": "Transitive", + "resolved": "0.16.1", + "contentHash": "1JdpgdWbowNgVx+aJatRyTEL5CS+hB5mjOJD57+R26XfbCpIsscJwL+Kcmk/ivaop6OABd3SX7M2DaAhYHXKtw==", + "dependencies": { + "Platform.Delegates": "0.1.1", + "Platform.Numbers": "0.9.0", + "Platform.Setters": "0.5.0", + "Platform.Threading": "0.4.0" + } + }, + "Platform.Data.Doublets.FFI": { + "type": "Transitive", + "resolved": "0.9.0", + "contentHash": "NRgGyihV3sdmvNW9SLnwBV14zkq89GJlTgyeiB+EKT2cMOkCjZtd55C/M0Sy5tLkZ076rBcup4jLFKmejnS2tg==" + }, + "Platform.Delegates": { + "type": "Transitive", + "resolved": "0.1.1", + "contentHash": "+182klZgGEJ+yWtxz3BMrTCYcpSXjvJzrYzpK+W2UecAaDKLvLyehjPlMfta+dM/XlKbPt53Wf20vhI1kul0SQ==" + }, + "Platform.Disposables": { + "type": "Transitive", + "resolved": "0.4.0", + "contentHash": "ko0W4cw5XhNxJANdO1T8srpWRVqpPiXtCohumlbTu33kivRBEmk1ivDsySaNhKTyuYjJnrjRYrgA1iEb2sS+/Q==", + "dependencies": { + "Platform.Exceptions": "0.5.0" + } + }, + "Platform.Exceptions": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "htu3iyPVJinbLxU1I1cpc2qFlur7IF2fTc9u2jqQ+3X1Lyzd3aeK9Y7gJu9ZkdnIO3tbdO6wxr1HZG4UhaIG8A==" + }, + "Platform.Interfaces": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "OBViyjgCYWaXH8xhibYL/fbwHfPP/+qVUDxcFGH22IaYzbrFbfNR5TxW6gsk3LR9QXLANnrLsuI7phZueoj/Tw==" + }, + "Platform.IO": { + "type": "Transitive", + "resolved": "0.5.1", + "contentHash": "MZ+uZIrDZuNZZu8J8gv8RcXwGihSTEo9UvFoRHCGIXQnaQxnJ0gBIF5s/zSjJ1+h8c78lv/wmlSvh4zMzwyJIQ==", + "dependencies": { + "Platform.Threading": "0.5.0", + "Platform.Unsafe": "0.6.1" + } + }, + "Platform.Memory": { + "type": "Transitive", + "resolved": "0.4.1", + "contentHash": "Jz4Xb0B8pmh6BU9hhvqMgK2L+R0fCxSk1bGWL7WK/WYKAYSuGaJUAvzWdJLTECyfs+w6PVUxXK53q1OXxHRmMQ==", + "dependencies": { + "Platform.IO": "0.5.1" + } + }, + "Platform.Numbers": { + "type": "Transitive", + "resolved": "0.9.0", + "contentHash": "0GgZSH0p+jP933DKtLzCfOAdkJ1cNKrSrisXgMKy0zTMBZOxMxzs83IVXwBju66vm5N/w6lFq8HwCLW0gQsYVA==", + "dependencies": { + "Platform.Converters": "0.4.0" + } + }, + "Platform.Random": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "GX24Xx45cUIr9EZl55evdwb7oYN/nimpoNqDG4zoMDDLqeEpL35zX8IiHHDYITV9F4bjrCrSEiRCVsRFAsi7SQ==", + "dependencies": { + "Platform.Ranges": "0.5.0" + } + }, + "Platform.Ranges": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "CUhZM/4F714NAyrtWtkBx78L4xJpFZ1wp+6ivFvpAXx6DsaBcBsyDxjn5eoLB9LSUYxB11gFkwwmXmJhMgZ3qw==", + "dependencies": { + "Platform.Exceptions": "0.5.0" + } + }, + "Platform.Reflection": { + "type": "Transitive", + "resolved": "0.7.0", + "contentHash": "CvpRnDWH5YArcXlY7lcY0mDYQCtnJAeZsqZ4/IdllJoj7z8JwTufeo/zK62Vn6K0ulTyHNhb+/T3HFusxnrfxA==", + "dependencies": { + "Platform.Collections": "0.4.0" + } + }, + "Platform.Setters": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "vZG+VnYc4u2x5vnBY7rqaFHtOS2AeoLwGV+02JBFa9pxluHwVTu4nVcVBTJTlenZSfAoZUNVsV1StnUWvINkCQ==", + "dependencies": { + "Platform.Delegates": "0.1.1", + "Platform.Interfaces": "0.5.0" + } + }, + "Platform.Singletons": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "fCfNw4CqPL6HzsUJVVYI/OLYXGkjmdDCe+QWThW8hPoNNlqMZ+FWbfBXbXpCuDYw9EYhLDlhuTAQMbqNt7kCzg==", + "dependencies": { + "Platform.Reflection": "0.7.0" + } + }, + "Platform.Threading": { + "type": "Transitive", + "resolved": "0.5.0", + "contentHash": "Q0p5oVDbhQ76IGHi/vYIX+de9T270XZmNgjNbHtobc+AoJbIWGf+kZXTzq+MmEfXS2ANtGNe9uWN1a7Nzp7olQ==", + "dependencies": { + "Platform.Collections": "0.4.0" + } + }, + "Platform.Timestamps": { + "type": "Transitive", + "resolved": "0.3.0", + "contentHash": "Lmhm8Vp0DobTlxSVTPZZE+3ABTrvuofcmTQOWZv6TyiZ5NyFBH6m61vZdnIkMa/3lQguKvGfgMsUT97p0t5x7w==", + "dependencies": { + "Platform.Interfaces": "0.5.0" + } + }, + "Platform.Unsafe": { + "type": "Transitive", + "resolved": "0.6.1", + "contentHash": "GbKgPC7lfw+q2GqQEch0NSxHOZXxoLTj3ChjdRZIQBvZnGWzYH6nat8glfTJseOmgdabIxlwogFECoai0O3akA==", + "dependencies": { + "Platform.Numbers": "0.7.0" + } + } + } + } +} \ No newline at end of file diff --git a/csharp/global.json b/csharp/global.json new file mode 100644 index 0000000..c8dd625 --- /dev/null +++ b/csharp/global.json @@ -0,0 +1 @@ +{"sdk": {"version": "10.0.100", "rollForward": "latestFeature"}} From 2396e6609c45ab673bef4f1ee979cd2421ea13cb Mon Sep 17 00:00:00 2001 From: konard Date: Tue, 6 Oct 2026 23:42:38 +0000 Subject: [PATCH 5/6] Run isolated benchmark matrices and publish complete reproducible reports --- .github/workflows/cpp.yml | 94 ----------- .github/workflows/rust.yml | 259 +++++++++++++++++++------------ .gitignore | 5 + .gitkeep | 1 - README.md | 108 +++++++++---- cpp/Benchmarks/AllBenchmarks.cpp | 23 --- cpp/Benchmarks/Create.cpp | 158 ------------------- cpp/Benchmarks/Delete.cpp | 159 ------------------- cpp/Benchmarks/Each/All.cpp | 121 --------------- cpp/Benchmarks/Each/Concrete.cpp | 169 -------------------- cpp/Benchmarks/Each/Identity.cpp | 170 -------------------- cpp/Benchmarks/Each/Incoming.cpp | 169 -------------------- cpp/Benchmarks/Each/Outgoing.cpp | 169 -------------------- cpp/Benchmarks/SetupTeardown.cpp | 28 ---- cpp/Benchmarks/Update.cpp | 113 -------------- cpp/CMakeLists.txt | 24 --- cpp/LinksPSQL/Client.h | 15 -- cpp/LinksPSQL/ISQL.h | 129 --------------- cpp/LinksPSQL/PSQLBase.h | 83 ---------- cpp/LinksPSQL/Transaction.h | 15 -- cpp/conanfile.txt | 8 - cpp/out.py | 143 ----------------- experiments/run_benchmarks.sh | 28 ++++ rust/tests/same_behavior.rs | 30 ++-- scripts/benchmark_header.py | 42 +++++ scripts/benchmark_report.py | 14 +- scripts/requirements.txt | 1 + scripts/test_benchmark_report.py | 13 ++ 28 files changed, 355 insertions(+), 1936 deletions(-) delete mode 100644 .github/workflows/cpp.yml delete mode 100644 .gitkeep delete mode 100644 cpp/Benchmarks/AllBenchmarks.cpp delete mode 100644 cpp/Benchmarks/Create.cpp delete mode 100644 cpp/Benchmarks/Delete.cpp delete mode 100644 cpp/Benchmarks/Each/All.cpp delete mode 100644 cpp/Benchmarks/Each/Concrete.cpp delete mode 100644 cpp/Benchmarks/Each/Identity.cpp delete mode 100644 cpp/Benchmarks/Each/Incoming.cpp delete mode 100644 cpp/Benchmarks/Each/Outgoing.cpp delete mode 100644 cpp/Benchmarks/SetupTeardown.cpp delete mode 100644 cpp/Benchmarks/Update.cpp delete mode 100644 cpp/CMakeLists.txt delete mode 100644 cpp/LinksPSQL/Client.h delete mode 100644 cpp/LinksPSQL/ISQL.h delete mode 100644 cpp/LinksPSQL/PSQLBase.h delete mode 100644 cpp/LinksPSQL/Transaction.h delete mode 100644 cpp/conanfile.txt delete mode 100644 cpp/out.py create mode 100755 experiments/run_benchmarks.sh create mode 100644 scripts/benchmark_header.py create mode 100644 scripts/requirements.txt diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml deleted file mode 100644 index 3c54f85..0000000 --- a/.github/workflows/cpp.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Benchmark C++ version - -on: - push: - branches: - - add_cpp_benchmark - -jobs: - benchmark: - runs-on: ubuntu-latest - - services: - postgres: - image: postgres:latest - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Set up CMake and Conan - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install utilities - run: | - sudo apt update - sudo apt install -y cmake build-essential - pip install conan - - name: Set up Conan dependencies - run: | - conan profile detect - echo " - [settings] - arch=x86_64 - build_type=Release - compiler=gcc - compiler.cppstd=gnu20 - compiler.libcxx=libstdc++11 - compiler.version=11 - os=Linux - " > ~/.conan2/profiles/default - conan profile show - - name: Install dependencies - run: | - git clone https://github.com/linksplatform/conan-center-index - cd conan-center-index && cd recipes - conan create platform.interfaces/all --version 0.3.41 - conan create platform.delegates/all --version 0.3.7 - conan create platform.exceptions/all --version 0.3.2 - conan create platform.converters/all --version 0.2.0 - conan create platform.hashing/all --version 0.5.6 --build=missing - conan create platform.ranges/all --version 0.2.0 - conan create platform.random/all --version 0.2.0 - conan create platform.equality/all --version 0.0.1 - conan create platform.collections/all --version 0.2.1 - conan create platform.setters/all --version 0.1.0 - conan create platform.data/all --version 0.1.1 - conan create platform.numbers/all --version 0.1.0 - conan create platform.memory/all --version 0.1.0 --build=missing - conan create platform.collections.methods/all --version 0.3.0 - conan create platform.data.doublets/all --version 0.1.0 - - name: Build benchmark - run: | - cd cpp - conan install . -of=build --build=missing - cmake -G "Unix Makefiles" -DCMAKE_TOOLCHAIN_FILE=/home/runner/work/Comparisons.PostgreSQLVSDoublets/Comparisons.PostgreSQLVSDoublets/cpp/build/conan_toolchain.cmake -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_BUILD_TYPE=Release -S . -B build - cd build - make - - name: Run benchmark - run: ./cpp/build/bin/PSQLBench --benchmark_out=cpp/out.txt --benchmark_out_format=console - - name: Prepare benchmark results - run: | - cd cpp - pip install numpy matplotlib - python3 out.py - - name: Save benchmark results - uses: actions/upload-artifact@v4 - with: - name: Benchmark results - path: | - cpp/Bench1.png - cpp/Bench2.png diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index da511e6..61c790d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,132 +1,191 @@ -name: Benchmark Rust version +name: Test and benchmark Rust and C# on: push: - branches: - - main - - master + branches: [main, master] + paths: ['rust/**', 'csharp/**', 'scripts/**', 'experiments/**', 'rust-toolchain.toml', 'rustfmt.toml', '.github/workflows/rust.yml'] pull_request: - branches: - - main - - master + branches: [main, master] + paths: ['rust/**', 'csharp/**', 'scripts/**', 'experiments/**', 'rust-toolchain.toml', 'rustfmt.toml', '.github/workflows/rust.yml'] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: benchmarks-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: - toolchain: stable - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + POSTGRES_CONNECTION: 'host=localhost port=5432 user=postgres password=postgres dbname=postgres' + NPGSQL_CONNECTION: 'Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=postgres' + POSTGRES_VERSION: '17.6' jobs: - benchmark: + sizes: runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + background: ${{ steps.matrix.outputs.background }} + steps: + - id: matrix + env: + EVENT: ${{ github.event_name }} + run: | + if [ "$EVENT" = pull_request ]; then + echo 'background=[100]' >> "$GITHUB_OUTPUT" + else + echo 'background=[100,1000]' >> "$GITHUB_OUTPUT" + fi + test: + runs-on: ubuntu-latest + timeout-minutes: 15 services: postgres: - image: postgres:latest + image: postgres:17.6 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - + options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + ports: ['5432:5432'] steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Rust - uses: dtolnay/rust-toolchain@master + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable with: - toolchain: ${{env.toolchain}} components: rustfmt, clippy + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python3 -m unittest discover -s scripts -v + - run: python3 experiments/reproduce_report_ratio.py + - run: cargo fmt --manifest-path rust/Cargo.toml -- --check + - run: cargo clippy --locked --manifest-path rust/Cargo.toml --all-targets --all-features -- -D warnings + - run: cargo test --locked --manifest-path rust/Cargo.toml -- --include-ignored + + test-csharp: + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgres: + image: postgres:17.6 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + ports: ['5432:5432'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - run: dotnet restore csharp/PostgreSQLVSDoublets.slnx --locked-mode + - run: dotnet format csharp/PostgreSQLVSDoublets.slnx --verify-no-changes --no-restore + - run: dotnet build csharp/PostgreSQLVSDoublets.slnx -c Release --no-restore --warnaserror + - run: dotnet test csharp/PostgreSQLVSDoublets.slnx -c Release --no-build --blame-hang-timeout 2m - - name: Wait for PostgreSQL to be ready - run: | - echo "Waiting for PostgreSQL to be fully ready..." - for i in {1..30}; do - if pg_isready -h localhost -p 5432 -U postgres > /dev/null 2>&1; then - echo "PostgreSQL is ready!" - break - fi - echo "Attempt $i: PostgreSQL not ready yet..." - sleep 2 - done - - - name: Build benchmark - run: cargo build --release --all-features --manifest-path rust/Cargo.toml - - - name: Run benchmark - working-directory: rust + benchmark: + needs: [sizes, test, test-csharp] + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + language: [rust, csharp] + backend: [psql, doublets] + background: ${{ fromJSON(needs.sizes.outputs.background) }} + services: + postgres: + image: ${{ matrix.backend == 'psql' && 'postgres:17.6' || '' }} env: - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - # Use 100 links for main/master branch benchmarks, 10 for pull requests - BENCHMARK_LINKS: ${{ (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')) && '100' || '10' }} - # Use 1000 background links for main/master branch, 100 for pull requests - BENCHMARK_BACKGROUND_LINKS: ${{ (github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')) && '1000' || '100' }} + options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + ports: ['5432:5432'] + env: + BENCHMARK_BACKEND: ${{ matrix.backend }} + BENCHMARK_BACKGROUND_LINKS: ${{ matrix.background }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: dtolnay/rust-toolchain@stable + if: matrix.language == 'rust' + - uses: actions/setup-dotnet@v4 + if: matrix.language == 'csharp' + with: + dotnet-version: '10.0.x' + - name: Build benchmark before measuring + env: + LANGUAGE: ${{ matrix.language }} run: | - set -o pipefail - cargo bench --bench bench -- --output-format bencher | tee out.txt - - - name: Prepare benchmark results - if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master') + if [ "$LANGUAGE" = rust ]; then + cargo bench --locked --manifest-path rust/Cargo.toml --bench bench --no-run + else + dotnet restore csharp/PostgreSQLVSDoublets/PostgreSQLVSDoublets.csproj --locked-mode + dotnet build csharp/PostgreSQLVSDoublets/PostgreSQLVSDoublets.csproj -c Release --no-restore --warnaserror + fi + - name: Measure one backend and size + env: + LANGUAGE: ${{ matrix.language }} run: | - git config --global user.email "linksplatform@gmail.com" - git config --global user.name "LinksPlatformBencher" - cd rust - pip install numpy matplotlib - python3 out.py - cd .. - - # Create Docs directory if it doesn't exist - mkdir -p Docs - - # Copy generated images - cp -f rust/bench_rust.png Docs/ - cp -f rust/bench_rust_log_scale.png Docs/ - - # Update README with latest results - if [ -f rust/results.md ]; then - # Replace the results section in README.md - python3 -c " - import re - - with open('rust/results.md', 'r') as f: - results = f.read() - - with open('README.md', 'r') as f: - readme = f.read() - - # Pattern to find and replace the results table - pattern = r'(\| Operation.*?\n\|[-|]+\n(?:\|.*?\n)*)' - if re.search(pattern, readme): - readme = re.sub(pattern, results.strip() + '\n', readme) - - with open('README.md', 'w') as f: - f.write(readme) - " + set -o pipefail + export BENCHMARK_LINKS=$((BENCHMARK_BACKGROUND_LINKS / 10)) + mkdir -p results + output="results/$LANGUAGE-$BENCHMARK_BACKGROUND_LINKS-$BENCHMARK_BACKEND.txt" + if [ "$BENCHMARK_BACKEND" = psql ]; then + export POSTGRES_VERSION=$(psql "$POSTGRES_CONNECTION" -Atc 'SHOW server_version') fi - - # Commit changes if any - git add Docs README.md - if git diff --staged --quiet; then - echo "No changes to commit" + python3 scripts/benchmark_header.py "$LANGUAGE" "$BENCHMARK_BACKEND" > "$output" + if [ "$LANGUAGE" = rust ]; then + (cd rust && cargo bench --locked --bench bench -- --output-format bencher --noplot) | tee -a "$output" else - git commit -m "Update benchmark results" - git push origin HEAD + dotnet run --project csharp/PostgreSQLVSDoublets -c Release --no-build | tee -a "$output" fi + - uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.language }}-${{ matrix.background }}-${{ matrix.backend }} + path: results/*.txt + if-no-files-found: error - - name: Save benchmark results - uses: actions/upload-artifact@v4 + report: + needs: benchmark + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: actions/download-artifact@v4 with: - name: Benchmark results + pattern: results-* + path: results + merge-multiple: true + - run: python3 -m pip install -r scripts/requirements.txt + - name: Generate tables and charts + run: python3 scripts/benchmark_report.py results --readme README.md --charts Docs > results/report.md + - uses: actions/upload-artifact@v4 + with: + name: benchmark-report path: | - rust/bench_rust.png - rust/bench_rust_log_scale.png - rust/out.txt + README.md + Docs/bench_*.png + results/ + - name: Publish results on the default branch + if: github.event_name == 'push' + run: | + git config user.email 'linksplatform@gmail.com' + git config user.name 'LinksPlatformBencher' + git add README.md Docs + if ! git diff --staged --quiet; then + git commit -m 'Update Rust and C# benchmark results' + git pull --rebase origin "$GITHUB_REF_NAME" + git push origin "HEAD:$GITHUB_REF_NAME" + fi diff --git a/.gitignore b/.gitignore index 90f7c20..3f25c8e 100644 --- a/.gitignore +++ b/.gitignore @@ -370,3 +370,8 @@ MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ + +# Benchmark output and integration-test evidence +*.links +results/ +ci-logs/ diff --git a/.gitkeep b/.gitkeep deleted file mode 100644 index 3f99036..0000000 --- a/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# .gitkeep file auto-generated at 2026-10-06T23:30:30.621Z for PR creation at branch issue-42-5c34a9a6fc98 for issue https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/issues/42 \ No newline at end of file diff --git a/README.md b/README.md index 6ecd8f4..a48db0a 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,90 @@ # Comparisons.PostgreSQLVSDoublets -The comparison between PostgreSQL and LinksPlatform's Doublets (links) on basic database operations with links (create, read, delete, update). -All benchmarks ran with 3000 links in background to increase size of indexes and 1000 are actively created/updated/deleted. +PostgreSQL and LinksPlatform Doublets compared on the same basic link operations in Rust and C#. A doublet link has an ID, a source and a target. See [the links theory](https://habr.com/ru/articles/895896) for the representation. -In this particular benchmark we decided not to increase the number of links as PostgreSQL will not be able to handle it at all in timeframe what GitHub Actions limit allows to use for free. Remember that to get accurate result we ran this benchmark multiple times. +Each language runs four Doublets stores (united/split, in RAM/memory-mapped files) and two PostgreSQL modes (autocommit/explicit transaction). PostgreSQL uses a `bigint` identity primary key and separate B-tree indexes on source and target. Doublets runs inside the benchmark process; PostgreSQL uses a local server and includes driver and network costs. Memory-mapped Doublets writes use the OS page cache without an explicit disk flush; their durability differs from PostgreSQL commits. -## Task +## Operations -Both databases used to store and retrieve doublet-links representation. To support storage, and all basic CRUD operations that provide Turing completeness for links as in [the links theory](https://habr.com/ru/articles/895896). +- **Create**: create `BENCHMARK_LINKS` point links (`id = source = target`). +- **Update**: change the last `BENCHMARK_LINKS` background points to `(0, 0)` and back, timing both updates. +- **Delete**: prepare `BENCHMARK_LINKS` additional points, then delete those points in reverse order. +- **Each All**: enumerate all background links once (`[*, *, *]`). +- **Each Identity**: enumerate `[id, *, *]` for each background point. +- **Each Concrete**: enumerate `[*, source, target]` for each background point. +- **Each Outgoing**: enumerate `[*, source, *]` for each background point. +- **Each Incoming**: enumerate `[*, *, target]` for each background point. -## Operations -- **Create** – insert point link (link with id = source = target) -- **Update** – basic link update operation -- **Delete** – basic link delete operation -- **Each All** – take all links matching `[*, *, *]` constraint -- **Each Incoming** – take all links matching `[*, *, target]` constraint -- **Each Outgoing** – take all links matching `[*, source, *]` constraint -- **Each Concrete** – take all links matching `[*, source, target]` constraint -- **Each Identity** – take all links matching `[id, *, *]` constraint +Both languages recreate `BENCHMARK_BACKGROUND_LINKS` points before each iteration and clear the store afterwards. Background creation, delete preparation, table/index setup and cleanup are excluded from the measured time. In transaction mode the timed SQL operations run inside an explicit transaction; its begin and commit happen outside the timed interval. Thus these numbers measure operation batches, rather than committed transactions or individual links. Indexed reads perform one query per background point, independently of `BENCHMARK_LINKS`. + +Both harnesses use one second of warm-up, ten flat samples and a one-second measurement target. Iteration counts are calibrated using total wall time including preparation, as Criterion 0.4 does for `iter_custom`; reported samples include only timed operations. The report shows medians and compares each Doublets median with the fastest PostgreSQL median. Medians within 5%, or overlapping ± one standard deviation ranges, are labelled `≈ same`. Other cells show `× faster` or `× slower`. Linear charts give very short bars a minimum visible width; log charts use the measured values. ## Results -The results below represent the amount of time (ns) the operation takes per iteration. -- First picture shows time in a pixel scale (for doublets just minimum value is shown, otherwise it will be not present on the graph). -- Second picture shows time in a logarithmic scale (to see difference clearly, because it is around 2-3 orders of magnitude). +Every generated table records the server and library versions, CPU, UTC date and source Actions run. CI runs on separate runners for each language, backend and size. PRs use 100 background / 10 active links. Default-branch runs also measure 1,000 background / 100 active links. A report job assembles all tables and linear/log charts, uploads them for PR review, and commits results after default-branch pushes. Changes only to this README or the charts do not start benchmarks. + + ### Rust -![Image of Rust benchmark (pixel scale)](https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/blob/main/Docs/bench_rust.png?raw=true) -![Image of Rust benchmark (log scale)](https://github.com/linksplatform/Comparisons.PostgreSQLVSDoublets/blob/main/Docs/bench_rust_log_scale.png?raw=true) -### Raw benchmark results (all numbers are in nanoseconds) +_Updated results will be recorded after the first run of the new workflow._ + +### C# + +_Updated results will be recorded after the first run of the new workflow._ + + +Results depend on workload, server configuration, versions and runner hardware. Use the provenance and sizes above each table when comparing runs. + +## Running locally + +Install stable Rust, .NET 10, Python 3.11+ and PostgreSQL. The CI server image is pinned to `postgres:17.6`. Use a disposable database: the benchmarks create and drop a table named `links`, and correctness tests reset that table. + +```bash +docker run -d --name comparison-postgres -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres postgres:17.6 +export POSTGRES_CONNECTION='host=localhost port=5432 user=postgres password=postgres dbname=postgres' +export NPGSQL_CONNECTION='Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=postgres' +export POSTGRES_VERSION=17.6 +export BENCHMARK_BACKGROUND_LINKS=100 +export BENCHMARK_LINKS=10 +python3 -m pip install -r scripts/requirements.txt + +# Report regressions and backend correctness. +python3 -m unittest discover -s scripts -v +cargo fmt --manifest-path rust/Cargo.toml -- --check +cargo clippy --locked --manifest-path rust/Cargo.toml --all-targets --all-features -- -D warnings +cargo test --locked --manifest-path rust/Cargo.toml -- --include-ignored +dotnet restore csharp/PostgreSQLVSDoublets.slnx --locked-mode +dotnet format csharp/PostgreSQLVSDoublets.slnx --verify-no-changes --no-restore +dotnet build csharp/PostgreSQLVSDoublets.slnx -c Release --no-restore --warnaserror +dotnet test csharp/PostgreSQLVSDoublets.slnx -c Release --no-build --blame-hang-timeout 2m -| Operation | Doublets United Volatile | Doublets United NonVolatile | Doublets Split Volatile | Doublets Split NonVolatile | PSQL NonTransaction | PSQL Transaction | -|---------------|--------------------------|-----------------------------|-------------------------|----------------------------|---------------------|------------------| -| Create | 9035 (12162.4x faster) | 8807 (12477.2x faster) | 7005 (15686.9x faster) | 7086 (15507.6x faster) | 145975967 | 109886881 | -| Update | 33210 (4323.7x faster) | 32724 (4387.9x faster) | 7826 (18347.9x faster) | 7756 (18513.5x faster) | 181785688 | 143590948 | -| Delete | 17427 (2096.8x faster) | 17237 (2119.9x faster) | 12612 (2897.4x faster) | 12659 (2886.6x faster) | 54113187 | 36541411 | -| Each All | 1903 (394.1x faster) | 1903 (394.1x faster) | 2520 (297.6x faster) | 2220 (337.8x faster) | 772260 | 750025 | -| Each Identity | 32364 (10720.2x faster) | 32376 (10716.2x faster) | 32675 (10618.2x faster) | 32678 (10617.2x faster) | 365907001 | 346948519 | -| Each Concrete | 47228 (7710.2x faster) | 46435 (7841.9x faster) | 42978 (8472.6x faster) | 43041 (8460.2x faster) | 388005509 | 364137549 | -| Each Outgoing | 60270 (5830.3x faster) | 59978 (5858.7x faster) | 36132 (9725.2x faster) | 36206 (9705.3x faster) | 369609106 | 351390605 | -| Each Incoming | 59812 (5849.0x faster) | 59956 (5835.0x faster) | 36733 (9523.9x faster) | 36753 (9518.7x faster) | 362811363 | 349842592 | +# Run backends sequentially to avoid CPU contention. +mkdir -p results +for language in rust csharp; do + for backend in psql doublets; do + export BENCHMARK_BACKEND="$backend" + output="results/$language-$BENCHMARK_BACKGROUND_LINKS-$backend.txt" + python3 scripts/benchmark_header.py "$language" "$backend" > "$output" + if [ "$language" = rust ]; then + (cd rust && cargo bench --locked --bench bench -- --output-format bencher --noplot) >> "$output" + else + dotnet run --project csharp/PostgreSQLVSDoublets -c Release --no-build >> "$output" + fi + done +done +python3 scripts/benchmark_report.py results --readme README.md --charts Docs +``` -## Conclusion +`rust/out.py` remains an entry point to the same report generator and accepts the same arguments. Output files must be named `--.txt`; a report requires both complete backends for each language and size it publishes. The C++ prototype and its dormant workflow have been removed. -As we can see in this comparison, Doublets are from 1746 to 15745 times faster than PostgreSQL in write operations, and from 100 to 9694 times faster in read operations. +| Variable | Default | Meaning | +| --- | --- | --- | +| `BENCHMARK_BACKGROUND_LINKS` | `1000` | Background point links per iteration; must be positive. | +| `BENCHMARK_LINKS` | `100` | Active links for writes; must be positive and no greater than the background. | +| `BENCHMARK_BACKEND` | `all` | `psql`, `doublets` or `all`; select one for isolated measurements. | +| `POSTGRES_CONNECTION` | `user=postgres dbname=postgres password=postgres host=localhost port=5432` | Rust connection string in postgres/libpq format, including custom hosts, ports and credentials. | +| `NPGSQL_CONNECTION` | `Host=localhost;Port=5432;Username=postgres;Password=postgres;Database=postgres` | C# Npgsql connection string. | +| `POSTGRES_VERSION` | `unknown version` | Server version recorded by the header script; CI reads `SHOW server_version`. | -To get fresh numbers, please fork the repository and rerun benchmark in GitHub Actions. +Rust PostgreSQL tests are ignored unless `--include-ignored` is supplied. C# PostgreSQL tests are explicitly skipped without `NPGSQL_CONNECTION`; CI configures both connections and runs all variants. Report tracing is off by default; set `DEBUG = True` in `scripts/benchmark_report.py` when investigating parsing. diff --git a/cpp/Benchmarks/AllBenchmarks.cpp b/cpp/Benchmarks/AllBenchmarks.cpp deleted file mode 100644 index 46e3a5c..0000000 --- a/cpp/Benchmarks/AllBenchmarks.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -const std::string options {"user=postgres dbname=postgres password=postgres host=localhost port=5432"}; -constexpr std::uint64_t BACKGROUND_LINKS {3000}; - -#include "SetupTeardown.cpp" -#include "Create.cpp" -#include "Delete.cpp" -#include "Each/Identity.cpp" -#include "Each/Concrete.cpp" -#include "Each/Outgoing.cpp" -#include "Each/Incoming.cpp" -#include "Each/All.cpp" -#include "Update.cpp" - -BENCHMARK_MAIN(); diff --git a/cpp/Benchmarks/Create.cpp b/cpp/Benchmarks/Create.cpp deleted file mode 100644 index 285bf3b..0000000 --- a/cpp/Benchmarks/Create.cpp +++ /dev/null @@ -1,158 +0,0 @@ -static void BM_PSQLCreateLinksWithoutTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - Client> table {options}; - auto background = BACKGROUND_LINKS; - Setup(table); - for (auto _: state) { - for (std::uint64_t i {}; i < state.range(0); ++i) { - CreatePoint(table); - } - state.PauseTiming(); - for (std::uint64_t i = background + state.range(0); i > background; --i) { - Delete(table, {i, i, i}); - } - background += state.range(0); - state.ResumeTiming(); - } - Teardown(table); -} - -static void BM_PSQLCreateLinksWithTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - { - Transaction> transaction {options}; - Setup(transaction); - } - auto background = BACKGROUND_LINKS; - for (auto _: state) { - { - state.PauseTiming(); - Transaction> transaction {options}; - state.ResumeTiming(); - for (std::uint64_t i {}; i < state.range(0); ++i) { - CreatePoint(transaction); - } - } - state.PauseTiming(); - { - Transaction> transaction {options}; - for (std::uint64_t i = background + state.range(0); i > background; --i) { - Delete(transaction, {i, i, i}); - } - background += state.range(0); - } - state.ResumeTiming(); - } - Transaction> transaction {options}; - Teardown(transaction); -} - -static void BM_DoubletsUnitedCreateLinksFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - std::filesystem::path path {"united.links"}; - UnitedMemoryLinks> storage {FileMappedResizableDirectMemory{path.string()}}; - auto handler = [&storage] (std::vector before, std::vector after) { - return storage.Constants.Continue; - }; - Setup(storage); - for (auto _: state) { - for (std::size_t i {}; i < state.range(0); ++i) { - CreatePoint(storage); - } - state.PauseTiming(); - for (std::uint64_t i = BACKGROUND_LINKS + state.range(0); i > BACKGROUND_LINKS; --i) { - storage.Delete({i, i, i}, handler); - } - state.ResumeTiming(); - } - Teardown(storage); -} - -static void BM_DoubletsUnitedCreateLinksRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory memory {}; - UnitedMemoryLinks, HeapResizableDirectMemory> storage {std::move(memory)}; - auto handler = [&storage] (std::vector before, std::vector after) { - return storage.Constants.Continue; - }; - Setup(storage); - for (auto _: state) { - for (std::size_t i {}; i < state.range(0); ++i) { - CreatePoint(storage); - } - state.PauseTiming(); - for (std::uint64_t i = BACKGROUND_LINKS + state.range(0); i > BACKGROUND_LINKS; --i) { - storage.Delete({i, i, i}, handler); - } - state.ResumeTiming(); - } - Teardown(storage); -} - -static void BM_DoubletsSplitCreateLinksFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - std::filesystem::path split_data {"split_data.links"}, split_index {"split_index.links"}; - SplitMemoryLinks> storage { - FileMappedResizableDirectMemory{split_data.string()}, - FileMappedResizableDirectMemory{split_index.string()} - }; - auto handler = [&storage] (std::vector before, std::vector after) { - return storage.Constants.Continue; - }; - Setup(storage); - for (auto _: state) { - for (std::size_t i {}; i < state.range(0); ++i) { - CreatePoint(storage); - } - state.PauseTiming(); - for (std::uint64_t i = BACKGROUND_LINKS + state.range(0); i > BACKGROUND_LINKS; --i) { - storage.Delete({i, i, i}, handler); - } - state.ResumeTiming(); - } - Teardown(storage); -} - -static void BM_DoubletsSplitCreateLinksRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory data {}, index {}; - SplitMemoryLinks, HeapResizableDirectMemory> storage {std::move(data), std::move(index)}; - auto handler = [&storage] (std::vector before, std::vector after) { - return storage.Constants.Continue; - }; - Setup(storage); - for (auto _: state) { - for (std::size_t i {}; i < state.range(0); ++i) { - CreatePoint(storage); - } - state.PauseTiming(); - for (std::uint64_t i = BACKGROUND_LINKS + state.range(0); i > BACKGROUND_LINKS; --i) { - storage.Delete({i, i, i}, handler); - } - state.ResumeTiming(); - } - Teardown(storage); -} - -BENCHMARK(BM_PSQLCreateLinksWithoutTransaction)->Name("BM_PSQL/Create/NonTransaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_PSQLCreateLinksWithTransaction)->Name("BM_PSQL/Create/Transaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedCreateLinksFile)->Name("BM_Doublets/United/Create/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedCreateLinksRAM)->Name("BM_Doublets/United/Create/Volatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitCreateLinksFile)->Name("BM_Doublets/Split/Create/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitCreateLinksRAM)->Name("BM_Doublets/Split/Create/Volatile")->Arg(1000)->MinWarmUpTime(20); diff --git a/cpp/Benchmarks/Delete.cpp b/cpp/Benchmarks/Delete.cpp deleted file mode 100644 index 6f0db8a..0000000 --- a/cpp/Benchmarks/Delete.cpp +++ /dev/null @@ -1,159 +0,0 @@ -static void BM_PSQLDeleteLinksWithoutTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - Client> table {options}; - Setup(table); - auto setup = [&table, &state] { - for (std::uint64_t i = BACKGROUND_LINKS + 1; i <= BACKGROUND_LINKS + state.range(0); ++i) { - CreatePoint(table); - } - }; - auto background {BACKGROUND_LINKS - state.range(0)}; - for (auto _: state) { - state.PauseTiming(); - setup(); - background += state.range(0); - state.ResumeTiming(); - for (std::uint64_t i = background + state.range(0); i > background; --i) { - Delete(table, {i, i, i}); - } - } - Teardown(table); -} - -static void BM_PSQLDeleteLinksWithTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - { - Transaction> transaction {options}; - Setup(transaction); - } - auto background {BACKGROUND_LINKS - state.range(0)}; - auto fill = [options = options, &state] { - Transaction> transaction {options}; - for (std::uint64_t i {}; i < state.range(0); ++i) { - CreatePoint(transaction); - } - }; - for (auto _: state) { - state.PauseTiming(); - fill(); - background += state.range(0); - Transaction> transaction {options}; - state.ResumeTiming(); - for (std::uint64_t i = background + state.range(0); i > background; --i) { - Delete(transaction, {i, i, i}); - } - } - Transaction> transaction {options}; - Teardown(transaction); -} - -static void BM_DoubletsUnitedDeleteLinksFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace Platform::Collections; - using namespace SetupTeardown; - std::filesystem::path path {"united.links"}; - UnitedMemoryLinks> storage {FileMappedResizableDirectMemory{path.string()}}; - auto fill = [&storage, &state] { - for (std::size_t i {}; i < state.range(0); ++i) { - CreatePoint(storage); - } - }; - Setup(storage); - for (auto _: state) { - state.PauseTiming(); - fill(); - state.ResumeTiming(); - for (std::uint64_t i = BACKGROUND_LINKS + state.range(0); i > BACKGROUND_LINKS; --i) { - Delete(storage, {i, i, i}); - } - } - Teardown(storage); -} - -static void BM_DoubletsUnitedDeleteLinksRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory memory {}; - UnitedMemoryLinks, HeapResizableDirectMemory> storage {std::move(memory)}; - auto fill = [&storage, &state] { - for (std::size_t i {}; i < state.range(0); ++i) { - CreatePoint(storage); - } - }; - Setup(storage); - for (auto _: state) { - state.PauseTiming(); - fill(); - state.ResumeTiming(); - for (std::uint64_t i = BACKGROUND_LINKS + state.range(0); i > BACKGROUND_LINKS; --i) { - Delete(storage, {i, i, i}); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitDeleteLinksFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - std::filesystem::path split_index {"split_index.links"}, split_data {"split_data.links"}; - SplitMemoryLinks> storage { - FileMappedResizableDirectMemory{split_data.string()}, - FileMappedResizableDirectMemory{split_index.string()} - }; - auto fill = [&storage, &state] { - for (std::size_t i {}; i < state.range(0); ++i) { - CreatePoint(storage); - } - }; - Setup(storage); - for (auto _: state) { - state.PauseTiming(); - fill(); - state.ResumeTiming(); - for (std::uint64_t i = BACKGROUND_LINKS + state.range(0); i > BACKGROUND_LINKS; --i) { - Delete(storage, {i, i, i}); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitDeleteLinksRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory index {}, data {}; - SplitMemoryLinks, HeapResizableDirectMemory> storage {std::move(data), std::move(index)}; - auto fill = [&storage, &state] { - for (std::size_t i {}; i < state.range(0); ++i) { - CreatePoint(storage); - } - }; - Setup(storage); - for (auto _: state) { - state.PauseTiming(); - fill(); - state.ResumeTiming(); - for (std::uint64_t i = BACKGROUND_LINKS + state.range(0); i > BACKGROUND_LINKS; --i) { - Delete(storage, {i, i, i}); - } - } - Teardown(storage); -} - -BENCHMARK(BM_PSQLDeleteLinksWithoutTransaction)->Name("BM_PSQL/Delete/NonTransaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_PSQLDeleteLinksWithTransaction)->Name("BM_PSQL/Delete/Transaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedDeleteLinksFile)->Name("BM_Doublets/United/Delete/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedDeleteLinksRAM)->Name("BM_Doublets/United/Delete/Volatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitDeleteLinksFile)->Name("BM_Doublets/Split/Delete/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitDeleteLinksRAM)->Name("BM_Doublets/Split/Delete/Volatile")->Arg(1000)->MinWarmUpTime(20); diff --git a/cpp/Benchmarks/Each/All.cpp b/cpp/Benchmarks/Each/All.cpp deleted file mode 100644 index 343bfec..0000000 --- a/cpp/Benchmarks/Each/All.cpp +++ /dev/null @@ -1,121 +0,0 @@ -static void BM_PSQLEachAllWithoutTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - Client> table {options}; - auto any {table.Constants.Any}, $continue {table.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(table); - for (auto _: state) { - table.Each({any, any, any}, handler); - } - Teardown(table); -} - -static void BM_PSQLEachAllWithTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - std::uint64_t any {}, $continue {}; - { - Transaction> table {options}; - Setup(table); - any = table.Constants.Any; - $continue = table.Constants.Continue; - } - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - for (auto _: state) { - state.PauseTiming(); - Transaction> table {options}; - state.ResumeTiming(); - table.Each({any, any, any}, handler); - } - Transaction> table {options}; - Teardown(table); -} - -static void BM_DoubletsUnitedEachAllFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - std::filesystem::path path {"united.links"}; - UnitedMemoryLinks> storage {FileMappedResizableDirectMemory{path.string()}}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - storage.Each({any, any, any}, handler); - } - Teardown(storage); -} - -static void BM_DoubletsUnitedEachAllRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory memory {}; - UnitedMemoryLinks, HeapResizableDirectMemory> storage {std::move(memory)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - storage.Each({any, any, any}, handler); - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachAllFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - std::filesystem::path split_data {"split_data.links"}, split_index {"split_index.links"}; - SplitMemoryLinks> storage { - FileMappedResizableDirectMemory{split_data.string()}, - FileMappedResizableDirectMemory{split_index.string()} - }; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - storage.Each({any, any, any}, handler); - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachAllRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory data {}, index {}; - SplitMemoryLinks, HeapResizableDirectMemory> storage {std::move(data), std::move(index)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - storage.Each({any, any, any}, handler); - } - Teardown(storage); -} - -BENCHMARK(BM_PSQLEachAllWithoutTransaction)->Name("BM_PSQL/Each/All/NonTransaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_PSQLEachAllWithTransaction)->Name("BM_PSQL/Each/All/Transaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachAllFile)->Name("BM_Doublets/United/Each/All/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachAllRAM)->Name("BM_Doublets/United/Each/All/Volatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachAllFile)->Name("BM_Doublets/Split/Each/All/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachAllRAM)->Name("BM_Doublets/Split/Each/All/Volatile")->Arg(1000)->MinWarmUpTime(20); diff --git a/cpp/Benchmarks/Each/Concrete.cpp b/cpp/Benchmarks/Each/Concrete.cpp deleted file mode 100644 index 076780c..0000000 --- a/cpp/Benchmarks/Each/Concrete.cpp +++ /dev/null @@ -1,169 +0,0 @@ -static void BM_PSQLEachConcreteWithoutTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - Client> table {options}; - auto any {table.Constants.Any}, $continue {table.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(table); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - table.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - table.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - table.Each({any, i, i}, handler); - } - } - Teardown(table); -} - -static void BM_PSQLEachConcreteWithTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - std::uint64_t any {}, $continue {}; - { - Transaction> table {options}; - Setup(table); - any = table.Constants.Any; - $continue = table.Constants.Continue; - } - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - for (auto _: state) { - state.PauseTiming(); - Transaction> table {options}; - state.ResumeTiming(); - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - table.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - table.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - table.Each({any, i, i}, handler); - } - } - Transaction> table {options}; - Teardown(table); -} - -static void BM_DoubletsUnitedEachConcreteFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - std::filesystem::path path {"united.links"}; - UnitedMemoryLinks> storage {FileMappedResizableDirectMemory{path.string()}}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, i, i}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsUnitedEachConcreteRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory memory {}; - UnitedMemoryLinks, HeapResizableDirectMemory> storage {std::move(memory)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, i, i}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachConcreteFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - std::filesystem::path split_data {"split_data.links"}, split_index {"split_index.links"}; - SplitMemoryLinks> storage{ - FileMappedResizableDirectMemory{split_data.string()}, - FileMappedResizableDirectMemory{split_index.string()} - }; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, i, i}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachConcreteRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory data {}, index {}; - SplitMemoryLinks, HeapResizableDirectMemory> storage {std::move(data), std::move(index)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, i, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, i, i}, handler); - } - } - Teardown(storage); -} - -BENCHMARK(BM_PSQLEachConcreteWithoutTransaction)->Name("BM_PSQL/Each/Concrete/NonTransaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_PSQLEachConcreteWithTransaction)->Name("BM_PSQL/Each/Concrete/Transaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachConcreteFile)->Name("BM_Doublets/United/Each/Concrete/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachConcreteRAM)->Name("BM_Doublets/United/Each/Concrete/Volatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachConcreteFile)->Name("BM_Doublets/Split/Each/Concrete/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachConcreteRAM)->Name("BM_Doublets/Split/Each/Concrete/Volatile")->Arg(1000)->MinWarmUpTime(20); diff --git a/cpp/Benchmarks/Each/Identity.cpp b/cpp/Benchmarks/Each/Identity.cpp deleted file mode 100644 index 991d088..0000000 --- a/cpp/Benchmarks/Each/Identity.cpp +++ /dev/null @@ -1,170 +0,0 @@ -static void BM_PSQLEachIdentityWithoutTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - Client> table {options}; - auto any {table.Constants.Any}; - auto $continue {table.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(table); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - table.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - table.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - table.Each({i, any, any}, handler); - } - } - Teardown(table); -} - -static void BM_PSQLEachIdentityWithTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - std::uint64_t any {}, $continue {}; - { - Transaction> table {options}; - Setup(table); - any = table.Constants.Any; - $continue = table.Constants.Continue; - } - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - for (auto _: state) { - state.PauseTiming(); - Transaction> table {options}; - state.ResumeTiming(); - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - table.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - table.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - table.Each({i, any, any}, handler); - } - } - Transaction> table {options}; - Teardown(table); -} - -static void BM_DoubletsUnitedEachIdentityFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - std::filesystem::path path {"united.links"}; - UnitedMemoryLinks> storage {FileMappedResizableDirectMemory{path.string()}}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({i, any, any}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsUnitedEachIdentityRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory memory {}; - UnitedMemoryLinks, HeapResizableDirectMemory> storage(std::move(memory)); - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({i, any, any}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachIdentityFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - std::filesystem::path split_data {"split_data.links"}, split_index {"split_index.links"}; - SplitMemoryLinks> storage { - FileMappedResizableDirectMemory{split_data.string()}, - FileMappedResizableDirectMemory{split_index.string()} - }; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({i, any, any}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachIdentityRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory data {}, index {}; - SplitMemoryLinks, HeapResizableDirectMemory> storage {std::move(data), std::move(index)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({i, any, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({i, any, any}, handler); - } - } - Teardown(storage); -} - -BENCHMARK(BM_PSQLEachIdentityWithoutTransaction)->Name("BM_PSQL/Each/Identity/NonTransaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_PSQLEachIdentityWithTransaction)->Name("BM_PSQL/Each/Identity/Transaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachIdentityFile)->Name("BM_Doublets/United/Each/Identity/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachIdentityRAM)->Name("BM_Doublets/United/Each/Identity/Volatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachIdentityFile)->Name("BM_Doublets/Split/Each/Identity/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachIdentityRAM)->Name("BM_Doublets/Split/Each/Identity/Volatile")->Arg(1000)->MinWarmUpTime(20); \ No newline at end of file diff --git a/cpp/Benchmarks/Each/Incoming.cpp b/cpp/Benchmarks/Each/Incoming.cpp deleted file mode 100644 index 81b6b55..0000000 --- a/cpp/Benchmarks/Each/Incoming.cpp +++ /dev/null @@ -1,169 +0,0 @@ -static void BM_PSQLEachIncomingWithoutTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - Client> table {options}; - auto any {table.Constants.Any}, $continue {table.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(table); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - table.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - table.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - table.Each({any, any, i}, handler); - } - } - Teardown(table); -} - -static void BM_PSQLEachIncomingWithTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - std::uint64_t any {}, $continue {}; - { - Transaction> table {options}; - Setup(table); - any = table.Constants.Any; - $continue = table.Constants.Continue; - } - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - for (auto _: state) { - state.PauseTiming(); - Transaction> table {options}; - state.ResumeTiming(); - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - table.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - table.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - table.Each({any, any, i}, handler); - } - } - Transaction> table {options}; - Teardown(table); -} - -static void BM_DoubletsUnitedEachIncomingFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - std::filesystem::path path {"united.links"}; - UnitedMemoryLinks> storage {FileMappedResizableDirectMemory{path.string()}}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, any, i}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsUnitedEachIncomingRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory memory {}; - UnitedMemoryLinks, HeapResizableDirectMemory> storage {std::move(memory)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, any, i}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachIncomingFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - std::filesystem::path split_data {"split_data.links"}, split_index {"split_index.links"}; - SplitMemoryLinks> storage { - FileMappedResizableDirectMemory{split_data.string()}, - FileMappedResizableDirectMemory{split_index.string()} - }; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, any, i}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachIncomingRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory data {}, index {}; - SplitMemoryLinks, HeapResizableDirectMemory> storage {std::move(data), std::move(index)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, any, i}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, any, i}, handler); - } - } - Teardown(storage); -} - -BENCHMARK(BM_PSQLEachIncomingWithoutTransaction)->Name("BM_PSQL/Each/Incoming/NonTransaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_PSQLEachIncomingWithTransaction)->Name("BM_PSQL/Each/Incoming/Transaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachIncomingFile)->Name("BM_Doublets/United/Each/Incoming/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachIncomingRAM)->Name("BM_Doublets/United/Each/Incoming/Volatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachIncomingFile)->Name("BM_Doublets/Split/Each/Incoming/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachIncomingRAM)->Name("BM_Doublets/Split/Each/Incoming/Volatile")->Arg(1000)->MinWarmUpTime(20); diff --git a/cpp/Benchmarks/Each/Outgoing.cpp b/cpp/Benchmarks/Each/Outgoing.cpp deleted file mode 100644 index 90193a3..0000000 --- a/cpp/Benchmarks/Each/Outgoing.cpp +++ /dev/null @@ -1,169 +0,0 @@ -static void BM_PSQLEachOutgoingWithoutTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - Client> table {options}; - auto any {table.Constants.Any}, $continue {table.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(table); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - table.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - table.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - table.Each({any, i, any}, handler); - } - } - Teardown(table); -} - -static void BM_PSQLEachOutgoingWithTransaction(benchmark::State& state) { - using namespace SetupTeardown; - using namespace Platform::Data::Doublets; - using namespace PostgreSQL; - std::uint64_t any {}, $continue {}; - { - Transaction> table {options}; - Setup(table); - any = table.Constants.Any; - $continue = table.Constants.Continue; - } - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - for (auto _: state) { - state.PauseTiming(); - Transaction> table {options}; - state.ResumeTiming(); - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - table.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - table.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - table.Each({any, i, any}, handler); - } - } - Transaction> table {options}; - Teardown(table); -} - -static void BM_DoubletsUnitedEachOutgoingFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - std::filesystem::path path {"united.links"}; - UnitedMemoryLinks> storage {FileMappedResizableDirectMemory{path.string()}}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, i, any}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsUnitedEachOutgoingRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::United::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory memory {}; - UnitedMemoryLinks, HeapResizableDirectMemory> storage {std::move(memory)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, i, any}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachOutgoingFile(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - std::filesystem::path split_data {"split_data.links"}, split_index {"split_index.links"}; - SplitMemoryLinks> storage { - FileMappedResizableDirectMemory{split_data.string()}, - FileMappedResizableDirectMemory{split_index.string()} - }; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, i, any}, handler); - } - } - Teardown(storage); -} - -static void BM_DoubletsSplitEachOutgoingRAM(benchmark::State& state) { - using namespace Platform::Memory; - using namespace Platform::Data::Doublets; - using namespace Memory::Split::Generic; - using namespace SetupTeardown; - HeapResizableDirectMemory data {}, index {}; - SplitMemoryLinks, HeapResizableDirectMemory> storage {std::move(data), std::move(index)}; - auto any {storage.Constants.Any}, $continue {storage.Constants.Continue}; - auto handler = [$continue] (std::vector vec) { - return $continue; - }; - Setup(storage); - for (auto _: state) { - for (std::uint64_t i = 1; i <= state.range(0); ++i) { - storage.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS/2 - state.range(0)/2 + 1; i <= BACKGROUND_LINKS/2 + state.range(0)/2; ++i) { - storage.Each({any, i, any}, handler); - } - for (std::uint64_t i = BACKGROUND_LINKS - state.range(0) + 1; i <= BACKGROUND_LINKS; ++i) { - storage.Each({any, i, any}, handler); - } - } - Teardown(storage); -} - -BENCHMARK(BM_PSQLEachOutgoingWithoutTransaction)->Name("BM_PSQL/Each/Outgoing/NonTransaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_PSQLEachOutgoingWithTransaction)->Name("BM_PSQL/Each/Outgoing/Transaction")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachOutgoingFile)->Name("BM_Doublets/United/Each/Outgoing/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsUnitedEachOutgoingRAM)->Name("BM_Doublets/United/Each/Outgoing/Volatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachOutgoingFile)->Name("BM_Doublets/Split/Each/Outgoing/NonVolatile")->Arg(1000)->MinWarmUpTime(20); -BENCHMARK(BM_DoubletsSplitEachOutgoingRAM)->Name("BM_Doublets/Split/Each/Outgoing/Volatile")->Arg(1000)->MinWarmUpTime(20); diff --git a/cpp/Benchmarks/SetupTeardown.cpp b/cpp/Benchmarks/SetupTeardown.cpp deleted file mode 100644 index 4f0dec2..0000000 --- a/cpp/Benchmarks/SetupTeardown.cpp +++ /dev/null @@ -1,28 +0,0 @@ -namespace SetupTeardown -{ - - template - void Setup(TStorage& storage) { - using namespace Platform::Data::Doublets; - for (std::uint64_t i {}; i < BACKGROUND_LINKS; ++i) { - CreatePoint(storage); - } - } - - template