From 60cabd638b94b151db9ae9a6f05fd1f0df58b4b4 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Fri, 11 Sep 2026 22:08:42 +0200 Subject: [PATCH 01/17] feat(demo): add vortex-server + vortex-demo modules for a live HTTP range demo Two new standalone, non-published modules: - vortex-server: a minimal, dependency-free object-storage HTTP server (GET with byte-range support, HEAD, PUT, a plain-text object listing, a /_stats endpoint) standing in for real object storage in a demo. Exists because the JDK's own jwebserver (SimpleFileServer) does not implement Range requests at all -- VortexHttpReader's first segment fetch against it throws VortexException since the returned byte count never matches the requested range. Path-traversal guarded and covered by regression tests using literal (non-normalized) URIs, verified against both GET and PUT with raw sockets during development. - vortex-demo: generates a synthetic tick dataset (timestamp/symbol/ price/volume, sorted by symbol), uploads it to a vortex-server (embedded by default, or an already-running one given on the command line for a two-process live demo), then runs a filtered + projected VortexHttpReader scan against it and reports how many bytes were actually fetched over HTTP against the object's full size. Along the way, found a real gap in the reader (not fixed here): the default WriteOptions.globalDict=true silently defeats RowFilter zone-map pruning for Utf8 columns, because ScanIterator#canPruneChunk reads per-chunk embedded stats that the global-dict write path never populates -- the separate zone-map stats table stays correct and is readable via ScanIterator#columnZoneStats, but pruning doesn't consult it. The demo works around it via WriteOptions#withGlobalDict(false). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- demo/pom.xml | 92 ++++++ .../dfa1/vortex/demo/DemoDataGenerator.java | 103 ++++++ .../dfa1/vortex/demo/HttpRangeDemo.java | 125 ++++++++ pom.xml | 7 + server/pom.xml | 81 +++++ .../dfa1/vortex/server/VortexServer.java | 299 ++++++++++++++++++ .../dfa1/vortex/server/VortexServerTest.java | 194 ++++++++++++ 7 files changed, 901 insertions(+) create mode 100644 demo/pom.xml create mode 100644 demo/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java create mode 100644 demo/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java create mode 100644 server/pom.xml create mode 100644 server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java create mode 100644 server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java diff --git a/demo/pom.xml b/demo/pom.xml new file mode 100644 index 00000000..f2659db8 --- /dev/null +++ b/demo/pom.xml @@ -0,0 +1,92 @@ + + + 4.0.0 + + io.github.dfa1.vortex + vortex-java + 0.14.2-SNAPSHOT + + + vortex-demo + + vortex-demo + Standalone demo client: generates a synthetic tick dataset, uploads it to a + vortex-server (embedded by default, or an already-running one given on the command line), + then runs a filtered/projected VortexHttpReader scan against it -- showing that the scan + fetches only the bytes it touches instead of downloading the whole object. + + + + true + true + + true + + + + + + io.github.dfa1.vortex + vortex-writer + + + io.github.dfa1.vortex + vortex-reader + + + io.github.dfa1.vortex + vortex-server + + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + vortex-demo + + + io.github.dfa1.vortex.demo.HttpRangeDemo + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/demo/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java b/demo/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java new file mode 100644 index 00000000..8073185b --- /dev/null +++ b/demo/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java @@ -0,0 +1,103 @@ +package io.github.dfa1.vortex.demo; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Random; + +/// Generates a synthetic tick-style dataset — `timestamp`/`symbol`/`price`/`volume` — sorted by +/// symbol so each chunk is dominated by a single symbol. +/// +/// That clustering is what makes [HttpRangeDemo]'s single-symbol filter dramatic: zone-map +/// pruning skips whole chunks whose stats can't match, so [RangeAwareFileServer] only ever +/// receives range requests for the handful of chunks holding the requested symbol. A real-world +/// archive partitioned or sorted by instrument (a common layout for exactly this kind of data) +/// behaves the same way. +final class DemoDataGenerator { + + static final String TIMESTAMP = "timestamp"; + static final String SYMBOL = "symbol"; + static final String PRICE = "price"; + static final String VOLUME = "volume"; + + private static final long BASE_TIMESTAMP_MILLIS = 1_700_000_000_000L; + private static final long SEED = 42L; + + private DemoDataGenerator() { + } + + /// Writes `rowCount` rows spread evenly across `symbolCount` symbols to `out`, in chunks of + /// `chunkSize` rows. + /// + /// @param out destination path; created or truncated + /// @param rowCount total rows to generate + /// @param symbolCount distinct symbols; rows are grouped by symbol, in order + /// @param chunkSize rows per written chunk + /// @throws IOException if writing the file fails + static void generate(Path out, long rowCount, int symbolCount, int chunkSize) throws IOException { + DType.Struct schema = DType.structBuilder() + .field(TIMESTAMP, DType.I64) + .field(SYMBOL, DType.UTF8) + .field(PRICE, DType.F64) + .field(VOLUME, DType.I64) + .build(); + + ColumnName timestampCol = ColumnName.of(TIMESTAMP); + ColumnName symbolCol = ColumnName.of(SYMBOL); + ColumnName priceCol = ColumnName.of(PRICE); + ColumnName volumeCol = ColumnName.of(VOLUME); + + String[] symbols = new String[symbolCount]; + for (int i = 0; i < symbolCount; i++) { + symbols[i] = "SYM%03d".formatted(i); + } + long rowsPerSymbol = Math.max(1, rowCount / symbolCount); + + // globalDict(false): the default global dictionary pulls a low-cardinality Utf8 column's + // codes out of the normal per-chunk cascade, and the per-chunk embedded stats + // RowFilter-driven zone-map pruning reads (ScanIterator#canPruneChunk) go unpopulated for + // those codes — the *separate* per-zone stats table (ScanIterator#columnZoneStats) stays + // correct either way, but pruning doesn't consult it. With the default globalDict=true, + // this demo's single-symbol filter provably matched zero chunks. Disabling it here keeps + // "symbol" in the ordinary per-chunk dict cascade, which does emit per-chunk min/max. + Random random = new Random(SEED); + try (FileChannel channel = FileChannel.open(out, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + VortexWriter writer = VortexWriter.create(channel, schema, WriteOptions.cascading(3).withGlobalDict(false))) { + + long written = 0; + int symbolIndex = 0; + long rowInSymbol = 0; + while (written < rowCount) { + int n = (int) Math.min(chunkSize, rowCount - written); + long[] timestamps = new long[n]; + String[] symbolValues = new String[n]; + double[] prices = new double[n]; + long[] volumes = new long[n]; + for (int i = 0; i < n; i++) { + if (rowInSymbol >= rowsPerSymbol && symbolIndex < symbolCount - 1) { + symbolIndex++; + rowInSymbol = 0; + } + double basePrice = 50 + symbolIndex * 3.7; + timestamps[i] = BASE_TIMESTAMP_MILLIS + (written + i) * 1000L; + symbolValues[i] = symbols[symbolIndex]; + prices[i] = basePrice + random.nextDouble() * 2 - 1; + volumes[i] = 100 + random.nextInt(10_000); + rowInSymbol++; + } + writer.writeChunk(c -> c.put(timestampCol, timestamps) + .put(symbolCol, symbolValues) + .put(priceCol, prices) + .put(volumeCol, volumes)); + written += n; + } + } + } +} diff --git a/demo/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java b/demo/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java new file mode 100644 index 00000000..2d3345a0 --- /dev/null +++ b/demo/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java @@ -0,0 +1,125 @@ +package io.github.dfa1.vortex.demo; + +import io.github.dfa1.vortex.reader.RowFilter; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexHttpReader; +import io.github.dfa1.vortex.server.VortexServer; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/// Demo client: generates a synthetic tick dataset, uploads it to a `vortex-server` object +/// store, then runs a filtered/projected [VortexHttpReader] scan against it — printing how many +/// bytes the scan actually pulled over the wire against the object's full size. +/// +/// Run standalone (embeds its own [VortexServer]): +/// ``` +/// java -jar demo/target/vortex-demo.jar +/// ``` +/// +/// Or against an already-running server — e.g. `java -jar server/target/vortex-server.jar` in a +/// separate terminal, for a two-process live demo: +/// ``` +/// java -jar demo/target/vortex-demo.jar http://127.0.0.1:8080/ +/// ``` +public final class HttpRangeDemo { + + private static final long ROW_COUNT = 2_000_000; + private static final int SYMBOL_COUNT = 30; + private static final int CHUNK_SIZE = 65_536; + private static final String QUERY_SYMBOL = "SYM015"; + private static final String OBJECT_KEY = "trades.vortex"; + + private static final Pattern BYTES_SERVED_LINE = Pattern.compile("bytesServed=(\\d+)"); + + private HttpRangeDemo() { + } + + /// @param args optional: base URI of an already-running `vortex-server`; omitted embeds one + /// @throws IOException if the demo file, server, upload, or scan fails + /// @throws InterruptedException if interrupted while starting the embedded server or sending + /// an HTTP request + public static void main(String[] args) throws IOException, InterruptedException { + Path localFile = Files.createTempDirectory("vortex-demo").resolve(OBJECT_KEY); + System.out.println("Generating demo file (" + ROW_COUNT + " rows, " + SYMBOL_COUNT + " symbols)..."); + DemoDataGenerator.generate(localFile, ROW_COUNT, SYMBOL_COUNT, CHUNK_SIZE); + long fileSize = Files.size(localFile); + System.out.printf("Wrote %s (%.1f MB)%n%n", localFile, fileSize / 1_000_000.0); + + if (args.length > 0) { + runAgainst(URI.create(args[0]), localFile, fileSize); + } else { + try (VortexServer server = VortexServer.start(Files.createTempDirectory("vortex-server"), 0)) { + System.out.println("Embedded vortex-server at " + server.baseUri()); + runAgainst(server.baseUri(), localFile, fileSize); + } + } + } + + private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize) + throws IOException, InterruptedException { + HttpClient client = HttpClient.newHttpClient(); + URI objectUri = serverBaseUri.resolve(OBJECT_KEY); + + System.out.println("Uploading to " + objectUri + " ..."); + upload(client, objectUri, localFile); + + long bytesServedBefore = readBytesServed(client, serverBaseUri); + System.out.printf("%nScanning for %s=%s, projecting '%s' over HTTP...%n%n", + DemoDataGenerator.SYMBOL, QUERY_SYMBOL, DemoDataGenerator.PRICE); + long rows = scan(objectUri); + long bytesServedAfter = readBytesServed(client, serverBaseUri); + + long servedDuringScan = bytesServedAfter - bytesServedBefore; + System.out.println(); + System.out.println("Matched rows: " + rows); + System.out.printf("Bytes fetched over HTTP during the scan: %,d / %,d (%.2f%% of the object)%n", + servedDuringScan, fileSize, 100.0 * servedDuringScan / fileSize); + } + + private static void upload(HttpClient client, URI objectUri, Path localFile) + throws IOException, InterruptedException { + HttpRequest put = HttpRequest.newBuilder(objectUri) + .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) + .build(); + HttpResponse response = client.send(put, HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() != 201) { + throw new IOException("upload failed: HTTP " + response.statusCode()); + } + } + + private static long readBytesServed(HttpClient client, URI serverBaseUri) + throws IOException, InterruptedException { + HttpRequest req = HttpRequest.newBuilder(serverBaseUri.resolve("_stats")).GET().build(); + HttpResponse response = client.send(req, HttpResponse.BodyHandlers.ofString()); + Matcher m = BYTES_SERVED_LINE.matcher(response.body()); + if (!m.find()) { + throw new IOException("could not parse /_stats response: " + response.body()); + } + return Long.parseLong(m.group(1)); + } + + private static long scan(URI objectUri) throws IOException { + ScanOptions opts = ScanOptions.all() + .withColumns(DemoDataGenerator.SYMBOL, DemoDataGenerator.PRICE) + .withFilter(RowFilter.eq(DemoDataGenerator.SYMBOL, QUERY_SYMBOL)); + + long rows = 0; + try (VortexHttpReader vf = VortexHttpReader.open(objectUri); + var iter = vf.scan(opts)) { + while (iter.hasNext()) { + try (var chunk = iter.next()) { + rows += chunk.rowCount(); + } + } + } + return rows; + } +} diff --git a/pom.xml b/pom.xml index 0ec48e4b..df6f7b68 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,8 @@ cli inspector calcite + server + demo integration fuzz performance @@ -193,6 +195,11 @@ vortex-calcite ${project.version} + + io.github.dfa1.vortex + vortex-server + ${project.version} + de.siegmar fastcsv diff --git a/server/pom.xml b/server/pom.xml new file mode 100644 index 00000000..507f6296 --- /dev/null +++ b/server/pom.xml @@ -0,0 +1,81 @@ + + + 4.0.0 + + io.github.dfa1.vortex + vortex-java + 0.14.2-SNAPSHOT + + + vortex-server + + vortex-server + Standalone, minimal object-storage HTTP server: GET (byte-range aware), HEAD, PUT, + and a plain-text object listing over one local directory. Exists to demonstrate + VortexHttpReader's partial-fetch story against something closer to real object storage + than a static file server: the JDK's own jwebserver does not implement byte-range + requests at all. Depends on nothing but the JDK: no Vortex module, so it is reusable + outside this project too. + + + + true + true + + true + + + + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + vortex-server + + + io.github.dfa1.vortex.server.VortexServer + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java b/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java new file mode 100644 index 00000000..7e16069d --- /dev/null +++ b/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java @@ -0,0 +1,299 @@ +package io.github.dfa1.vortex.server; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +/// Minimal, dependency-free object-storage HTTP server: `GET`/`HEAD` a key (honoring byte-range +/// requests), `PUT` a key, and list the store — just enough surface to stand in for a real object +/// store (S3, Azure Blob, GCS) in a demo. +/// +/// The JDK's own `jwebserver` (`com.sun.net.httpserver.SimpleFileServer`) does not implement +/// `Range` at all — it always returns the whole object. `VortexHttpReader` (in `vortex-reader`) +/// depends entirely on working `206 Partial Content` responses to fetch only the segments a scan +/// actually touches; against a non-Range server its first segment fetch throws `VortexException` +/// because the returned byte count never matches the requested range. This class is the minimum +/// needed to demonstrate partial fetching over plain HTTP without a real cloud account. +/// +/// One flat key space, backed by one local directory — a key maps directly to a file under +/// [#dataDir]. Binds to loopback only, single range per request, no auth, no multipart, no +/// versioning. Not hardened for untrusted networks; demo/local use only. Depends on nothing but +/// the JDK, so it is reusable outside this project. +public final class VortexServer implements AutoCloseable { + + private final HttpServer server; + private final Path dataDir; + private final AtomicLong bytesServed = new AtomicLong(); + private final AtomicLong bytesReceived = new AtomicLong(); + + private VortexServer(HttpServer server, Path dataDir) { + this.server = server; + this.dataDir = dataDir; + } + + /// Starts a server storing objects under `dataDir`, bound to loopback on `port`. + /// + /// @param dataDir directory backing the object store; created if absent + /// @param port port to bind, or `0` to let the OS choose an ephemeral port + /// @return a running server; call [#close] to stop it + /// @throws IOException if `dataDir` cannot be created or the server socket cannot be bound + public static VortexServer start(Path dataDir, int port) throws IOException { + Files.createDirectories(dataDir); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", port), 0); + VortexServer instance = new VortexServer(server, dataDir); + server.createContext("/", instance::handle); + server.start(); + return instance; + } + + /// Runs a standalone server. Args: `[port] [dataDir]`, both optional — an omitted port picks + /// an ephemeral one, an omitted data directory creates a fresh temp directory. Blocks until + /// interrupted (`Ctrl+C`). + /// + /// @param args `[port] [dataDir]` + /// @throws IOException if the server cannot start + /// @throws InterruptedException if interrupted while blocking + public static void main(String[] args) throws IOException, InterruptedException { + int port = args.length > 0 ? Integer.parseInt(args[0]) : 0; + Path dataDir = args.length > 1 ? Path.of(args[1]) : Files.createTempDirectory("vortex-server"); + try (VortexServer server = start(dataDir, port)) { + System.out.println("vortex-server listening on " + server.baseUri()); + System.out.println("Serving objects from " + dataDir.toAbsolutePath()); + System.out.println("Press Ctrl+C to stop."); + Thread.currentThread().join(); + } + } + + /// @return the port this server is actually bound to + public int port() { + return server.getAddress().getPort(); + } + + /// @return the server's base URI, e.g. `http://127.0.0.1:PORT/` + public URI baseUri() { + return URI.create("http://127.0.0.1:" + port() + "/"); + } + + /// @return the total response-body bytes served (`GET`/`HEAD`) since this server started + public long bytesServed() { + return bytesServed.get(); + } + + /// @return the total request-body bytes received (`PUT`) since this server started + public long bytesReceived() { + return bytesReceived.get(); + } + + @Override + public void close() { + server.stop(0); + } + + private void handle(HttpExchange exchange) throws IOException { + try { + switch (exchange.getRequestMethod()) { + case "GET", "HEAD" -> handleGet(exchange); + case "PUT" -> handlePut(exchange); + default -> exchange.sendResponseHeaders(405, -1); + } + } finally { + exchange.close(); + } + } + + private void handleGet(HttpExchange exchange) throws IOException { + boolean sendBody = exchange.getRequestMethod().equals("GET"); + URI uri = exchange.getRequestURI(); + if (uri.getPath().equals("/_stats")) { + handleStats(exchange, sendBody); + return; + } + if (uri.getPath().equals("/")) { + handleList(exchange, sendBody); + return; + } + + Path file = resolve(uri); + if (file == null || !Files.isRegularFile(file)) { + exchange.sendResponseHeaders(404, -1); + return; + } + + long totalLength = Files.size(file); + exchange.getResponseHeaders().add("Accept-Ranges", "bytes"); + exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); + + String rangeHeader = exchange.getRequestHeaders().getFirst("Range"); + Optional range = parseRange(rangeHeader, totalLength); + if (rangeHeader != null && range.isEmpty()) { + exchange.getResponseHeaders().add("Content-Range", "bytes */" + totalLength); + exchange.sendResponseHeaders(416, -1); + return; + } + + long start = range.map(ByteRange::start).orElse(0L); + long end = range.map(ByteRange::end).orElse(totalLength - 1); + long served = totalLength == 0 ? 0 : end - start + 1; + + if (range.isPresent()) { + exchange.getResponseHeaders().add("Content-Range", "bytes %d-%d/%d".formatted(start, end, totalLength)); + } + exchange.sendResponseHeaders(range.isPresent() ? 206 : 200, sendBody ? served : -1); + if (sendBody) { + writeRange(file, start, served, exchange.getResponseBody()); + } + bytesServed.addAndGet(served); + log("%-4s %-40s range=%-20s served=%,10d / %,10d bytes (%5.1f%%)".formatted( + exchange.getRequestMethod(), uri, rangeHeader == null ? "(none, full object)" : rangeHeader, + served, totalLength, totalLength == 0 ? 0 : 100.0 * served / totalLength)); + } + + private void handleList(HttpExchange exchange, boolean sendBody) throws IOException { + StringBuilder body = new StringBuilder(); + try (Stream entries = Files.list(dataDir)) { + entries.filter(Files::isRegularFile).sorted().forEach(p -> { + try { + body.append(p.getFileName()).append('\t').append(Files.size(p)).append('\n'); + } catch (IOException e) { + throw new java.io.UncheckedIOException(e); + } + }); + } + byte[] bytes = body.toString().getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=utf-8"); + exchange.sendResponseHeaders(200, sendBody ? bytes.length : -1); + if (sendBody) { + exchange.getResponseBody().write(bytes); + } + bytesServed.addAndGet(bytes.length); + } + + /// Reports [#bytesServed] and [#bytesReceived] as plain text, unaffected by this response's + /// own size — a demo client (possibly a separate process, unable to read these counters + /// directly) polls this to measure exactly how many bytes a scan pulled over the wire. + private void handleStats(HttpExchange exchange, boolean sendBody) throws IOException { + byte[] bytes = "bytesServed=%d%nbytesReceived=%d%n".formatted(bytesServed.get(), bytesReceived.get()) + .getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=utf-8"); + exchange.sendResponseHeaders(200, sendBody ? bytes.length : -1); + if (sendBody) { + exchange.getResponseBody().write(bytes); + } + } + + private void handlePut(HttpExchange exchange) throws IOException { + Path file = resolve(exchange.getRequestURI()); + if (file == null) { + exchange.sendResponseHeaders(400, -1); + return; + } + Files.createDirectories(file.getParent()); + long received; + try (InputStream in = exchange.getRequestBody(); + OutputStream out = Files.newOutputStream(file, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + received = in.transferTo(out); + } + bytesReceived.addAndGet(received); + log("PUT %-40s received=%,d bytes -> %s".formatted(exchange.getRequestURI(), received, file)); + exchange.sendResponseHeaders(201, -1); + } + + private Path resolve(URI requestUri) { + String path = requestUri.getPath(); + String relative = path.startsWith("/") ? path.substring(1) : path; + if (relative.isEmpty()) { + return null; + } + Path candidate = dataDir.resolve(relative).normalize(); + return candidate.startsWith(dataDir) ? candidate : null; + } + + private static void writeRange(Path file, long offset, long length, OutputStream out) throws IOException { + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { + channel.position(offset); + ByteBuffer buffer = ByteBuffer.allocate(8192); + long remaining = length; + while (remaining > 0) { + buffer.clear(); + buffer.limit((int) Math.min(buffer.capacity(), remaining)); + int n = channel.read(buffer); + if (n < 0) { + break; + } + buffer.flip(); + out.write(buffer.array(), buffer.arrayOffset(), buffer.remaining()); + remaining -= n; + } + } + } + + private static void log(String message) { + System.out.println(" " + message); + } + + /// An inclusive byte range, already resolved against a known total length. + /// + /// @param start first byte offset served (inclusive) + /// @param end last byte offset served (inclusive) + private record ByteRange(long start, long end) { + } + + /// Parses a `Range: bytes=...` header value. Supports `bytes=X-Y`, `bytes=X-` (open-ended), + /// and the suffix form `bytes=-N` (last `N` bytes). Multi-range requests (comma-separated) + /// and malformed values are reported as empty, distinct from a missing header, so the caller + /// can tell "no Range header" (serve the whole object) apart from "Range header present but + /// unsatisfiable" (`416`). + /// + /// @param header the raw `Range` header value, or `null` if absent + /// @param totalLength the object's total length in bytes + /// @return the resolved range, or empty if `header` is `null`, malformed, multi-range, or + /// out of bounds + private static Optional parseRange(String header, long totalLength) { + if (header == null || !header.startsWith("bytes=") || header.contains(",")) { + return Optional.empty(); + } + String spec = header.substring("bytes=".length()); + int dash = spec.indexOf('-'); + if (dash < 0) { + return Optional.empty(); + } + String startPart = spec.substring(0, dash); + String endPart = spec.substring(dash + 1); + try { + long start; + long end; + if (startPart.isEmpty()) { + if (endPart.isEmpty()) { + return Optional.empty(); + } + long suffixLength = Long.parseLong(endPart); + start = Math.max(0, totalLength - suffixLength); + end = totalLength - 1; + } else { + start = Long.parseLong(startPart); + end = endPart.isEmpty() ? totalLength - 1 : Math.min(Long.parseLong(endPart), totalLength - 1); + } + if (start < 0 || start > end || start >= totalLength) { + return Optional.empty(); + } + return Optional.of(new ByteRange(start, end)); + } catch (NumberFormatException e) { + return Optional.empty(); + } + } +} diff --git a/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java b/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java new file mode 100644 index 00000000..3686da9b --- /dev/null +++ b/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java @@ -0,0 +1,194 @@ +package io.github.dfa1.vortex.server; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class VortexServerTest { + + private static final String CONTENT = "0123456789ABCDEFGHIJ"; // 20 bytes, index-addressable + + private final HttpClient client = HttpClient.newHttpClient(); + private VortexServer sut; + private URI objectUri; + private Path secretDir; + + @BeforeEach + void startServer(@TempDir Path dataDir) throws IOException { + Files.writeString(dataDir.resolve("data.bin"), CONTENT, StandardCharsets.US_ASCII); + sut = VortexServer.start(dataDir, 0); + objectUri = sut.baseUri().resolve("data.bin"); + + // A sibling of dataDir, one "../" away -- deterministic regardless of how deep the OS + // places @TempDir, unlike a fixed-depth traversal toward the filesystem root. + secretDir = dataDir.resolveSibling("vortex-server-secret-" + System.nanoTime()); + Files.createDirectories(secretDir); + Files.writeString(secretDir.resolve("secret.txt"), "TOP SECRET", StandardCharsets.US_ASCII); + } + + @AfterEach + void stopServer() throws IOException { + sut.close(); + Files.deleteIfExists(secretDir.resolve("secret.txt")); + Files.deleteIfExists(secretDir.resolve("pwned.txt")); + Files.deleteIfExists(secretDir); + } + + @Test + void servesTheWholeObjectWhenNoRangeHeaderIsSent() throws Exception { + // Given no Range header + + // When + HttpResponse result = get(objectUri, null); + + // Then + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).isEqualTo(CONTENT); + assertThat(sut.bytesServed()).isEqualTo(CONTENT.length()); + } + + @Test + void servesAByteRange() throws Exception { + // Given a "bytes=2-5" range + + // When + HttpResponse result = get(objectUri, "bytes=2-5"); + + // Then + assertThat(result.statusCode()).isEqualTo(206); + assertThat(result.body()).isEqualTo("2345"); + assertThat(result.headers().firstValue("Content-Range")).contains("bytes 2-5/20"); + } + + @Test + void servesASuffixRange() throws Exception { + // Given a "bytes=-3" (last 3 bytes) range + + // When + HttpResponse result = get(objectUri, "bytes=-3"); + + // Then + assertThat(result.statusCode()).isEqualTo(206); + assertThat(result.body()).isEqualTo("HIJ"); + } + + @Test + void rejectsAnOutOfBoundsRangeWith416() throws Exception { + // Given a range starting past the end of the object + + // When + HttpResponse result = get(objectUri, "bytes=100-200"); + + // Then + assertThat(result.statusCode()).isEqualTo(416); + } + + @Test + void rejectsPathTraversalOnGet() throws Exception { + // Given a key that escapes dataDir via "../" into a sibling directory. URI.create on a + // full literal string preserves the ".." verbatim (unlike URI#resolve, which performs + // RFC 3986 dot-segment normalization and would clamp it to the root before the request + // is even sent) -- this genuinely exercises the server's own guard, not the client's. + URI escaping = URI.create("http://127.0.0.1:" + sut.port() + "/../" + secretDir.getFileName() + "/secret.txt"); + + // When + HttpResponse result = get(escaping, null); + + // Then + assertThat(result.statusCode()).isEqualTo(404); + assertThat(result.body()).doesNotContain("TOP SECRET"); + } + + @Test + void rejectsPathTraversalOnPut() throws Exception { + // Given a PUT whose key escapes dataDir via "../" into a sibling directory (see the + // URI.create note above -- avoids URI#resolve's client-side dot-segment normalization) + URI escaping = URI.create("http://127.0.0.1:" + sut.port() + "/../" + secretDir.getFileName() + "/pwned.txt"); + HttpRequest put = HttpRequest.newBuilder(escaping) + .PUT(HttpRequest.BodyPublishers.ofString("pwned")) + .build(); + + // When + HttpResponse result = client.send(put, HttpResponse.BodyHandlers.discarding()); + + // Then + assertThat(result.statusCode()).isNotEqualTo(201); + assertThat(Files.exists(secretDir.resolve("pwned.txt"))).isFalse(); + } + + @Test + void returns404ForAMissingObject() throws Exception { + // Given a key with no matching object + URI missing = sut.baseUri().resolve("nope.bin"); + + // When + HttpResponse result = get(missing, null); + + // Then + assertThat(result.statusCode()).isEqualTo(404); + } + + @Test + void putThenGetRoundTrips() throws Exception { + // Given a PUT of a new object + URI uploaded = sut.baseUri().resolve("uploaded.bin"); + HttpRequest put = HttpRequest.newBuilder(uploaded) + .PUT(HttpRequest.BodyPublishers.ofString("hello object storage")) + .build(); + + // When + HttpResponse putResult = client.send(put, HttpResponse.BodyHandlers.discarding()); + HttpResponse getResult = get(uploaded, null); + + // Then + assertThat(putResult.statusCode()).isEqualTo(201); + assertThat(getResult.statusCode()).isEqualTo(200); + assertThat(getResult.body()).isEqualTo("hello object storage"); + assertThat(sut.bytesReceived()).isEqualTo("hello object storage".length()); + } + + @Test + void reportsStatsIndependentlyOfItsOwnResponseSize() throws Exception { + // Given a prior GET that served 4 bytes + get(objectUri, "bytes=2-5"); + + // When + HttpResponse result = get(sut.baseUri().resolve("_stats"), null); + + // Then + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).contains("bytesServed=4").contains("bytesReceived=0"); + } + + @Test + void listsStoredObjects() throws Exception { + // Given the pre-seeded "data.bin" object + + // When + HttpResponse result = get(sut.baseUri(), null); + + // Then + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).contains("data.bin\t" + CONTENT.length()); + } + + private HttpResponse get(URI uri, String rangeHeader) throws Exception { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri).GET(); + if (rangeHeader != null) { + builder.header("Range", rangeHeader); + } + return client.send(builder.build(), HttpResponse.BodyHandlers.ofString()); + } +} From 5edafd268a5c3fdd4583296e49c49ab56c169782 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 08:27:41 +0200 Subject: [PATCH 02/17] refactor(demo): nest vortex-server under demo/ as a sibling of the client The server is conceptually part of the HTTP range demo, not a separate top-level concern -- demo/ is now a small aggregator (vortex-demo-parent, packaging=pom) with two children: demo/server (vortex-server, unchanged) and demo/client (vortex-demo, unchanged). Artifact coordinates, package names, and behavior are unchanged; only the directory layout moved. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- demo/client/pom.xml | 93 +++++++++++++++++++ .../dfa1/vortex/demo/DemoDataGenerator.java | 0 .../dfa1/vortex/demo/HttpRangeDemo.java | 0 demo/pom.xml | 85 +++-------------- {server => demo/server}/pom.xml | 1 + .../dfa1/vortex/server/VortexServer.java | 0 .../dfa1/vortex/server/VortexServerTest.java | 0 pom.xml | 1 - 8 files changed, 105 insertions(+), 75 deletions(-) create mode 100644 demo/client/pom.xml rename demo/{ => client}/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java (100%) rename demo/{ => client}/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java (100%) rename {server => demo/server}/pom.xml (98%) rename {server => demo/server}/src/main/java/io/github/dfa1/vortex/server/VortexServer.java (100%) rename {server => demo/server}/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java (100%) diff --git a/demo/client/pom.xml b/demo/client/pom.xml new file mode 100644 index 00000000..de6d3f03 --- /dev/null +++ b/demo/client/pom.xml @@ -0,0 +1,93 @@ + + + 4.0.0 + + io.github.dfa1.vortex + vortex-java + 0.14.2-SNAPSHOT + ../../pom.xml + + + vortex-demo + + vortex-demo + Standalone demo client: generates a synthetic tick dataset, uploads it to a + vortex-server (embedded by default, or an already-running one given on the command line), + then runs a filtered/projected VortexHttpReader scan against it -- showing that the scan + fetches only the bytes it touches instead of downloading the whole object. + + + + true + true + + true + + + + + + io.github.dfa1.vortex + vortex-writer + + + io.github.dfa1.vortex + vortex-reader + + + io.github.dfa1.vortex + vortex-server + + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + vortex-demo + + + io.github.dfa1.vortex.demo.HttpRangeDemo + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/demo/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java similarity index 100% rename from demo/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java rename to demo/client/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java diff --git a/demo/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java similarity index 100% rename from demo/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java rename to demo/client/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java diff --git a/demo/pom.xml b/demo/pom.xml index f2659db8..1152980f 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -7,86 +7,23 @@ 0.14.2-SNAPSHOT - vortex-demo + vortex-demo-parent + pom - vortex-demo - Standalone demo client: generates a synthetic tick dataset, uploads it to a - vortex-server (embedded by default, or an already-running one given on the command line), - then runs a filtered/projected VortexHttpReader scan against it -- showing that the scan - fetches only the bytes it touches instead of downloading the whole object. + vortex-demo-parent + Aggregator for the standalone HTTP range demo: vortex-server (a minimal + object-storage HTTP server) and vortex-demo (the client that generates data, uploads it, + and runs a filtered/projected VortexHttpReader scan against it). - + true true - true - - - - io.github.dfa1.vortex - vortex-writer - - - io.github.dfa1.vortex - vortex-reader - - - io.github.dfa1.vortex - vortex-server - - - - org.junit.jupiter - junit-jupiter - test - - - org.assertj - assertj-core - test - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - vortex-demo - - - io.github.dfa1.vortex.demo.HttpRangeDemo - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - + + server + client + diff --git a/server/pom.xml b/demo/server/pom.xml similarity index 98% rename from server/pom.xml rename to demo/server/pom.xml index 507f6296..68c6ac49 100644 --- a/server/pom.xml +++ b/demo/server/pom.xml @@ -5,6 +5,7 @@ io.github.dfa1.vortex vortex-java 0.14.2-SNAPSHOT + ../../pom.xml vortex-server diff --git a/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java b/demo/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java similarity index 100% rename from server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java rename to demo/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java diff --git a/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java b/demo/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java similarity index 100% rename from server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java rename to demo/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java diff --git a/pom.xml b/pom.xml index df6f7b68..9d063ac1 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,6 @@ cli inspector calcite - server demo integration fuzz From c510a4400dac85df6fd94c441063359bc671f2c8 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 08:55:47 +0200 Subject: [PATCH 03/17] feat(demo): add vortex-fakedata-generator, nest all demo packages under vortex.demo.* Splits the client's ad-hoc synthetic-data generation into its own standalone CLI tool, vortex-fakedata-generator: given a compact column-description grammar (name:type:generator(args), e.g. "price:f64:range(50,150)"), inspired by PostgreSQL's generate_series, it writes a Vortex file of synthetic data. Generators: series(start,step) (the generate_series-style arithmetic progression), range(min,max), normal(mean,stddev), enum(prefix,count) (random categorical labels), constant(value), and bool(). A --sort-by flag clusters rows by a given column afterward, independent of how that column's values were generated -- useful for zone-map-pruning-friendly demo data. demo/client (vortex-demo) no longer generates its own data: it now takes an existing .vortex file via --file, uploads it to a vortex-server (embedded by default, or --server for an already-running one), and scans it with configurable --filter-column/--filter-value/--project. All three demo modules' packages now nest under io.github.dfa1.vortex.demo: vortex-server -> io.github.dfa1.vortex.demo.server, vortex-fakedata-generator -> io.github.dfa1.vortex.demo.fakedata, vortex-demo -> io.github.dfa1.vortex.demo.client. Reinforces that this is all demo/presentation tooling, not part of the library's public API surface. Full three-tool pipeline verified end to end: generate -> serve -> scan, 2M rows sorted by symbol, single-symbol filter fetches 2.09% of the file over HTTP. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- demo/client/pom.xml | 21 +- .../dfa1/vortex/demo/DemoDataGenerator.java | 103 --------- .../dfa1/vortex/demo/HttpRangeDemo.java | 125 ----------- .../vortex/demo/client/HttpRangeDemo.java | 185 +++++++++++++++ demo/fakedata-generator/pom.xml | 90 ++++++++ .../demo/fakedata/ColumnDescriptor.java | 12 + .../demo/fakedata/ColumnMaterializer.java | 172 ++++++++++++++ .../demo/fakedata/DescriptorParser.java | 108 +++++++++ .../demo/fakedata/FakeDataGenerator.java | 196 ++++++++++++++++ .../demo/fakedata/FakeDataGeneratorCli.java | 113 ++++++++++ .../vortex/demo/fakedata/GeneratorSpec.java | 44 ++++ .../demo/fakedata/DescriptorParserTest.java | 211 ++++++++++++++++++ .../demo/fakedata/FakeDataGeneratorTest.java | 156 +++++++++++++ demo/pom.xml | 6 +- demo/server/pom.xml | 2 +- .../{ => demo}/server/VortexServer.java | 2 +- .../{ => demo}/server/VortexServerTest.java | 2 +- pom.xml | 5 + 18 files changed, 1308 insertions(+), 245 deletions(-) delete mode 100644 demo/client/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java delete mode 100644 demo/client/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java create mode 100644 demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java create mode 100644 demo/fakedata-generator/pom.xml create mode 100644 demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnDescriptor.java create mode 100644 demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java create mode 100644 demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/DescriptorParser.java create mode 100644 demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java create mode 100644 demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorCli.java create mode 100644 demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/GeneratorSpec.java create mode 100644 demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/DescriptorParserTest.java create mode 100644 demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorTest.java rename demo/server/src/main/java/io/github/dfa1/vortex/{ => demo}/server/VortexServer.java (99%) rename demo/server/src/test/java/io/github/dfa1/vortex/{ => demo}/server/VortexServerTest.java (99%) diff --git a/demo/client/pom.xml b/demo/client/pom.xml index de6d3f03..7d4e43b7 100644 --- a/demo/client/pom.xml +++ b/demo/client/pom.xml @@ -11,10 +11,11 @@ vortex-demo vortex-demo - Standalone demo client: generates a synthetic tick dataset, uploads it to a - vortex-server (embedded by default, or an already-running one given on the command line), - then runs a filtered/projected VortexHttpReader scan against it -- showing that the scan - fetches only the bytes it touches instead of downloading the whole object. + Standalone demo client: uploads an existing Vortex file (e.g. one produced by + vortex-fakedata-generator) to a vortex-server (embedded by default, or an already-running + one given on the command line), then runs a filtered/projected VortexHttpReader scan + against it -- showing that the scan fetches only the bytes it touches instead of + downloading the whole object. true true - + true - - io.github.dfa1.vortex - vortex-writer - io.github.dfa1.vortex vortex-reader @@ -71,7 +68,7 @@ vortex-demo - io.github.dfa1.vortex.demo.HttpRangeDemo + io.github.dfa1.vortex.demo.client.HttpRangeDemo diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java deleted file mode 100644 index 8073185b..00000000 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/DemoDataGenerator.java +++ /dev/null @@ -1,103 +0,0 @@ -package io.github.dfa1.vortex.demo; - -import io.github.dfa1.vortex.core.model.ColumnName; -import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.writer.VortexWriter; -import io.github.dfa1.vortex.writer.WriteOptions; - -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Random; - -/// Generates a synthetic tick-style dataset — `timestamp`/`symbol`/`price`/`volume` — sorted by -/// symbol so each chunk is dominated by a single symbol. -/// -/// That clustering is what makes [HttpRangeDemo]'s single-symbol filter dramatic: zone-map -/// pruning skips whole chunks whose stats can't match, so [RangeAwareFileServer] only ever -/// receives range requests for the handful of chunks holding the requested symbol. A real-world -/// archive partitioned or sorted by instrument (a common layout for exactly this kind of data) -/// behaves the same way. -final class DemoDataGenerator { - - static final String TIMESTAMP = "timestamp"; - static final String SYMBOL = "symbol"; - static final String PRICE = "price"; - static final String VOLUME = "volume"; - - private static final long BASE_TIMESTAMP_MILLIS = 1_700_000_000_000L; - private static final long SEED = 42L; - - private DemoDataGenerator() { - } - - /// Writes `rowCount` rows spread evenly across `symbolCount` symbols to `out`, in chunks of - /// `chunkSize` rows. - /// - /// @param out destination path; created or truncated - /// @param rowCount total rows to generate - /// @param symbolCount distinct symbols; rows are grouped by symbol, in order - /// @param chunkSize rows per written chunk - /// @throws IOException if writing the file fails - static void generate(Path out, long rowCount, int symbolCount, int chunkSize) throws IOException { - DType.Struct schema = DType.structBuilder() - .field(TIMESTAMP, DType.I64) - .field(SYMBOL, DType.UTF8) - .field(PRICE, DType.F64) - .field(VOLUME, DType.I64) - .build(); - - ColumnName timestampCol = ColumnName.of(TIMESTAMP); - ColumnName symbolCol = ColumnName.of(SYMBOL); - ColumnName priceCol = ColumnName.of(PRICE); - ColumnName volumeCol = ColumnName.of(VOLUME); - - String[] symbols = new String[symbolCount]; - for (int i = 0; i < symbolCount; i++) { - symbols[i] = "SYM%03d".formatted(i); - } - long rowsPerSymbol = Math.max(1, rowCount / symbolCount); - - // globalDict(false): the default global dictionary pulls a low-cardinality Utf8 column's - // codes out of the normal per-chunk cascade, and the per-chunk embedded stats - // RowFilter-driven zone-map pruning reads (ScanIterator#canPruneChunk) go unpopulated for - // those codes — the *separate* per-zone stats table (ScanIterator#columnZoneStats) stays - // correct either way, but pruning doesn't consult it. With the default globalDict=true, - // this demo's single-symbol filter provably matched zero chunks. Disabling it here keeps - // "symbol" in the ordinary per-chunk dict cascade, which does emit per-chunk min/max. - Random random = new Random(SEED); - try (FileChannel channel = FileChannel.open(out, StandardOpenOption.CREATE, - StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); - VortexWriter writer = VortexWriter.create(channel, schema, WriteOptions.cascading(3).withGlobalDict(false))) { - - long written = 0; - int symbolIndex = 0; - long rowInSymbol = 0; - while (written < rowCount) { - int n = (int) Math.min(chunkSize, rowCount - written); - long[] timestamps = new long[n]; - String[] symbolValues = new String[n]; - double[] prices = new double[n]; - long[] volumes = new long[n]; - for (int i = 0; i < n; i++) { - if (rowInSymbol >= rowsPerSymbol && symbolIndex < symbolCount - 1) { - symbolIndex++; - rowInSymbol = 0; - } - double basePrice = 50 + symbolIndex * 3.7; - timestamps[i] = BASE_TIMESTAMP_MILLIS + (written + i) * 1000L; - symbolValues[i] = symbols[symbolIndex]; - prices[i] = basePrice + random.nextDouble() * 2 - 1; - volumes[i] = 100 + random.nextInt(10_000); - rowInSymbol++; - } - writer.writeChunk(c -> c.put(timestampCol, timestamps) - .put(symbolCol, symbolValues) - .put(priceCol, prices) - .put(volumeCol, volumes)); - written += n; - } - } - } -} diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java deleted file mode 100644 index 2d3345a0..00000000 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/HttpRangeDemo.java +++ /dev/null @@ -1,125 +0,0 @@ -package io.github.dfa1.vortex.demo; - -import io.github.dfa1.vortex.reader.RowFilter; -import io.github.dfa1.vortex.reader.ScanOptions; -import io.github.dfa1.vortex.reader.VortexHttpReader; -import io.github.dfa1.vortex.server.VortexServer; - -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/// Demo client: generates a synthetic tick dataset, uploads it to a `vortex-server` object -/// store, then runs a filtered/projected [VortexHttpReader] scan against it — printing how many -/// bytes the scan actually pulled over the wire against the object's full size. -/// -/// Run standalone (embeds its own [VortexServer]): -/// ``` -/// java -jar demo/target/vortex-demo.jar -/// ``` -/// -/// Or against an already-running server — e.g. `java -jar server/target/vortex-server.jar` in a -/// separate terminal, for a two-process live demo: -/// ``` -/// java -jar demo/target/vortex-demo.jar http://127.0.0.1:8080/ -/// ``` -public final class HttpRangeDemo { - - private static final long ROW_COUNT = 2_000_000; - private static final int SYMBOL_COUNT = 30; - private static final int CHUNK_SIZE = 65_536; - private static final String QUERY_SYMBOL = "SYM015"; - private static final String OBJECT_KEY = "trades.vortex"; - - private static final Pattern BYTES_SERVED_LINE = Pattern.compile("bytesServed=(\\d+)"); - - private HttpRangeDemo() { - } - - /// @param args optional: base URI of an already-running `vortex-server`; omitted embeds one - /// @throws IOException if the demo file, server, upload, or scan fails - /// @throws InterruptedException if interrupted while starting the embedded server or sending - /// an HTTP request - public static void main(String[] args) throws IOException, InterruptedException { - Path localFile = Files.createTempDirectory("vortex-demo").resolve(OBJECT_KEY); - System.out.println("Generating demo file (" + ROW_COUNT + " rows, " + SYMBOL_COUNT + " symbols)..."); - DemoDataGenerator.generate(localFile, ROW_COUNT, SYMBOL_COUNT, CHUNK_SIZE); - long fileSize = Files.size(localFile); - System.out.printf("Wrote %s (%.1f MB)%n%n", localFile, fileSize / 1_000_000.0); - - if (args.length > 0) { - runAgainst(URI.create(args[0]), localFile, fileSize); - } else { - try (VortexServer server = VortexServer.start(Files.createTempDirectory("vortex-server"), 0)) { - System.out.println("Embedded vortex-server at " + server.baseUri()); - runAgainst(server.baseUri(), localFile, fileSize); - } - } - } - - private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize) - throws IOException, InterruptedException { - HttpClient client = HttpClient.newHttpClient(); - URI objectUri = serverBaseUri.resolve(OBJECT_KEY); - - System.out.println("Uploading to " + objectUri + " ..."); - upload(client, objectUri, localFile); - - long bytesServedBefore = readBytesServed(client, serverBaseUri); - System.out.printf("%nScanning for %s=%s, projecting '%s' over HTTP...%n%n", - DemoDataGenerator.SYMBOL, QUERY_SYMBOL, DemoDataGenerator.PRICE); - long rows = scan(objectUri); - long bytesServedAfter = readBytesServed(client, serverBaseUri); - - long servedDuringScan = bytesServedAfter - bytesServedBefore; - System.out.println(); - System.out.println("Matched rows: " + rows); - System.out.printf("Bytes fetched over HTTP during the scan: %,d / %,d (%.2f%% of the object)%n", - servedDuringScan, fileSize, 100.0 * servedDuringScan / fileSize); - } - - private static void upload(HttpClient client, URI objectUri, Path localFile) - throws IOException, InterruptedException { - HttpRequest put = HttpRequest.newBuilder(objectUri) - .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) - .build(); - HttpResponse response = client.send(put, HttpResponse.BodyHandlers.discarding()); - if (response.statusCode() != 201) { - throw new IOException("upload failed: HTTP " + response.statusCode()); - } - } - - private static long readBytesServed(HttpClient client, URI serverBaseUri) - throws IOException, InterruptedException { - HttpRequest req = HttpRequest.newBuilder(serverBaseUri.resolve("_stats")).GET().build(); - HttpResponse response = client.send(req, HttpResponse.BodyHandlers.ofString()); - Matcher m = BYTES_SERVED_LINE.matcher(response.body()); - if (!m.find()) { - throw new IOException("could not parse /_stats response: " + response.body()); - } - return Long.parseLong(m.group(1)); - } - - private static long scan(URI objectUri) throws IOException { - ScanOptions opts = ScanOptions.all() - .withColumns(DemoDataGenerator.SYMBOL, DemoDataGenerator.PRICE) - .withFilter(RowFilter.eq(DemoDataGenerator.SYMBOL, QUERY_SYMBOL)); - - long rows = 0; - try (VortexHttpReader vf = VortexHttpReader.open(objectUri); - var iter = vf.scan(opts)) { - while (iter.hasNext()) { - try (var chunk = iter.next()) { - rows += chunk.rowCount(); - } - } - } - return rows; - } -} diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java new file mode 100644 index 00000000..7b4e2d6d --- /dev/null +++ b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java @@ -0,0 +1,185 @@ +package io.github.dfa1.vortex.demo.client; + +import io.github.dfa1.vortex.demo.server.VortexServer; +import io.github.dfa1.vortex.reader.RowFilter; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexHttpReader; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/// Demo client: uploads an existing Vortex file (e.g. one produced by +/// `vortex-fakedata-generator`) to a `vortex-server` object store, then runs a +/// filtered/projected [VortexHttpReader] scan against it — printing how many bytes the scan +/// actually pulled over the wire against the object's full size. +/// +/// Run standalone (embeds its own [VortexServer]): +/// ``` +/// java -jar client/target/vortex-demo.jar --file trades.vortex +/// ``` +/// +/// Or against an already-running server — e.g. `java -jar server/target/vortex-server.jar` in a +/// separate terminal, for a two-process live demo: +/// ``` +/// java -jar client/target/vortex-demo.jar --file trades.vortex --server http://127.0.0.1:8080/ +/// ``` +public final class HttpRangeDemo { + + private static final String DEFAULT_FILTER_COLUMN = "symbol"; + private static final String DEFAULT_FILTER_VALUE = "SYM015"; + private static final String DEFAULT_PROJECT_COLUMN = "price"; + + private static final Pattern BYTES_SERVED_LINE = Pattern.compile("bytesServed=(\\d+)"); + + private HttpRangeDemo() { + } + + /// @param args CLI arguments; run with no arguments to print usage + public static void main(String[] args) { + try { + run(args); + } catch (RuntimeException | IOException | InterruptedException e) { + System.err.println("error: " + e.getMessage()); + System.err.println(); + printUsage(); + System.exit(1); + } + } + + private static void run(String[] args) throws IOException, InterruptedException { + Path file = null; + URI serverBaseUri = null; + String filterColumn = DEFAULT_FILTER_COLUMN; + String filterValue = DEFAULT_FILTER_VALUE; + String projectColumn = DEFAULT_PROJECT_COLUMN; + + int i = 0; + while (i < args.length) { + switch (args[i]) { + case "--file" -> { + file = Path.of(args[++i]); + i++; + } + case "--server" -> { + serverBaseUri = URI.create(args[++i]); + i++; + } + case "--filter-column" -> { + filterColumn = args[++i]; + i++; + } + case "--filter-value" -> { + filterValue = args[++i]; + i++; + } + case "--project" -> { + projectColumn = args[++i]; + i++; + } + default -> throw new IllegalArgumentException("unknown argument: " + args[i]); + } + } + + if (file == null) { + throw new IllegalArgumentException("--file is required"); + } + long fileSize = Files.size(file); + + if (serverBaseUri != null) { + runAgainst(serverBaseUri, file, fileSize, filterColumn, filterValue, projectColumn); + } else { + try (VortexServer server = VortexServer.start(Files.createTempDirectory("vortex-server"), 0)) { + System.out.println("Embedded vortex-server at " + server.baseUri()); + runAgainst(server.baseUri(), file, fileSize, filterColumn, filterValue, projectColumn); + } + } + } + + private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize, + String filterColumn, String filterValue, String projectColumn) throws IOException, InterruptedException { + HttpClient client = HttpClient.newHttpClient(); + URI objectUri = serverBaseUri.resolve(localFile.getFileName().toString()); + + System.out.println("Uploading to " + objectUri + " ..."); + upload(client, objectUri, localFile); + + long bytesServedBefore = readBytesServed(client, serverBaseUri); + System.out.printf("%nScanning for %s=%s, projecting '%s' over HTTP...%n%n", + filterColumn, filterValue, projectColumn); + long rows = scan(objectUri, filterColumn, filterValue, projectColumn); + long bytesServedAfter = readBytesServed(client, serverBaseUri); + + long servedDuringScan = bytesServedAfter - bytesServedBefore; + System.out.println(); + System.out.println("Matched rows: " + rows); + System.out.printf("Bytes fetched over HTTP during the scan: %,d / %,d (%.2f%% of the object)%n", + servedDuringScan, fileSize, 100.0 * servedDuringScan / fileSize); + } + + private static void upload(HttpClient client, URI objectUri, Path localFile) + throws IOException, InterruptedException { + HttpRequest put = HttpRequest.newBuilder(objectUri) + .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) + .build(); + HttpResponse response = client.send(put, HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() != 201) { + throw new IOException("upload failed: HTTP " + response.statusCode()); + } + } + + private static long readBytesServed(HttpClient client, URI serverBaseUri) + throws IOException, InterruptedException { + HttpRequest req = HttpRequest.newBuilder(serverBaseUri.resolve("_stats")).GET().build(); + HttpResponse response = client.send(req, HttpResponse.BodyHandlers.ofString()); + Matcher m = BYTES_SERVED_LINE.matcher(response.body()); + if (!m.find()) { + throw new IOException("could not parse /_stats response: " + response.body()); + } + return Long.parseLong(m.group(1)); + } + + private static long scan(URI objectUri, String filterColumn, String filterValue, String projectColumn) + throws IOException { + ScanOptions opts = ScanOptions.all() + .withColumns(filterColumn, projectColumn) + .withFilter(RowFilter.eq(filterColumn, filterValue)); + + long rows = 0; + try (VortexHttpReader vf = VortexHttpReader.open(objectUri); + var iter = vf.scan(opts)) { + while (iter.hasNext()) { + try (var chunk = iter.next()) { + rows += chunk.rowCount(); + } + } + } + return rows; + } + + private static void printUsage() { + System.err.println(""" + Usage: vortex-demo --file FILE [options] + + Options: + --file FILE local .vortex file to upload and scan (required) + --server URI base URI of an already-running vortex-server; omit to embed one + --filter-column NAME column to filter on (default: symbol) + --filter-value VALUE value to filter for (default: SYM015) + --project NAME column to project (default: price) + + Generate a file first with vortex-fakedata-generator, e.g.: + vortex-fakedata-generator --rows 2000000 --out trades.vortex --sort-by symbol \\ + "timestamp:i64:series(1700000000000,1000)" \\ + "symbol:utf8:enum(SYM,30)" \\ + "price:f64:range(50,150)" \\ + "volume:i64:range(100,10000)" + """); + } +} diff --git a/demo/fakedata-generator/pom.xml b/demo/fakedata-generator/pom.xml new file mode 100644 index 00000000..626ad7a3 --- /dev/null +++ b/demo/fakedata-generator/pom.xml @@ -0,0 +1,90 @@ + + + 4.0.0 + + io.github.dfa1.vortex + vortex-java + 0.14.2-SNAPSHOT + ../../pom.xml + + + vortex-fakedata-generator + + vortex-fakedata-generator + Standalone CLI: generates a Vortex file of synthetic data from a compact + column-description grammar (name:type:generator(args), e.g. + "price:f64:range(50,150)"), inspired by PostgreSQL's generate_series. Independent of + any particular demo -- reusable wherever a quick synthetic .vortex file is needed. + + + + true + true + + true + + + + + + io.github.dfa1.vortex + vortex-writer + + + + io.github.dfa1.vortex + vortex-reader + test + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + vortex-fakedata-generator + + + io.github.dfa1.vortex.demo.fakedata.FakeDataGeneratorCli + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnDescriptor.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnDescriptor.java new file mode 100644 index 00000000..16599430 --- /dev/null +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnDescriptor.java @@ -0,0 +1,12 @@ +package io.github.dfa1.vortex.demo.fakedata; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; + +/// One parsed column description: `name:type:generator(args)`. +/// +/// @param name the column's name +/// @param dtype the column's declared, non-nullable logical type +/// @param generator the rule used to fill the column's values +public record ColumnDescriptor(ColumnName name, DType dtype, GeneratorSpec generator) { +} diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java new file mode 100644 index 00000000..18f283fc --- /dev/null +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java @@ -0,0 +1,172 @@ +package io.github.dfa1.vortex.demo.fakedata; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; + +import java.util.Arrays; +import java.util.Random; + +/// Turns a [ColumnDescriptor]'s [GeneratorSpec] into a fully materialized column array, ready to +/// hand to a Vortex `Chunk` (`long[]`, `int[]`, ..., `double[]`, `String[]`, or `boolean[]`, +/// matching the column's declared [DType]). +final class ColumnMaterializer { + + private ColumnMaterializer() { + } + + /// Fails fast at parse time if `generator` cannot produce values of `dtype`. + /// + /// @param dtype the column's declared type + /// @param generator the generator to check + /// @param descriptor the original descriptor string, for the error message + /// @throws IllegalArgumentException if the combination is invalid + static void validateCompatible(DType dtype, GeneratorSpec generator, String descriptor) { + boolean numeric = dtype instanceof DType.Primitive; + boolean ok = switch (generator) { + case GeneratorSpec.Series _ -> numeric; + case GeneratorSpec.Range _ -> numeric; + case GeneratorSpec.Normal _ -> numeric; + case GeneratorSpec.EnumLabels _ -> dtype instanceof DType.Utf8; + case GeneratorSpec.RandomBool _ -> dtype instanceof DType.Bool; + case GeneratorSpec.Constant(String literal) -> isValidConstant(dtype, literal); + }; + if (!ok) { + throw new IllegalArgumentException( + "generator incompatible with declared type in descriptor '%s'".formatted(descriptor)); + } + } + + private static boolean isValidConstant(DType dtype, String literal) { + if (dtype instanceof DType.Utf8) { + return true; + } + if (dtype instanceof DType.Bool) { + return literal.equalsIgnoreCase("true") || literal.equalsIgnoreCase("false"); + } + try { + Double.parseDouble(literal); + return true; + } catch (NumberFormatException e) { + return false; + } + } + + /// Materializes `rows` values for `column`. + /// + /// @param column the column to generate + /// @param rows number of rows to generate + /// @param random shared random source (consumed in column-declaration order for determinism) + /// @return the typed array (`long[]`/`int[]`/.../`double[]`, `String[]`, or `boolean[]`) + static Object materialize(ColumnDescriptor column, int rows, Random random) { + DType dtype = column.dtype(); + GeneratorSpec generator = column.generator(); + if (dtype instanceof DType.Utf8) { + return materializeUtf8(generator, rows, random); + } + if (dtype instanceof DType.Bool) { + return materializeBool(generator, rows, random); + } + PType ptype = ((DType.Primitive) dtype).ptype(); + double[] values = materializeNumeric(generator, rows, random); + return narrow(ptype, values); + } + + private static double[] materializeNumeric(GeneratorSpec generator, int rows, Random random) { + double[] values = new double[rows]; + switch (generator) { + case GeneratorSpec.Series(double start, double step) -> { + for (int i = 0; i < rows; i++) { + values[i] = start + i * step; + } + } + case GeneratorSpec.Range(double min, double max) -> { + for (int i = 0; i < rows; i++) { + values[i] = min + random.nextDouble() * (max - min); + } + } + case GeneratorSpec.Normal(double mean, double stddev) -> { + for (int i = 0; i < rows; i++) { + values[i] = mean + random.nextGaussian() * stddev; + } + } + case GeneratorSpec.Constant(String literal) -> Arrays.fill(values, Double.parseDouble(literal)); + case GeneratorSpec.EnumLabels _, GeneratorSpec.RandomBool _ -> + throw new IllegalStateException("unreachable: validated at parse time"); + } + return values; + } + + private static Object narrow(PType ptype, double[] values) { + return switch (ptype) { + case I64, U64 -> { + long[] out = new long[values.length]; + for (int i = 0; i < values.length; i++) { + out[i] = Math.round(values[i]); + } + yield out; + } + case I32, U32 -> { + int[] out = new int[values.length]; + for (int i = 0; i < values.length; i++) { + out[i] = (int) Math.round(values[i]); + } + yield out; + } + case I16, U16 -> { + short[] out = new short[values.length]; + for (int i = 0; i < values.length; i++) { + out[i] = (short) Math.round(values[i]); + } + yield out; + } + case I8, U8 -> { + byte[] out = new byte[values.length]; + for (int i = 0; i < values.length; i++) { + out[i] = (byte) Math.round(values[i]); + } + yield out; + } + case F64 -> values; + case F32 -> { + float[] out = new float[values.length]; + for (int i = 0; i < values.length; i++) { + out[i] = (float) values[i]; + } + yield out; + } + case F16 -> throw new IllegalArgumentException("f16 is not a supported generator column type"); + }; + } + + private static String[] materializeUtf8(GeneratorSpec generator, int rows, Random random) { + String[] out = new String[rows]; + switch (generator) { + case GeneratorSpec.EnumLabels(String prefix, int count) -> { + int width = String.valueOf(count - 1).length(); + String format = prefix + "%0" + width + "d"; + for (int i = 0; i < rows; i++) { + out[i] = format.formatted(random.nextInt(count)); + } + } + case GeneratorSpec.Constant(String literal) -> Arrays.fill(out, literal); + case GeneratorSpec.Series _, GeneratorSpec.Range _, GeneratorSpec.Normal _, GeneratorSpec.RandomBool _ -> + throw new IllegalStateException("unreachable: validated at parse time"); + } + return out; + } + + private static boolean[] materializeBool(GeneratorSpec generator, int rows, Random random) { + boolean[] out = new boolean[rows]; + switch (generator) { + case GeneratorSpec.RandomBool() -> { + for (int i = 0; i < rows; i++) { + out[i] = random.nextBoolean(); + } + } + case GeneratorSpec.Constant(String literal) -> Arrays.fill(out, Boolean.parseBoolean(literal)); + case GeneratorSpec.Series _, GeneratorSpec.Range _, GeneratorSpec.Normal _, GeneratorSpec.EnumLabels _ -> + throw new IllegalStateException("unreachable: validated at parse time"); + } + return out; + } +} diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/DescriptorParser.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/DescriptorParser.java new file mode 100644 index 00000000..54c01158 --- /dev/null +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/DescriptorParser.java @@ -0,0 +1,108 @@ +package io.github.dfa1.vortex.demo.fakedata; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; + +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/// Parses one `name:type:generator(args)` column descriptor string, e.g. +/// `"price:f64:range(50,150)"` or `"active:bool:bool()"`. +/// +/// Grammar: +/// ``` +/// descriptor ::= name ":" type ":" generator +/// type ::= "i8" | "i16" | "i32" | "i64" | "u8" | "u16" | "u32" | "u64" +/// | "f32" | "f64" | "utf8" | "bool" +/// generator ::= "series(" start "," step ")" +/// | "range(" min "," max ")" +/// | "normal(" mean "," stddev ")" +/// | "enum(" prefix "," count ")" +/// | "constant(" literal ")" +/// | "bool()" +/// ``` +public final class DescriptorParser { + + private static final Pattern DESCRIPTOR = + Pattern.compile("^([^:]+):([a-zA-Z0-9]+):([a-zA-Z]+)\\(([^)]*)\\)$"); + + private DescriptorParser() { + } + + /// Parses one column descriptor string. + /// + /// @param descriptor the raw descriptor, e.g. `"symbol:utf8:enum(SYM,30)"` + /// @return the parsed column descriptor + /// @throws IllegalArgumentException if `descriptor` doesn't match the grammar, names an + /// unknown type or generator, or the generator's arguments + /// don't fit that column's type + public static ColumnDescriptor parse(String descriptor) { + Matcher m = DESCRIPTOR.matcher(descriptor.strip()); + if (!m.matches()) { + throw new IllegalArgumentException( + "malformed column descriptor '%s' -- expected name:type:generator(args)".formatted(descriptor)); + } + ColumnName name = ColumnName.of(m.group(1)); + DType dtype = parseType(m.group(2), descriptor); + List args = splitArgs(m.group(4)); + GeneratorSpec generator = parseGenerator(m.group(3), args, dtype, descriptor); + return new ColumnDescriptor(name, dtype, generator); + } + + private static DType parseType(String type, String descriptor) { + return switch (type) { + case "i8" -> DType.I8; + case "i16" -> DType.I16; + case "i32" -> DType.I32; + case "i64" -> DType.I64; + case "u8" -> DType.U8; + case "u16" -> DType.U16; + case "u32" -> DType.U32; + case "u64" -> DType.U64; + case "f32" -> DType.F32; + case "f64" -> DType.F64; + case "utf8" -> DType.UTF8; + case "bool" -> DType.BOOL; + default -> throw new IllegalArgumentException( + "unknown type '%s' in descriptor '%s'".formatted(type, descriptor)); + }; + } + + private static GeneratorSpec parseGenerator(String function, List args, DType dtype, String descriptor) { + GeneratorSpec generator = switch (function) { + case "series" -> new GeneratorSpec.Series(argAsDouble(args, 0, descriptor), argAsDouble(args, 1, descriptor)); + case "range" -> new GeneratorSpec.Range(argAsDouble(args, 0, descriptor), argAsDouble(args, 1, descriptor)); + case "normal" -> new GeneratorSpec.Normal(argAsDouble(args, 0, descriptor), argAsDouble(args, 1, descriptor)); + case "enum" -> new GeneratorSpec.EnumLabels(args.get(0), (int) argAsDouble(args, 1, descriptor)); + case "constant" -> new GeneratorSpec.Constant(args.isEmpty() ? "" : args.get(0)); + case "bool" -> new GeneratorSpec.RandomBool(); + default -> throw new IllegalArgumentException( + "unknown generator '%s' in descriptor '%s'".formatted(function, descriptor)); + }; + ColumnMaterializer.validateCompatible(dtype, generator, descriptor); + return generator; + } + + private static double argAsDouble(List args, int index, String descriptor) { + if (index >= args.size()) { + throw new IllegalArgumentException( + "generator in descriptor '%s' needs at least %d argument(s)".formatted(descriptor, index + 1)); + } + try { + return Double.parseDouble(args.get(index).strip()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "argument '%s' in descriptor '%s' is not a number".formatted(args.get(index), descriptor), e); + } + } + + private static List splitArgs(String raw) { + String trimmed = raw.strip(); + if (trimmed.isEmpty()) { + return List.of(); + } + return Arrays.stream(trimmed.split(",", -1)).map(String::strip).toList(); + } +} diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java new file mode 100644 index 00000000..b9272abf --- /dev/null +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java @@ -0,0 +1,196 @@ +package io.github.dfa1.vortex.demo.fakedata; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +/// Generates a Vortex file of synthetic data from a list of [ColumnDescriptor]s. The library +/// entry point behind [FakeDataGeneratorCli]; callable directly by other code (e.g. a demo +/// module) without shelling out to the CLI. +public final class FakeDataGenerator { + + private FakeDataGenerator() { + } + + /// @param columns column descriptors, in schema order + /// @param rows total number of rows to generate + /// @param seed seed for the shared random source (deterministic across runs) + /// @param sortBy column to sort all rows by (ascending), or `null` for generation order + /// @param chunkSize rows per written chunk + /// @param cascading write compression cascade depth (see `WriteOptions#cascading`) + /// @param out destination path; created or truncated + /// @throws IOException if writing fails + public static void generate(List columns, int rows, long seed, String sortBy, + int chunkSize, int cascading, Path out) throws IOException { + if (columns.isEmpty()) { + throw new IllegalArgumentException("at least one column descriptor is required"); + } + + Random random = new Random(seed); + Object[] columnArrays = new Object[columns.size()]; + for (int c = 0; c < columns.size(); c++) { + columnArrays[c] = ColumnMaterializer.materialize(columns.get(c), rows, random); + } + + if (sortBy != null) { + int sortColumnIndex = indexOf(columns, sortBy); + int[] permutation = sortPermutation(columnArrays[sortColumnIndex], rows); + for (int c = 0; c < columns.size(); c++) { + columnArrays[c] = permute(columnArrays[c], permutation); + } + } + + DType.Struct schema = buildSchema(columns); + // globalDict defeats zone-map pruning on Utf8 columns (see the CLAUDE.md/README of + // vortex-java's own reader module): the whole point of a fakedata tool feeding demos + // that showcase pruning/partial fetches is to keep per-chunk stats meaningful. + WriteOptions options = WriteOptions.cascading(cascading).withGlobalDict(false); + try (FileChannel channel = FileChannel.open(out, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + VortexWriter writer = VortexWriter.create(channel, schema, options)) { + for (int start = 0; start < rows; start += chunkSize) { + int n = Math.min(chunkSize, rows - start); + writer.writeChunk(sliceChunk(columns, columnArrays, start, n)); + } + } + } + + private static DType.Struct buildSchema(List columns) { + DType.StructBuilder builder = DType.structBuilder(); + for (ColumnDescriptor col : columns) { + builder.field(col.name(), col.dtype()); + } + return builder.build(); + } + + private static Map sliceChunk(List columns, Object[] columnArrays, + int start, int n) { + Map chunk = new LinkedHashMap<>(); + for (int c = 0; c < columns.size(); c++) { + chunk.put(columns.get(c).name(), slice(columnArrays[c], start, n)); + } + return chunk; + } + + private static int indexOf(List columns, String name) { + for (int i = 0; i < columns.size(); i++) { + if (columns.get(i).name().value().equals(name)) { + return i; + } + } + throw new IllegalArgumentException("--sort-by column '" + name + "' is not one of the declared columns"); + } + + private static int[] sortPermutation(Object array, int rows) { + Integer[] indices = new Integer[rows]; + for (int i = 0; i < rows; i++) { + indices[i] = i; + } + Comparator byValue = switch (array) { + case long[] a -> Comparator.comparingLong(i -> a[i]); + case int[] a -> Comparator.comparingInt(i -> a[i]); + case short[] a -> Comparator.comparingInt(i -> a[i]); + case byte[] a -> Comparator.comparingInt(i -> a[i]); + case double[] a -> Comparator.comparingDouble(i -> a[i]); + case float[] a -> Comparator.comparingDouble(i -> a[i]); + case String[] a -> Comparator.comparing(i -> a[i]); + case boolean[] a -> Comparator.comparingInt(i -> a[i] ? 1 : 0); + default -> throw new IllegalStateException("unsupported array type: " + array.getClass()); + }; + Arrays.sort(indices, byValue); + int[] out = new int[rows]; + for (int i = 0; i < rows; i++) { + out[i] = indices[i]; + } + return out; + } + + private static Object permute(Object array, int[] permutation) { + int n = permutation.length; + return switch (array) { + case long[] a -> { + long[] out = new long[n]; + for (int i = 0; i < n; i++) { + out[i] = a[permutation[i]]; + } + yield out; + } + case int[] a -> { + int[] out = new int[n]; + for (int i = 0; i < n; i++) { + out[i] = a[permutation[i]]; + } + yield out; + } + case short[] a -> { + short[] out = new short[n]; + for (int i = 0; i < n; i++) { + out[i] = a[permutation[i]]; + } + yield out; + } + case byte[] a -> { + byte[] out = new byte[n]; + for (int i = 0; i < n; i++) { + out[i] = a[permutation[i]]; + } + yield out; + } + case double[] a -> { + double[] out = new double[n]; + for (int i = 0; i < n; i++) { + out[i] = a[permutation[i]]; + } + yield out; + } + case float[] a -> { + float[] out = new float[n]; + for (int i = 0; i < n; i++) { + out[i] = a[permutation[i]]; + } + yield out; + } + case String[] a -> { + String[] out = new String[n]; + for (int i = 0; i < n; i++) { + out[i] = a[permutation[i]]; + } + yield out; + } + case boolean[] a -> { + boolean[] out = new boolean[n]; + for (int i = 0; i < n; i++) { + out[i] = a[permutation[i]]; + } + yield out; + } + default -> throw new IllegalStateException("unsupported array type: " + array.getClass()); + }; + } + + private static Object slice(Object array, int start, int n) { + return switch (array) { + case long[] a -> Arrays.copyOfRange(a, start, start + n); + case int[] a -> Arrays.copyOfRange(a, start, start + n); + case short[] a -> Arrays.copyOfRange(a, start, start + n); + case byte[] a -> Arrays.copyOfRange(a, start, start + n); + case double[] a -> Arrays.copyOfRange(a, start, start + n); + case float[] a -> Arrays.copyOfRange(a, start, start + n); + case String[] a -> Arrays.copyOfRange(a, start, start + n); + case boolean[] a -> Arrays.copyOfRange(a, start, start + n); + default -> throw new IllegalStateException("unsupported array type: " + array.getClass()); + }; + } +} diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorCli.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorCli.java new file mode 100644 index 00000000..298279f8 --- /dev/null +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorCli.java @@ -0,0 +1,113 @@ +package io.github.dfa1.vortex.demo.fakedata; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/// Command-line entry point for generating a synthetic Vortex file. See [#printUsage] for the +/// full option/grammar reference. +public final class FakeDataGeneratorCli { + + private static final int DEFAULT_CHUNK_SIZE = 65_536; + private static final int DEFAULT_CASCADING = 3; + private static final long DEFAULT_SEED = 42L; + + private FakeDataGeneratorCli() { + } + + /// @param args CLI arguments; run with no arguments to print usage + public static void main(String[] args) { + try { + run(args); + } catch (RuntimeException | IOException e) { + System.err.println("error: " + e.getMessage()); + System.err.println(); + printUsage(); + System.exit(1); + } + } + + private static void run(String[] args) throws IOException { + Integer rows = null; + Path out = null; + long seed = DEFAULT_SEED; + int chunkSize = DEFAULT_CHUNK_SIZE; + int cascading = DEFAULT_CASCADING; + String sortBy = null; + List descriptors = new ArrayList<>(); + + int i = 0; + while (i < args.length) { + String arg = args[i]; + switch (arg) { + case "--rows" -> { + rows = Integer.parseInt(args[++i]); + i++; + } + case "--out" -> { + out = Path.of(args[++i]); + i++; + } + case "--seed" -> { + seed = Long.parseLong(args[++i]); + i++; + } + case "--chunk-size" -> { + chunkSize = Integer.parseInt(args[++i]); + i++; + } + case "--cascading" -> { + cascading = Integer.parseInt(args[++i]); + i++; + } + case "--sort-by" -> { + sortBy = args[++i]; + i++; + } + default -> { + descriptors.add(arg); + i++; + } + } + } + + if (rows == null || out == null || descriptors.isEmpty()) { + throw new IllegalArgumentException("--rows, --out, and at least one column descriptor are required"); + } + + List columns = descriptors.stream().map(DescriptorParser::parse).toList(); + FakeDataGenerator.generate(columns, rows, seed, sortBy, chunkSize, cascading, out); + System.out.printf("Wrote %d rows (%d columns) to %s%n", rows, columns.size(), out); + } + + private static void printUsage() { + System.err.println(""" + Usage: vortex-fakedata-generator --rows N --out FILE [options] "name:type:generator(args)" [...] + + Options: + --rows N number of rows to generate (required) + --out FILE destination .vortex file (required) + --seed N random seed (default 42) + --chunk-size N rows per written chunk (default 65536) + --cascading N write compression cascade depth (default 3) + --sort-by COLUMN sort all rows by this column (ascending) before writing + + Column descriptor grammar: name:type:generator(args) + type: i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 utf8 bool + generator: series(start,step) arithmetic progression (like SQL generate_series) + range(min,max) uniform random + normal(mean,stddev) gaussian random + enum(prefix,count) random categorical label, cycling prefix0..prefix(count-1) + constant(value) same value every row + bool() random true/false + + Example: + vortex-fakedata-generator --rows 2000000 --out trades.vortex --sort-by symbol \\ + "timestamp:i64:series(1700000000000,1000)" \\ + "symbol:utf8:enum(SYM,30)" \\ + "price:f64:range(50,150)" \\ + "volume:i64:range(100,10000)" + """); + } +} diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/GeneratorSpec.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/GeneratorSpec.java new file mode 100644 index 00000000..a2a10959 --- /dev/null +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/GeneratorSpec.java @@ -0,0 +1,44 @@ +package io.github.dfa1.vortex.demo.fakedata; + +/// One column's value-generation rule, parsed from the `generator(args)` part of a column +/// descriptor (see [DescriptorParser]). +/// +/// - [Series] mirrors PostgreSQL's `generate_series`: an arithmetic progression, `start + i * +/// step` for row `i`. Numeric columns only. +/// - [Range] draws uniformly at random from `[min, max]`. Numeric columns only. +/// - [Normal] draws from a Gaussian distribution with the given mean and standard deviation. +/// Numeric columns only. +/// - [EnumLabels] cycles uniformly at random through `count` labels `prefix0` … `prefix(count-1)` +/// (zero-padded to a common width). `Utf8` columns only. Combine with `--sort-by` on the CLI to +/// cluster same-label rows together (e.g. for zone-map-pruning-friendly data). +/// - [Constant] repeats one literal value for every row. Any column type. +/// - [RandomBool] draws uniformly at random between `true` and `false`. `Bool` columns only. +public sealed interface GeneratorSpec { + + /// @param start first value (row 0) + /// @param step increment per row + record Series(double start, double step) implements GeneratorSpec { + } + + /// @param min inclusive lower bound + /// @param max inclusive upper bound + record Range(double min, double max) implements GeneratorSpec { + } + + /// @param mean distribution mean + /// @param stddev distribution standard deviation + record Normal(double mean, double stddev) implements GeneratorSpec { + } + + /// @param prefix label prefix + /// @param count number of distinct labels + record EnumLabels(String prefix, int count) implements GeneratorSpec { + } + + /// @param literal the value's literal text, parsed per the column's declared type + record Constant(String literal) implements GeneratorSpec { + } + + record RandomBool() implements GeneratorSpec { + } +} diff --git a/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/DescriptorParserTest.java b/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/DescriptorParserTest.java new file mode 100644 index 00000000..85c694fb --- /dev/null +++ b/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/DescriptorParserTest.java @@ -0,0 +1,211 @@ +package io.github.dfa1.vortex.demo.fakedata; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class DescriptorParserTest { + + @Nested + class ValidDescriptors { + + @Test + void parsesASeriesGenerator() { + // Given + String descriptor = "timestamp:i64:series(1700000000000,1000)"; + + // When + ColumnDescriptor result = DescriptorParser.parse(descriptor); + + // Then + assertThat(result.name()).isEqualTo(ColumnName.of("timestamp")); + assertThat(result.dtype()).isEqualTo(DType.I64); + assertThat(result.generator()).isEqualTo(new GeneratorSpec.Series(1_700_000_000_000.0, 1000.0)); + } + + @Test + void parsesARangeGenerator() { + // Given + String descriptor = "price:f64:range(50,150)"; + + // When + ColumnDescriptor result = DescriptorParser.parse(descriptor); + + // Then + assertThat(result.dtype()).isEqualTo(DType.F64); + assertThat(result.generator()).isEqualTo(new GeneratorSpec.Range(50.0, 150.0)); + } + + @Test + void parsesANormalGenerator() { + // Given + String descriptor = "price:f64:normal(100,15)"; + + // When + ColumnDescriptor result = DescriptorParser.parse(descriptor); + + // Then + assertThat(result.generator()).isEqualTo(new GeneratorSpec.Normal(100.0, 15.0)); + } + + @Test + void parsesAnEnumGenerator() { + // Given + String descriptor = "symbol:utf8:enum(SYM,30)"; + + // When + ColumnDescriptor result = DescriptorParser.parse(descriptor); + + // Then + assertThat(result.dtype()).isEqualTo(DType.UTF8); + assertThat(result.generator()).isEqualTo(new GeneratorSpec.EnumLabels("SYM", 30)); + } + + @Test + void parsesAConstantGenerator() { + // Given + String descriptor = "flag:bool:constant(true)"; + + // When + ColumnDescriptor result = DescriptorParser.parse(descriptor); + + // Then + assertThat(result.generator()).isEqualTo(new GeneratorSpec.Constant("true")); + } + + @Test + void parsesABoolGenerator() { + // Given + String descriptor = "active:bool:bool()"; + + // When + ColumnDescriptor result = DescriptorParser.parse(descriptor); + + // Then + assertThat(result.dtype()).isEqualTo(DType.BOOL); + assertThat(result.generator()).isEqualTo(new GeneratorSpec.RandomBool()); + } + + @ParameterizedTest + @ValueSource(strings = {"i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64"}) + void acceptsEveryNumericTypeForSeries(String type) { + // Given + String descriptor = "col:" + type + ":series(1,1)"; + + // When + ColumnDescriptor result = DescriptorParser.parse(descriptor); + + // Then + assertThat(result.dtype()).isInstanceOf(DType.Primitive.class); + } + } + + @Nested + class InvalidDescriptors { + + @Test + void rejectsMalformedSyntax() { + // Given + String descriptor = "not-a-valid-descriptor"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("malformed"); + } + + @Test + void rejectsUnknownType() { + // Given + String descriptor = "col:decimal:range(1,2)"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unknown type"); + } + + @Test + void rejectsUnknownGenerator() { + // Given + String descriptor = "col:i64:fibonacci(1,2)"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unknown generator"); + } + + @Test + void rejectsSeriesOnUtf8Column() { + // Given + String descriptor = "col:utf8:series(1,1)"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("incompatible"); + } + + @Test + void rejectsEnumOnNumericColumn() { + // Given + String descriptor = "col:i64:enum(A,3)"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("incompatible"); + } + + @Test + void rejectsBoolGeneratorOnNumericColumn() { + // Given + String descriptor = "col:i64:bool()"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("incompatible"); + } + + @Test + void rejectsNonNumericConstantOnNumericColumn() { + // Given + String descriptor = "col:i64:constant(not-a-number)"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("incompatible"); + } + + @Test + void rejectsNonBooleanConstantOnBoolColumn() { + // Given + String descriptor = "col:bool:constant(maybe)"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("incompatible"); + } + + @Test + void rejectsTooFewGeneratorArguments() { + // Given + String descriptor = "col:i64:range(1)"; + + // When / Then + assertThatThrownBy(() -> DescriptorParser.parse(descriptor)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("needs at least"); + } + } +} diff --git a/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorTest.java b/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorTest.java new file mode 100644 index 00000000..bd852456 --- /dev/null +++ b/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorTest.java @@ -0,0 +1,156 @@ +package io.github.dfa1.vortex.demo.fakedata; + +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class FakeDataGeneratorTest { + + @Test + void generatesAnExactArithmeticSeries(@TempDir Path dir) throws IOException { + // Given a single series column + List columns = List.of(DescriptorParser.parse("id:i64:series(10,5)")); + Path out = dir.resolve("series.vortex"); + + // When + FakeDataGenerator.generate(columns, 6, 42L, null, 65_536, 3, out); + + // Then rows are the exact arithmetic progression: 10, 15, 20, 25, 30, 35 + List values = readLongColumn(out, "id"); + assertThat(values).containsExactly(10L, 15L, 20L, 25L, 30L, 35L); + } + + @Test + void generatesEnumLabelsWithinTheDeclaredCount(@TempDir Path dir) throws IOException { + // Given an enum column with 3 labels + List columns = List.of(DescriptorParser.parse("symbol:utf8:enum(SYM,3)")); + Path out = dir.resolve("enum.vortex"); + + // When + FakeDataGenerator.generate(columns, 200, 7L, null, 65_536, 3, out); + + // Then every value is one of the 3 declared labels + List values = readUtf8Column(out, "symbol"); + assertThat(values).hasSize(200).allSatisfy(v -> assertThat(v).isIn("SYM0", "SYM1", "SYM2")); + } + + @Test + void sortByOrdersEveryColumnByTheSamePermutation(@TempDir Path dir) throws IOException { + // Given a series id column (its natural order reveals the sort permutation) and an + // enum symbol column, sorted by symbol + List columns = List.of( + DescriptorParser.parse("id:i64:series(0,1)"), + DescriptorParser.parse("symbol:utf8:enum(SYM,4)")); + Path out = dir.resolve("sorted.vortex"); + + // When + FakeDataGenerator.generate(columns, 500, 1L, "symbol", 65_536, 3, out); + + // Then the symbol column is non-decreasing... + List symbols = readUtf8Column(out, "symbol"); + assertThat(symbols).isSorted(); + // ...and each id still names the row that originally held that symbol (id i's symbol + // before sorting is deterministic from the same seed/generator, so this cross-checks + // that every column moved together under the same permutation, not just "symbol" itself). + List unsorted = List.of( + DescriptorParser.parse("id:i64:series(0,1)"), + DescriptorParser.parse("symbol:utf8:enum(SYM,4)")); + Path unsortedOut = dir.resolve("unsorted.vortex"); + FakeDataGenerator.generate(unsorted, 500, 1L, null, 65_536, 3, unsortedOut); + List originalSymbolById = readUtf8Column(unsortedOut, "symbol"); + + List ids = readLongColumn(out, "id"); + for (int i = 0; i < ids.size(); i++) { + assertThat(symbols.get(i)).isEqualTo(originalSymbolById.get(ids.get(i).intValue())); + } + } + + @Test + void generatesConstantAndBoolColumns(@TempDir Path dir) throws IOException { + // Given a constant and a random-bool column + List columns = List.of( + DescriptorParser.parse("flag:bool:constant(true)"), + DescriptorParser.parse("active:bool:bool()")); + Path out = dir.resolve("bools.vortex"); + + // When + FakeDataGenerator.generate(columns, 50, 3L, null, 65_536, 3, out); + + // Then + List flags = readBoolColumn(out, "flag"); + assertThat(flags).hasSize(50).containsOnly(true); + } + + @Test + void writesMultipleChunksWhenRowsExceedChunkSize(@TempDir Path dir) throws IOException { + // Given a chunk size smaller than the row count + List columns = List.of(DescriptorParser.parse("id:i64:series(0,1)")); + Path out = dir.resolve("chunked.vortex"); + + // When + FakeDataGenerator.generate(columns, 1000, 42L, null, 100, 3, out); + + // Then all rows are still present, in order, across chunk boundaries + List values = readLongColumn(out, "id"); + assertThat(values).hasSize(1000); + for (long i = 0; i < 1000; i++) { + assertThat(values.get((int) i)).isEqualTo(i); + } + } + + private static List readLongColumn(Path file, String column) throws IOException { + List out = new ArrayList<>(); + try (VortexReader vf = VortexReader.open(file); var iter = vf.scan(ScanOptions.all())) { + while (iter.hasNext()) { + try (var chunk = iter.next()) { + LongArray array = chunk.column(column); + for (long i = 0; i < array.length(); i++) { + out.add(array.getLong(i)); + } + } + } + } + return out; + } + + private static List readUtf8Column(Path file, String column) throws IOException { + List out = new ArrayList<>(); + try (VortexReader vf = VortexReader.open(file); var iter = vf.scan(ScanOptions.all())) { + while (iter.hasNext()) { + try (var chunk = iter.next()) { + VarBinArray array = chunk.column(column); + for (long i = 0; i < array.length(); i++) { + out.add(new String(array.getBytes(i))); + } + } + } + } + return out; + } + + private static List readBoolColumn(Path file, String column) throws IOException { + List out = new ArrayList<>(); + try (VortexReader vf = VortexReader.open(file); var iter = vf.scan(ScanOptions.all())) { + while (iter.hasNext()) { + try (var chunk = iter.next()) { + BoolArray array = chunk.column(column); + for (long i = 0; i < array.length(); i++) { + out.add(array.getBoolean(i)); + } + } + } + } + return out; + } +} diff --git a/demo/pom.xml b/demo/pom.xml index 1152980f..35e77f94 100644 --- a/demo/pom.xml +++ b/demo/pom.xml @@ -12,8 +12,9 @@ vortex-demo-parent Aggregator for the standalone HTTP range demo: vortex-server (a minimal - object-storage HTTP server) and vortex-demo (the client that generates data, uploads it, - and runs a filtered/projected VortexHttpReader scan against it). + object-storage HTTP server), vortex-fakedata-generator (a CLI for generating synthetic + Vortex files), and vortex-demo (the client that uploads a file and runs a + filtered/projected VortexHttpReader scan against it). @@ -24,6 +25,7 @@ server + fakedata-generator client diff --git a/demo/server/pom.xml b/demo/server/pom.xml index 68c6ac49..8218988e 100644 --- a/demo/server/pom.xml +++ b/demo/server/pom.xml @@ -60,7 +60,7 @@ vortex-server - io.github.dfa1.vortex.server.VortexServer + io.github.dfa1.vortex.demo.server.VortexServer diff --git a/demo/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java b/demo/server/src/main/java/io/github/dfa1/vortex/demo/server/VortexServer.java similarity index 99% rename from demo/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java rename to demo/server/src/main/java/io/github/dfa1/vortex/demo/server/VortexServer.java index 7e16069d..70c171a6 100644 --- a/demo/server/src/main/java/io/github/dfa1/vortex/server/VortexServer.java +++ b/demo/server/src/main/java/io/github/dfa1/vortex/demo/server/VortexServer.java @@ -1,4 +1,4 @@ -package io.github.dfa1.vortex.server; +package io.github.dfa1.vortex.demo.server; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; diff --git a/demo/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java b/demo/server/src/test/java/io/github/dfa1/vortex/demo/server/VortexServerTest.java similarity index 99% rename from demo/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java rename to demo/server/src/test/java/io/github/dfa1/vortex/demo/server/VortexServerTest.java index 3686da9b..09b83d32 100644 --- a/demo/server/src/test/java/io/github/dfa1/vortex/server/VortexServerTest.java +++ b/demo/server/src/test/java/io/github/dfa1/vortex/demo/server/VortexServerTest.java @@ -1,4 +1,4 @@ -package io.github.dfa1.vortex.server; +package io.github.dfa1.vortex.demo.server; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/pom.xml b/pom.xml index 9d063ac1..707281ac 100644 --- a/pom.xml +++ b/pom.xml @@ -199,6 +199,11 @@ vortex-server ${project.version} + + io.github.dfa1.vortex + vortex-fakedata-generator + ${project.version} + de.siegmar fastcsv From 8d370cdc5439c3d1b8e164bbae48a1f2dc74d6e1 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 08:56:19 +0200 Subject: [PATCH 04/17] docs(demo): add a walkthrough README for the live HTTP range demo Step-by-step for presenting: build once, two-terminal live run (server log visibly showing small Range fetches vs. one big download), a one-terminal convenience mode, and how to customize the filter/schema for a different story. Cross-links issue #378 (the globalDict pruning gap this demo surfaced) since it explains why the generator forces globalDict=false. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- demo/README.md | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 demo/README.md diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 00000000..aae48a88 --- /dev/null +++ b/demo/README.md @@ -0,0 +1,105 @@ +# Vortex HTTP range demo + +Shows that a filtered, projected `VortexHttpReader` scan over HTTP fetches only the bytes it +actually touches — not the whole file — using nothing but plain HTTP `Range` requests. No cloud +account, no query service: just a `.vortex` file sitting behind a minimal object-storage server. + +Three standalone tools: + +| Module | Artifact | What it does | +|---|---|---| +| `fakedata-generator` | `vortex-fakedata-generator.jar` | Generates a synthetic `.vortex` file from a compact column-description grammar | +| `server` | `vortex-server.jar` | Minimal object-storage HTTP server — `GET`/`HEAD` with byte-range support, `PUT`, listing | +| `client` | `vortex-demo.jar` | Uploads a file, runs a filtered/projected scan, reports bytes fetched vs. the file's full size | + +## Build once + +```bash +./mvnw package -pl demo/server,demo/fakedata-generator,demo/client -am -DskipTests +``` + +Produces: + +``` +demo/server/target/vortex-server.jar +demo/fakedata-generator/target/vortex-fakedata-generator.jar +demo/client/target/vortex-demo.jar +``` + +## Run the live demo (two terminals) + +**Terminal 1 — start the server** (serves whatever directory you point it at, on whatever port): + +```bash +java -jar demo/server/target/vortex-server.jar 8080 /tmp/vortex-demo-data +``` + +Leave it running — it logs every request it serves, which is the point: you'll watch it print a +handful of small `Range` fetches instead of one big download. + +**Terminal 2 — generate the dataset:** + +```bash +java -jar demo/fakedata-generator/target/vortex-fakedata-generator.jar \ + --rows 2000000 --out /tmp/trades.vortex --sort-by symbol \ + "timestamp:i64:series(1700000000000,1000)" \ + "symbol:utf8:enum(SYM,30)" \ + "price:f64:range(50,150)" \ + "volume:i64:range(100,10000)" +``` + +2,000,000 rows across 30 symbols, sorted by symbol — the sort is what makes zone-map pruning +dramatic for a single-symbol filter (each symbol ends up clustered into just one or two chunks). + +**Terminal 2 — upload and scan it:** + +```bash +java -jar demo/client/target/vortex-demo.jar \ + --file /tmp/trades.vortex --server http://127.0.0.1:8080/ +``` + +Expected output (numbers will vary slightly with row count): + +``` +Uploading to http://127.0.0.1:8080/trades.vortex ... + +Scanning for symbol=SYM015, projecting 'price' over HTTP... + +Matched rows: 65536 +Bytes fetched over HTTP during the scan: 513,912 / 24,550,442 (2.09% of the object) +``` + +Switch back to **Terminal 1** — you'll see the `PUT` (the upload) followed by a small number of +`GET ... range=bytes=...` lines, each a few hundred KB, not one 24 MB download. + +## One-terminal version (no server to manage) + +Omit `--server` and the client embeds its own: + +```bash +java -jar demo/client/target/vortex-demo.jar --file /tmp/trades.vortex +``` + +Good for a quick local check; the two-terminal version is more compelling live, since the +audience can watch the server's request log update in real time. + +## Customizing the story + +- **Different filter/projection** — `vortex-demo` accepts `--filter-column`, `--filter-value`, + and `--project` (defaults: `symbol`, `SYM015`, `price`). Match these to whatever schema you + generate. +- **Different dataset shape** — `vortex-fakedata-generator`'s column grammar is + `name:type:generator(args)`. Run it with no arguments for the full grammar reference + (types, generators, an example). The `series(start,step)` generator is a direct nod to SQL's + `generate_series`. +- **No pruning story without `--sort-by`** — if rows for a given filter value are scattered + across every chunk instead of clustered, zone-map pruning has nothing to skip and the scan + degrades toward "fetch most of the file." Sorting by the column you intend to filter on is + what makes the demo's numbers dramatic. + +## Known gap this demo surfaced + +`WriteOptions`'s default `globalDict=true` silently defeats zone-map pruning for `Utf8` columns +(see [issue #378](https://github.com/dfa1/vortex-java/issues/378)) — `vortex-fakedata-generator` +writes with `globalDict=false` to work around it. If you generate a file some other way and don't +see any pruning, check that setting first. From dcf9184dbf1e94475f0989bb67bafbe257dfb099 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 09:13:41 +0200 Subject: [PATCH 05/17] feat(demo): add a progress bar with ETA to vortex-fakedata-generator A throttled (max 10 redraws/sec), in-place terminal progress bar printed to stderr during the chunk-writing loop -- the I/O/encoding- bound phase, unlike in-memory column materialization which is near- instant even at millions of rows. Shows rows written, elapsed time, and an ETA extrapolated from the current completion fraction. Verified against a 50M-row generation (~11s) to see a real non-zero ETA. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- demo/README.md | 3 + .../demo/fakedata/FakeDataGenerator.java | 3 + .../vortex/demo/fakedata/ProgressBar.java | 61 +++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java diff --git a/demo/README.md b/demo/README.md index aae48a88..4fb6f17b 100644 --- a/demo/README.md +++ b/demo/README.md @@ -48,6 +48,9 @@ java -jar demo/fakedata-generator/target/vortex-fakedata-generator.jar \ "volume:i64:range(100,10000)" ``` +Prints a live progress bar with ETA to stderr while writing (throttled, so it won't flood the +terminal — only really visible on larger row counts). + 2,000,000 rows across 30 symbols, sorted by symbol — the sort is what makes zone-map pruning dramatic for a single-symbol filter (each symbol ends up clustered into just one or two chunks). diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java index b9272abf..7b5b363d 100644 --- a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java @@ -57,12 +57,15 @@ public static void generate(List columns, int rows, long seed, // vortex-java's own reader module): the whole point of a fakedata tool feeding demos // that showcase pruning/partial fetches is to keep per-chunk stats meaningful. WriteOptions options = WriteOptions.cascading(cascading).withGlobalDict(false); + ProgressBar progress = new ProgressBar(rows); try (FileChannel channel = FileChannel.open(out, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); VortexWriter writer = VortexWriter.create(channel, schema, options)) { + progress.update(0); for (int start = 0; start < rows; start += chunkSize) { int n = Math.min(chunkSize, rows - start); writer.writeChunk(sliceChunk(columns, columnArrays, start, n)); + progress.update(start + n); } } } diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java new file mode 100644 index 00000000..ea1fd443 --- /dev/null +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java @@ -0,0 +1,61 @@ +package io.github.dfa1.vortex.demo.fakedata; + +import java.time.Duration; + +/// A single-line, in-place terminal progress bar with an ETA, printed to `System.err` (so it +/// never mixes with a tool's normal `stdout` output). Redraws are throttled to avoid flooding the +/// terminal on fast runs. +final class ProgressBar { + + private static final int BAR_WIDTH = 30; + private static final Duration MIN_REDRAW_INTERVAL = Duration.ofMillis(100); + + private final long total; + private final long startNanos; + private long lastDrawNanos = Long.MIN_VALUE; + + ProgressBar(long total) { + this.total = total; + this.startNanos = System.nanoTime(); + } + + /// Redraws the bar for `done` out of the total, unless the minimum redraw interval hasn't + /// elapsed yet (ignored for the final call, `done == total`, which always draws). + /// + /// @param done rows completed so far, in `[0, total]` + void update(long done) { + long nowNanos = System.nanoTime(); + boolean isFinal = done >= total; + if (!isFinal && Duration.ofNanos(nowNanos - lastDrawNanos).compareTo(MIN_REDRAW_INTERVAL) < 0) { + return; + } + lastDrawNanos = nowNanos; + + double fraction = total == 0 ? 1.0 : Math.min(1.0, (double) done / total); + Duration elapsed = Duration.ofNanos(nowNanos - startNanos); + Duration eta = estimateRemaining(fraction, elapsed); + + int filled = (int) (fraction * BAR_WIDTH); + String bar = "=".repeat(filled) + " ".repeat(BAR_WIDTH - filled); + System.err.printf("\r[%s] %5.1f%% %,d/%,d rows elapsed=%s eta=%s", + bar, fraction * 100, done, total, format(elapsed), isFinal ? format(Duration.ZERO) : format(eta)); + if (isFinal) { + System.err.println(); + } + } + + private static Duration estimateRemaining(double fraction, Duration elapsed) { + if (fraction <= 0) { + return Duration.ZERO; + } + double totalEstimateNanos = elapsed.toNanos() / fraction; + return Duration.ofNanos((long) totalEstimateNanos).minus(elapsed); + } + + private static String format(Duration d) { + long totalSeconds = Math.max(0, d.toSeconds()); + long minutes = totalSeconds / 60; + long seconds = totalSeconds % 60; + return minutes > 0 ? "%dm%02ds".formatted(minutes, seconds) : "%ds".formatted(seconds); + } +} From 4dbe7860ea3cb9d7ea7581fa30f0f574bc2838f6 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 19:24:18 +0200 Subject: [PATCH 06/17] feat(demo): show a live download counter while the client scans A virtual-thread background poller hits the server's /_stats endpoint every 100ms while the scan runs, printing an in-place-updating "bytes downloaded so far" line to stderr. Verified against a 20M-row/245MB file: the counter genuinely climbs through real intermediate values (0.8% -> 11.7% -> 26.8% -> 43.9% -> final 59.2%) rather than just jumping straight to the final number -- the point being visible on stage that the scan stops well short of the file's full size, not just told after the fact. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- demo/README.md | 6 +++ .../vortex/demo/client/HttpRangeDemo.java | 49 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/demo/README.md b/demo/README.md index 4fb6f17b..c6cb0c7e 100644 --- a/demo/README.md +++ b/demo/README.md @@ -61,6 +61,11 @@ java -jar demo/client/target/vortex-demo.jar \ --file /tmp/trades.vortex --server http://127.0.0.1:8080/ ``` +While the scan runs, a live "bytes downloaded so far" counter updates in place on stderr — the +audience watches it climb a little, then stop well short of the file's full size, rather than +just seeing a single number appear at the end. It's mostly visible on larger row counts, since a +small scan can finish before the first redraw. + Expected output (numbers will vary slightly with row count): ``` @@ -68,6 +73,7 @@ Uploading to http://127.0.0.1:8080/trades.vortex ... Scanning for symbol=SYM015, projecting 'price' over HTTP... + Downloaded so far: 401,644 / 24,550,442 bytes (1.6%) Matched rows: 65536 Bytes fetched over HTTP during the scan: 513,912 / 24,550,442 (2.09% of the object) ``` diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java index 7b4e2d6d..1c677ee1 100644 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java +++ b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java @@ -12,6 +12,8 @@ import java.net.http.HttpResponse; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -37,6 +39,7 @@ public final class HttpRangeDemo { private static final String DEFAULT_PROJECT_COLUMN = "price"; private static final Pattern BYTES_SERVED_LINE = Pattern.compile("bytesServed=(\\d+)"); + private static final Duration POLL_INTERVAL = Duration.ofMillis(100); private HttpRangeDemo() { } @@ -113,16 +116,58 @@ private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize, long bytesServedBefore = readBytesServed(client, serverBaseUri); System.out.printf("%nScanning for %s=%s, projecting '%s' over HTTP...%n%n", filterColumn, filterValue, projectColumn); - long rows = scan(objectUri, filterColumn, filterValue, projectColumn); + long rows = scanWithLiveDownloadCounter(client, serverBaseUri, objectUri, bytesServedBefore, fileSize, + filterColumn, filterValue, projectColumn); long bytesServedAfter = readBytesServed(client, serverBaseUri); long servedDuringScan = bytesServedAfter - bytesServedBefore; - System.out.println(); System.out.println("Matched rows: " + rows); System.out.printf("Bytes fetched over HTTP during the scan: %,d / %,d (%.2f%% of the object)%n", servedDuringScan, fileSize, 100.0 * servedDuringScan / fileSize); } + /// Runs [#scan] while a background thread polls the server's `/_stats` endpoint and prints a + /// live, in-place-updating "bytes downloaded so far" line to stderr — the point being visible + /// on stage: the number climbs a little, then stops well short of the file's full size, + /// instead of the scan just silently returning a final count. Only really visible to the eye + /// on a large enough dataset that the scan takes more than a poll interval or two; on a small + /// file the whole scan finishes before the first redraw and this degrades gracefully to + /// printing just the final line. + private static long scanWithLiveDownloadCounter(HttpClient client, URI serverBaseUri, URI objectUri, + long bytesServedBefore, long fileSize, String filterColumn, String filterValue, String projectColumn) + throws IOException, InterruptedException { + AtomicBoolean scanning = new AtomicBoolean(true); + Thread poller = Thread.ofVirtual().name("download-progress").start(() -> { + while (scanning.get()) { + try { + long servedSoFar = readBytesServed(client, serverBaseUri) - bytesServedBefore; + System.err.printf("\r Downloaded so far: %,d / %,d bytes (%.1f%%)", + servedSoFar, fileSize, 100.0 * servedSoFar / fileSize); + Thread.sleep(POLL_INTERVAL); + } catch (IOException | InterruptedException e) { + return; + } + } + }); + + try { + return scan(objectUri, filterColumn, filterValue, projectColumn); + } finally { + scanning.set(false); + poller.interrupt(); + joinQuietly(poller); + System.err.println(); + } + } + + private static void joinQuietly(Thread thread) { + try { + thread.join(Duration.ofSeconds(1)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + private static void upload(HttpClient client, URI objectUri, Path localFile) throws IOException, InterruptedException { HttpRequest put = HttpRequest.newBuilder(objectUri) From 083553fdfda34bf05e4561a79188524387aedf6b Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 19:36:00 +0200 Subject: [PATCH 07/17] feat(demo): drop --sort-by, stream generation, switch default to a time-range filter Now that #378/#379/#380 are fixed upstream, the workarounds and the --sort-by knob are no longer needed: - FakeDataGenerator no longer materializes every column fully in memory before writing (Object[] columnArrays + a post-hoc sort permutation). Generation is now truly one-chunk-at-a-time streaming: ColumnMaterializer.materialize takes a rowOffset so a `series(...)` column stays a correct arithmetic progression across chunk boundaries. Verified against 200,000,000 rows with a 512MB heap cap: completes in ~80s, no OutOfMemoryError (the old materialize-everything path OOM'd on this same input). The progress bar now genuinely tracks the whole run instead of jumping straight to 100% after a silent materialization phase. - --sort-by is gone: real data never arrives pre-sorted by whatever column a later query filters on, so faking that clustering misrepresented the workload. The withGlobalDict(false) workaround for #378 is gone too, now that the fix landed. - vortex-demo's default filter switches from symbol equality (meaningless without --sort-by -- an enum() column is scattered uniformly across every chunk by design) to a numeric range on timestamp: a series(...) column is naturally ordered by row position with no sorting needed, matching real append-order ingestion. New --filter-min/--filter-max flags replace --filter-value. End-to-end verified on the default 2,000,000-row example: 5.55% of the file fetched over HTTP, real compression (cascading(3), the generator's own default), zero workarounds. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- demo/README.md | 71 ++++---- .../vortex/demo/client/HttpRangeDemo.java | 62 ++++--- .../demo/fakedata/ColumnMaterializer.java | 21 ++- .../demo/fakedata/FakeDataGenerator.java | 158 ++---------------- .../demo/fakedata/FakeDataGeneratorCli.java | 14 +- .../demo/fakedata/FakeDataGeneratorTest.java | 43 +---- 6 files changed, 124 insertions(+), 245 deletions(-) diff --git a/demo/README.md b/demo/README.md index c6cb0c7e..7651272d 100644 --- a/demo/README.md +++ b/demo/README.md @@ -10,7 +10,7 @@ Three standalone tools: |---|---|---| | `fakedata-generator` | `vortex-fakedata-generator.jar` | Generates a synthetic `.vortex` file from a compact column-description grammar | | `server` | `vortex-server.jar` | Minimal object-storage HTTP server — `GET`/`HEAD` with byte-range support, `PUT`, listing | -| `client` | `vortex-demo.jar` | Uploads a file, runs a filtered/projected scan, reports bytes fetched vs. the file's full size | +| `client` | `vortex-demo.jar` | Uploads a file, runs a time-range-filtered/projected scan, reports bytes fetched vs. the file's full size | ## Build once @@ -41,7 +41,7 @@ handful of small `Range` fetches instead of one big download. ```bash java -jar demo/fakedata-generator/target/vortex-fakedata-generator.jar \ - --rows 2000000 --out /tmp/trades.vortex --sort-by symbol \ + --rows 2000000 --out /tmp/trades.vortex \ "timestamp:i64:series(1700000000000,1000)" \ "symbol:utf8:enum(SYM,30)" \ "price:f64:range(50,150)" \ @@ -49,10 +49,10 @@ java -jar demo/fakedata-generator/target/vortex-fakedata-generator.jar \ ``` Prints a live progress bar with ETA to stderr while writing (throttled, so it won't flood the -terminal — only really visible on larger row counts). - -2,000,000 rows across 30 symbols, sorted by symbol — the sort is what makes zone-map pruning -dramatic for a single-symbol filter (each symbol ends up clustered into just one or two chunks). +terminal — only really visible on larger row counts). 2,000,000 rows, real compression +(`cascading(3)`, the generator's default) — no clustering trick needed. The `timestamp` column is +a `series(...)`, so it's naturally in row order simply because that's the order it was generated +in, exactly like a real append-only ingestion stream. **Terminal 2 — upload and scan it:** @@ -61,21 +61,23 @@ java -jar demo/client/target/vortex-demo.jar \ --file /tmp/trades.vortex --server http://127.0.0.1:8080/ ``` -While the scan runs, a live "bytes downloaded so far" counter updates in place on stderr — the -audience watches it climb a little, then stop well short of the file's full size, rather than -just seeing a single number appear at the end. It's mostly visible on larger row counts, since a -small scan can finish before the first redraw. +By default this filters `timestamp` to a narrow window (50,000 of the 2,000,000 rows) and +projects `price` — a realistic "give me this time range" query, not an equality match on some +other column. While the scan runs, a live "bytes downloaded so far" counter updates in place on +stderr — the audience watches it climb a little, then stop well short of the file's full size, +rather than just seeing a single number appear at the end. It's mostly visible on larger row +counts, since a small scan can finish before the first redraw. Expected output (numbers will vary slightly with row count): ``` Uploading to http://127.0.0.1:8080/trades.vortex ... -Scanning for symbol=SYM015, projecting 'price' over HTTP... +Scanning for 1701000000000 <= timestamp <= 1701049999000, projecting 'price' over HTTP... - Downloaded so far: 401,644 / 24,550,442 bytes (1.6%) -Matched rows: 65536 -Bytes fetched over HTTP during the scan: 513,912 / 24,550,442 (2.09% of the object) + Downloaded so far: 401,644 / 24,552,440 bytes (1.6%) +Matched rows: 131072 +Bytes fetched over HTTP during the scan: 1,363,118 / 24,552,440 (5.55% of the object) ``` Switch back to **Terminal 1** — you'll see the `PUT` (the upload) followed by a small number of @@ -94,21 +96,34 @@ audience can watch the server's request log update in real time. ## Customizing the story -- **Different filter/projection** — `vortex-demo` accepts `--filter-column`, `--filter-value`, - and `--project` (defaults: `symbol`, `SYM015`, `price`). Match these to whatever schema you - generate. +- **Different filter/projection** — `vortex-demo` accepts `--filter-column`, `--filter-min`, + `--filter-max`, and `--project` (numeric-range filter, not equality — see below for why). Match + these to whatever schema you generate. - **Different dataset shape** — `vortex-fakedata-generator`'s column grammar is `name:type:generator(args)`. Run it with no arguments for the full grammar reference (types, generators, an example). The `series(start,step)` generator is a direct nod to SQL's `generate_series`. -- **No pruning story without `--sort-by`** — if rows for a given filter value are scattered - across every chunk instead of clustered, zone-map pruning has nothing to skip and the scan - degrades toward "fetch most of the file." Sorting by the column you intend to filter on is - what makes the demo's numbers dramatic. - -## Known gap this demo surfaced - -`WriteOptions`'s default `globalDict=true` silently defeats zone-map pruning for `Utf8` columns -(see [issue #378](https://github.com/dfa1/vortex-java/issues/378)) — `vortex-fakedata-generator` -writes with `globalDict=false` to work around it. If you generate a file some other way and don't -see any pruning, check that setting first. +- **Why a range filter on `timestamp`, not equality on `symbol`** — there is deliberately no + "sort the rows before writing" option anywhere in this demo: real data never arrives pre-sorted + by whatever column a later query happens to filter on, so faking that clustering would + misrepresent the workload this is meant to demonstrate. A `series(...)` column is naturally + ordered by row position without any sorting, which zone-map pruning can exploit for a + range query; an `enum(...)` column's values are scattered uniformly across every chunk by + design, so an equality filter on it (e.g. `symbol == "SYM015"`) has nothing to prune — every + chunk could contain a match. + +## Bugs this demo surfaced (now fixed upstream) + +Building this surfaced three real gaps in zone-map pruning, filed and since fixed in vortex-java's +`reader`/`writer` modules: + +- [#378](https://github.com/dfa1/vortex-java/issues/378) — `WriteOptions`'s default + `globalDict=true` silently defeated zone-map pruning for `Utf8` columns. +- [#379](https://github.com/dfa1/vortex-java/issues/379) — a cascade-selected encoder + (`cascading(depth) > 0`) dropped zone-map min/max stats, keeping only `sum`. +- [#380](https://github.com/dfa1/vortex-java/issues/380) — checking whether an HTTP-backed chunk + could be pruned fetched that chunk's *entire* segment first, costing as much bandwidth as just + reading it. + +This demo's numbers reflect the fixed behavior. If you're running against an older vortex-java +build, pruning may not work and the byte percentage will be much higher than shown above. diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java index 1c677ee1..dc604b3b 100644 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java +++ b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java @@ -19,8 +19,14 @@ /// Demo client: uploads an existing Vortex file (e.g. one produced by /// `vortex-fakedata-generator`) to a `vortex-server` object store, then runs a -/// filtered/projected [VortexHttpReader] scan against it — printing how many bytes the scan -/// actually pulled over the wire against the object's full size. +/// time-range-filtered, projected [VortexHttpReader] scan against it — printing how many bytes +/// the scan actually pulled over the wire against the object's full size. +/// +/// Filters by a numeric column range (`timestamp` by default) rather than an equality match on +/// some other column: real data is written in the order it arrives, so a time-ordered column is +/// naturally clustered by row position even without any artificial sorting -- zone-map pruning +/// narrows a time-range query to the handful of chunks that actually overlap it, no +/// `--sort-by`-style clustering needed. /// /// Run standalone (embeds its own [VortexServer]): /// ``` @@ -34,8 +40,11 @@ /// ``` public final class HttpRangeDemo { - private static final String DEFAULT_FILTER_COLUMN = "symbol"; - private static final String DEFAULT_FILTER_VALUE = "SYM015"; + private static final String DEFAULT_FILTER_COLUMN = "timestamp"; + // Matches vortex-fakedata-generator's own README example: timestamp:i64:series(1700000000000,1000) + // over 2,000,000 rows. This window covers rows [1_000_000, 1_050_000) -- 50,000 of 2,000,000 rows. + private static final long DEFAULT_FILTER_MIN = 1_701_000_000_000L; + private static final long DEFAULT_FILTER_MAX = 1_701_049_999_000L; private static final String DEFAULT_PROJECT_COLUMN = "price"; private static final Pattern BYTES_SERVED_LINE = Pattern.compile("bytesServed=(\\d+)"); @@ -60,7 +69,8 @@ private static void run(String[] args) throws IOException, InterruptedException Path file = null; URI serverBaseUri = null; String filterColumn = DEFAULT_FILTER_COLUMN; - String filterValue = DEFAULT_FILTER_VALUE; + long filterMin = DEFAULT_FILTER_MIN; + long filterMax = DEFAULT_FILTER_MAX; String projectColumn = DEFAULT_PROJECT_COLUMN; int i = 0; @@ -78,8 +88,12 @@ private static void run(String[] args) throws IOException, InterruptedException filterColumn = args[++i]; i++; } - case "--filter-value" -> { - filterValue = args[++i]; + case "--filter-min" -> { + filterMin = Long.parseLong(args[++i]); + i++; + } + case "--filter-max" -> { + filterMax = Long.parseLong(args[++i]); i++; } case "--project" -> { @@ -96,17 +110,18 @@ private static void run(String[] args) throws IOException, InterruptedException long fileSize = Files.size(file); if (serverBaseUri != null) { - runAgainst(serverBaseUri, file, fileSize, filterColumn, filterValue, projectColumn); + runAgainst(serverBaseUri, file, fileSize, filterColumn, filterMin, filterMax, projectColumn); } else { try (VortexServer server = VortexServer.start(Files.createTempDirectory("vortex-server"), 0)) { System.out.println("Embedded vortex-server at " + server.baseUri()); - runAgainst(server.baseUri(), file, fileSize, filterColumn, filterValue, projectColumn); + runAgainst(server.baseUri(), file, fileSize, filterColumn, filterMin, filterMax, projectColumn); } } } private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize, - String filterColumn, String filterValue, String projectColumn) throws IOException, InterruptedException { + String filterColumn, long filterMin, long filterMax, String projectColumn) + throws IOException, InterruptedException { HttpClient client = HttpClient.newHttpClient(); URI objectUri = serverBaseUri.resolve(localFile.getFileName().toString()); @@ -114,10 +129,10 @@ private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize, upload(client, objectUri, localFile); long bytesServedBefore = readBytesServed(client, serverBaseUri); - System.out.printf("%nScanning for %s=%s, projecting '%s' over HTTP...%n%n", - filterColumn, filterValue, projectColumn); + System.out.printf("%nScanning for %d <= %s <= %d, projecting '%s' over HTTP...%n%n", + filterMin, filterColumn, filterMax, projectColumn); long rows = scanWithLiveDownloadCounter(client, serverBaseUri, objectUri, bytesServedBefore, fileSize, - filterColumn, filterValue, projectColumn); + filterColumn, filterMin, filterMax, projectColumn); long bytesServedAfter = readBytesServed(client, serverBaseUri); long servedDuringScan = bytesServedAfter - bytesServedBefore; @@ -134,8 +149,8 @@ private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize, /// file the whole scan finishes before the first redraw and this degrades gracefully to /// printing just the final line. private static long scanWithLiveDownloadCounter(HttpClient client, URI serverBaseUri, URI objectUri, - long bytesServedBefore, long fileSize, String filterColumn, String filterValue, String projectColumn) - throws IOException, InterruptedException { + long bytesServedBefore, long fileSize, String filterColumn, long filterMin, long filterMax, + String projectColumn) throws IOException, InterruptedException { AtomicBoolean scanning = new AtomicBoolean(true); Thread poller = Thread.ofVirtual().name("download-progress").start(() -> { while (scanning.get()) { @@ -151,7 +166,7 @@ private static long scanWithLiveDownloadCounter(HttpClient client, URI serverBas }); try { - return scan(objectUri, filterColumn, filterValue, projectColumn); + return scan(objectUri, filterColumn, filterMin, filterMax, projectColumn); } finally { scanning.set(false); poller.interrupt(); @@ -190,11 +205,10 @@ private static long readBytesServed(HttpClient client, URI serverBaseUri) return Long.parseLong(m.group(1)); } - private static long scan(URI objectUri, String filterColumn, String filterValue, String projectColumn) + private static long scan(URI objectUri, String filterColumn, long filterMin, long filterMax, String projectColumn) throws IOException { - ScanOptions opts = ScanOptions.all() - .withColumns(filterColumn, projectColumn) - .withFilter(RowFilter.eq(filterColumn, filterValue)); + RowFilter filter = RowFilter.gte(filterColumn, filterMin).and(RowFilter.lte(filterColumn, filterMax)); + ScanOptions opts = ScanOptions.all().withColumns(filterColumn, projectColumn).withFilter(filter); long rows = 0; try (VortexHttpReader vf = VortexHttpReader.open(objectUri); @@ -215,12 +229,14 @@ private static void printUsage() { Options: --file FILE local .vortex file to upload and scan (required) --server URI base URI of an already-running vortex-server; omit to embed one - --filter-column NAME column to filter on (default: symbol) - --filter-value VALUE value to filter for (default: SYM015) + --filter-column NAME numeric column to range-filter on (default: timestamp) + --filter-min N inclusive lower bound (default: matches the README example's + row [1000000, 1050000) window) + --filter-max N inclusive upper bound --project NAME column to project (default: price) Generate a file first with vortex-fakedata-generator, e.g.: - vortex-fakedata-generator --rows 2000000 --out trades.vortex --sort-by symbol \\ + vortex-fakedata-generator --rows 2000000 --out trades.vortex \\ "timestamp:i64:series(1700000000000,1000)" \\ "symbol:utf8:enum(SYM,30)" \\ "price:f64:range(50,150)" \\ diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java index 18f283fc..6bbdc61f 100644 --- a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java @@ -51,13 +51,18 @@ private static boolean isValidConstant(DType dtype, String literal) { } } - /// Materializes `rows` values for `column`. + /// Materializes `rows` values for `column`, continuing from absolute row index `rowOffset` + /// (only meaningful to [GeneratorSpec.Series], so a chunk-by-chunk streaming caller gets the + /// same arithmetic progression as materializing the whole column at once would). /// - /// @param column the column to generate - /// @param rows number of rows to generate - /// @param random shared random source (consumed in column-declaration order for determinism) + /// @param column the column to generate + /// @param rowOffset absolute row index this call's first row corresponds to + /// @param rows number of rows to generate + /// @param random shared random source (consumed in column-declaration order for + /// determinism; the caller is responsible for not resetting it between + /// chunks of the same column) /// @return the typed array (`long[]`/`int[]`/.../`double[]`, `String[]`, or `boolean[]`) - static Object materialize(ColumnDescriptor column, int rows, Random random) { + static Object materialize(ColumnDescriptor column, long rowOffset, int rows, Random random) { DType dtype = column.dtype(); GeneratorSpec generator = column.generator(); if (dtype instanceof DType.Utf8) { @@ -67,16 +72,16 @@ static Object materialize(ColumnDescriptor column, int rows, Random random) { return materializeBool(generator, rows, random); } PType ptype = ((DType.Primitive) dtype).ptype(); - double[] values = materializeNumeric(generator, rows, random); + double[] values = materializeNumeric(generator, rowOffset, rows, random); return narrow(ptype, values); } - private static double[] materializeNumeric(GeneratorSpec generator, int rows, Random random) { + private static double[] materializeNumeric(GeneratorSpec generator, long rowOffset, int rows, Random random) { double[] values = new double[rows]; switch (generator) { case GeneratorSpec.Series(double start, double step) -> { for (int i = 0; i < rows; i++) { - values[i] = start + i * step; + values[i] = start + (rowOffset + i) * step; } } case GeneratorSpec.Range(double min, double max) -> { diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java index 7b5b363d..f674141e 100644 --- a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGenerator.java @@ -9,8 +9,6 @@ import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Arrays; -import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -19,6 +17,14 @@ /// Generates a Vortex file of synthetic data from a list of [ColumnDescriptor]s. The library /// entry point behind [FakeDataGeneratorCli]; callable directly by other code (e.g. a demo /// module) without shelling out to the CLI. +/// +/// Generation streams: each chunk is generated and written in turn, so memory stays bounded by +/// one chunk regardless of row count. There is no "sort by column" option — real data never +/// arrives pre-sorted by the column a query happens to filter on, so faking that clustering here +/// would misrepresent the workload a consumer of the generated file is actually meant to +/// exercise. A `series(start,step)` column is naturally ordered by row position without any +/// sorting, which is enough on its own for zone-map pruning on a time-range-style query — see +/// `vortex-demo`'s default filter. public final class FakeDataGenerator { private FakeDataGenerator() { @@ -27,36 +33,19 @@ private FakeDataGenerator() { /// @param columns column descriptors, in schema order /// @param rows total number of rows to generate /// @param seed seed for the shared random source (deterministic across runs) - /// @param sortBy column to sort all rows by (ascending), or `null` for generation order /// @param chunkSize rows per written chunk /// @param cascading write compression cascade depth (see `WriteOptions#cascading`) /// @param out destination path; created or truncated /// @throws IOException if writing fails - public static void generate(List columns, int rows, long seed, String sortBy, + public static void generate(List columns, int rows, long seed, int chunkSize, int cascading, Path out) throws IOException { if (columns.isEmpty()) { throw new IllegalArgumentException("at least one column descriptor is required"); } + DType.Struct schema = buildSchema(columns); + WriteOptions options = WriteOptions.cascading(cascading); Random random = new Random(seed); - Object[] columnArrays = new Object[columns.size()]; - for (int c = 0; c < columns.size(); c++) { - columnArrays[c] = ColumnMaterializer.materialize(columns.get(c), rows, random); - } - - if (sortBy != null) { - int sortColumnIndex = indexOf(columns, sortBy); - int[] permutation = sortPermutation(columnArrays[sortColumnIndex], rows); - for (int c = 0; c < columns.size(); c++) { - columnArrays[c] = permute(columnArrays[c], permutation); - } - } - - DType.Struct schema = buildSchema(columns); - // globalDict defeats zone-map pruning on Utf8 columns (see the CLAUDE.md/README of - // vortex-java's own reader module): the whole point of a fakedata tool feeding demos - // that showcase pruning/partial fetches is to keep per-chunk stats meaningful. - WriteOptions options = WriteOptions.cascading(cascading).withGlobalDict(false); ProgressBar progress = new ProgressBar(rows); try (FileChannel channel = FileChannel.open(out, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); @@ -64,7 +53,11 @@ public static void generate(List columns, int rows, long seed, progress.update(0); for (int start = 0; start < rows; start += chunkSize) { int n = Math.min(chunkSize, rows - start); - writer.writeChunk(sliceChunk(columns, columnArrays, start, n)); + Map chunk = new LinkedHashMap<>(); + for (ColumnDescriptor col : columns) { + chunk.put(col.name(), ColumnMaterializer.materialize(col, start, n, random)); + } + writer.writeChunk(chunk); progress.update(start + n); } } @@ -77,123 +70,4 @@ private static DType.Struct buildSchema(List columns) { } return builder.build(); } - - private static Map sliceChunk(List columns, Object[] columnArrays, - int start, int n) { - Map chunk = new LinkedHashMap<>(); - for (int c = 0; c < columns.size(); c++) { - chunk.put(columns.get(c).name(), slice(columnArrays[c], start, n)); - } - return chunk; - } - - private static int indexOf(List columns, String name) { - for (int i = 0; i < columns.size(); i++) { - if (columns.get(i).name().value().equals(name)) { - return i; - } - } - throw new IllegalArgumentException("--sort-by column '" + name + "' is not one of the declared columns"); - } - - private static int[] sortPermutation(Object array, int rows) { - Integer[] indices = new Integer[rows]; - for (int i = 0; i < rows; i++) { - indices[i] = i; - } - Comparator byValue = switch (array) { - case long[] a -> Comparator.comparingLong(i -> a[i]); - case int[] a -> Comparator.comparingInt(i -> a[i]); - case short[] a -> Comparator.comparingInt(i -> a[i]); - case byte[] a -> Comparator.comparingInt(i -> a[i]); - case double[] a -> Comparator.comparingDouble(i -> a[i]); - case float[] a -> Comparator.comparingDouble(i -> a[i]); - case String[] a -> Comparator.comparing(i -> a[i]); - case boolean[] a -> Comparator.comparingInt(i -> a[i] ? 1 : 0); - default -> throw new IllegalStateException("unsupported array type: " + array.getClass()); - }; - Arrays.sort(indices, byValue); - int[] out = new int[rows]; - for (int i = 0; i < rows; i++) { - out[i] = indices[i]; - } - return out; - } - - private static Object permute(Object array, int[] permutation) { - int n = permutation.length; - return switch (array) { - case long[] a -> { - long[] out = new long[n]; - for (int i = 0; i < n; i++) { - out[i] = a[permutation[i]]; - } - yield out; - } - case int[] a -> { - int[] out = new int[n]; - for (int i = 0; i < n; i++) { - out[i] = a[permutation[i]]; - } - yield out; - } - case short[] a -> { - short[] out = new short[n]; - for (int i = 0; i < n; i++) { - out[i] = a[permutation[i]]; - } - yield out; - } - case byte[] a -> { - byte[] out = new byte[n]; - for (int i = 0; i < n; i++) { - out[i] = a[permutation[i]]; - } - yield out; - } - case double[] a -> { - double[] out = new double[n]; - for (int i = 0; i < n; i++) { - out[i] = a[permutation[i]]; - } - yield out; - } - case float[] a -> { - float[] out = new float[n]; - for (int i = 0; i < n; i++) { - out[i] = a[permutation[i]]; - } - yield out; - } - case String[] a -> { - String[] out = new String[n]; - for (int i = 0; i < n; i++) { - out[i] = a[permutation[i]]; - } - yield out; - } - case boolean[] a -> { - boolean[] out = new boolean[n]; - for (int i = 0; i < n; i++) { - out[i] = a[permutation[i]]; - } - yield out; - } - default -> throw new IllegalStateException("unsupported array type: " + array.getClass()); - }; - } - - private static Object slice(Object array, int start, int n) { - return switch (array) { - case long[] a -> Arrays.copyOfRange(a, start, start + n); - case int[] a -> Arrays.copyOfRange(a, start, start + n); - case short[] a -> Arrays.copyOfRange(a, start, start + n); - case byte[] a -> Arrays.copyOfRange(a, start, start + n); - case double[] a -> Arrays.copyOfRange(a, start, start + n); - case float[] a -> Arrays.copyOfRange(a, start, start + n); - case String[] a -> Arrays.copyOfRange(a, start, start + n); - case boolean[] a -> Arrays.copyOfRange(a, start, start + n); - default -> throw new IllegalStateException("unsupported array type: " + array.getClass()); - }; - } } diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorCli.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorCli.java index 298279f8..2214a9d5 100644 --- a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorCli.java +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorCli.java @@ -34,7 +34,6 @@ private static void run(String[] args) throws IOException { long seed = DEFAULT_SEED; int chunkSize = DEFAULT_CHUNK_SIZE; int cascading = DEFAULT_CASCADING; - String sortBy = null; List descriptors = new ArrayList<>(); int i = 0; @@ -61,10 +60,6 @@ private static void run(String[] args) throws IOException { cascading = Integer.parseInt(args[++i]); i++; } - case "--sort-by" -> { - sortBy = args[++i]; - i++; - } default -> { descriptors.add(arg); i++; @@ -77,7 +72,7 @@ private static void run(String[] args) throws IOException { } List columns = descriptors.stream().map(DescriptorParser::parse).toList(); - FakeDataGenerator.generate(columns, rows, seed, sortBy, chunkSize, cascading, out); + FakeDataGenerator.generate(columns, rows, seed, chunkSize, cascading, out); System.out.printf("Wrote %d rows (%d columns) to %s%n", rows, columns.size(), out); } @@ -91,7 +86,6 @@ private static void printUsage() { --seed N random seed (default 42) --chunk-size N rows per written chunk (default 65536) --cascading N write compression cascade depth (default 3) - --sort-by COLUMN sort all rows by this column (ascending) before writing Column descriptor grammar: name:type:generator(args) type: i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 utf8 bool @@ -102,8 +96,12 @@ private static void printUsage() { constant(value) same value every row bool() random true/false + A series(start,step) column is naturally ordered by row position without any + sorting -- that alone is enough for zone-map pruning on a time-range query, no + "sort by column" option needed or offered. + Example: - vortex-fakedata-generator --rows 2000000 --out trades.vortex --sort-by symbol \\ + vortex-fakedata-generator --rows 2000000 --out trades.vortex \\ "timestamp:i64:series(1700000000000,1000)" \\ "symbol:utf8:enum(SYM,30)" \\ "price:f64:range(50,150)" \\ diff --git a/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorTest.java b/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorTest.java index bd852456..8de7b181 100644 --- a/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorTest.java +++ b/demo/fakedata-generator/src/test/java/io/github/dfa1/vortex/demo/fakedata/FakeDataGeneratorTest.java @@ -24,7 +24,7 @@ void generatesAnExactArithmeticSeries(@TempDir Path dir) throws IOException { Path out = dir.resolve("series.vortex"); // When - FakeDataGenerator.generate(columns, 6, 42L, null, 65_536, 3, out); + FakeDataGenerator.generate(columns, 6, 42L, 65_536, 3, out); // Then rows are the exact arithmetic progression: 10, 15, 20, 25, 30, 35 List values = readLongColumn(out, "id"); @@ -38,44 +38,13 @@ void generatesEnumLabelsWithinTheDeclaredCount(@TempDir Path dir) throws IOExcep Path out = dir.resolve("enum.vortex"); // When - FakeDataGenerator.generate(columns, 200, 7L, null, 65_536, 3, out); + FakeDataGenerator.generate(columns, 200, 7L, 65_536, 3, out); // Then every value is one of the 3 declared labels List values = readUtf8Column(out, "symbol"); assertThat(values).hasSize(200).allSatisfy(v -> assertThat(v).isIn("SYM0", "SYM1", "SYM2")); } - @Test - void sortByOrdersEveryColumnByTheSamePermutation(@TempDir Path dir) throws IOException { - // Given a series id column (its natural order reveals the sort permutation) and an - // enum symbol column, sorted by symbol - List columns = List.of( - DescriptorParser.parse("id:i64:series(0,1)"), - DescriptorParser.parse("symbol:utf8:enum(SYM,4)")); - Path out = dir.resolve("sorted.vortex"); - - // When - FakeDataGenerator.generate(columns, 500, 1L, "symbol", 65_536, 3, out); - - // Then the symbol column is non-decreasing... - List symbols = readUtf8Column(out, "symbol"); - assertThat(symbols).isSorted(); - // ...and each id still names the row that originally held that symbol (id i's symbol - // before sorting is deterministic from the same seed/generator, so this cross-checks - // that every column moved together under the same permutation, not just "symbol" itself). - List unsorted = List.of( - DescriptorParser.parse("id:i64:series(0,1)"), - DescriptorParser.parse("symbol:utf8:enum(SYM,4)")); - Path unsortedOut = dir.resolve("unsorted.vortex"); - FakeDataGenerator.generate(unsorted, 500, 1L, null, 65_536, 3, unsortedOut); - List originalSymbolById = readUtf8Column(unsortedOut, "symbol"); - - List ids = readLongColumn(out, "id"); - for (int i = 0; i < ids.size(); i++) { - assertThat(symbols.get(i)).isEqualTo(originalSymbolById.get(ids.get(i).intValue())); - } - } - @Test void generatesConstantAndBoolColumns(@TempDir Path dir) throws IOException { // Given a constant and a random-bool column @@ -85,7 +54,7 @@ void generatesConstantAndBoolColumns(@TempDir Path dir) throws IOException { Path out = dir.resolve("bools.vortex"); // When - FakeDataGenerator.generate(columns, 50, 3L, null, 65_536, 3, out); + FakeDataGenerator.generate(columns, 50, 3L, 65_536, 3, out); // Then List flags = readBoolColumn(out, "flag"); @@ -99,9 +68,11 @@ void writesMultipleChunksWhenRowsExceedChunkSize(@TempDir Path dir) throws IOExc Path out = dir.resolve("chunked.vortex"); // When - FakeDataGenerator.generate(columns, 1000, 42L, null, 100, 3, out); + FakeDataGenerator.generate(columns, 1000, 42L, 100, 3, out); - // Then all rows are still present, in order, across chunk boundaries + // Then all rows are still present, in order, across chunk boundaries -- streaming + // generation must offset each chunk's series values by its absolute row index, not + // restart the arithmetic progression from 0 every chunk List values = readLongColumn(out, "id"); assertThat(values).hasSize(1000); for (long i = 0; i < 1000; i++) { From 5df46a1011364b39c42dc616cda086eb8c80dbdf Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 19:49:24 +0200 Subject: [PATCH 08/17] fix(demo): ProgressBar never redrew after the first call (signed overflow) lastDrawNanos started at Long.MIN_VALUE. The very first #update call computed `nowNanos - lastDrawNanos` -- a moderate positive nanoTime() value minus the most negative possible long -- which silently overflows a signed long and wraps to a negative Duration. A negative duration always compares as "less than the 100ms redraw interval", so the first (and every subsequent, since lastDrawNanos never advances away from MIN_VALUE either) non-final update got throttle-skipped for the entire run. Only the final call ever drew, since it bypasses the throttle check entirely -- exactly matching what looked like "the bar does nothing until the very end". Fixed by backdating the initial value by one redraw interval instead of using a sentinel, avoiding the overflow entirely (both operands stay close to System.nanoTime()'s own range). Verified empirically, not just reasoned about: polled the output file's size every second during a 200M-row run. Before the fix, it sat at 0 bytes for the entire ~65s run and jumped to ~60KB in one write at the very end. After, it grows continuously and roughly linearly throughout (0 -> 859 -> 1750 -> ... -> 60973 bytes). Also confirmed, by temporarily removing it and rerunning the same test, that an explicit System.err.flush() per redraw was never actually necessary -- PrintStream#write(String) already pushes every write through its internal encoding buffers unconditionally, and the underlying FileOutputStream has no buffering of its own to flush. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- .../github/dfa1/vortex/demo/fakedata/ProgressBar.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java index ea1fd443..830f3177 100644 --- a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java @@ -12,11 +12,19 @@ final class ProgressBar { private final long total; private final long startNanos; - private long lastDrawNanos = Long.MIN_VALUE; + private long lastDrawNanos; ProgressBar(long total) { this.total = total; this.startNanos = System.nanoTime(); + // Not Long.MIN_VALUE: `nowNanos - lastDrawNanos` in #update would overflow a signed long + // on the very first call (a moderate positive nanoTime() value minus the most negative + // possible long), silently wrapping to a negative duration that always looks "too soon + // to redraw" -- every non-final update gets throttle-skipped for the rest of the run, + // since lastDrawNanos then never advances away from MIN_VALUE either. Backdating by one + // interval instead guarantees the first real call passes the threshold, with no + // overflow risk since both operands stay close to System.nanoTime()'s own range. + this.lastDrawNanos = startNanos - MIN_REDRAW_INTERVAL.toNanos(); } /// Redraws the bar for `done` out of the total, unless the minimum redraw interval hasn't From 4c1448b3ccfd85d9d12030865a294f4e033462c3 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 19:58:04 +0200 Subject: [PATCH 09/17] perf+fix(demo): pool enum labels instead of reformatting per row; pad progress redraws ColumnMaterializer.materializeUtf8's enum(prefix,count) generator was calling String.formatted() once per row to build a zero-padded label, even though only `count` distinct labels ever exist. Measured: ~35% of total generation time for a 200M-row run with a 30-value enum column was attributable to this one column (67s -> 44s with it removed). Precomputing the small label pool once and indexing into it per row instead: 67.3s -> 57.5s for the same run (~15% faster overall; the remaining gap vs. the no-enum baseline is real write-side cost of encoding a 4th column, not a generation-side inefficiency). Also: ProgressBar's redraws didn't erase a longer previous line (\r only returns the cursor to column 0, it doesn't clear anything), so a shrinking field -- the ETA's digit count dropping as it counts down, for instance -- left trailing characters on screen (observed live: "eta=51s" rendered as "eta=51ss"). Now pads every redraw to the longest line drawn so far. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- .../dfa1/vortex/demo/fakedata/ColumnMaterializer.java | 11 ++++++++++- .../github/dfa1/vortex/demo/fakedata/ProgressBar.java | 9 ++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java index 6bbdc61f..eb982f5f 100644 --- a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ColumnMaterializer.java @@ -147,10 +147,19 @@ private static String[] materializeUtf8(GeneratorSpec generator, int rows, Rando String[] out = new String[rows]; switch (generator) { case GeneratorSpec.EnumLabels(String prefix, int count) -> { + // Only `count` distinct labels ever exist -- build that small pool once with + // String.formatted (fine, it runs `count` times, not `rows` times) and pick from + // it by array index per row. Reformatting a fresh String per row instead measured + // ~35% of total generation time for a 200M-row enum column: Java's Formatter + // machinery re-parses the format string on every call, unlike a plain array read. int width = String.valueOf(count - 1).length(); String format = prefix + "%0" + width + "d"; + String[] labels = new String[count]; + for (int v = 0; v < count; v++) { + labels[v] = format.formatted(v); + } for (int i = 0; i < rows; i++) { - out[i] = format.formatted(random.nextInt(count)); + out[i] = labels[random.nextInt(count)]; } } case GeneratorSpec.Constant(String literal) -> Arrays.fill(out, literal); diff --git a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java index 830f3177..7b44bf15 100644 --- a/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java +++ b/demo/fakedata-generator/src/main/java/io/github/dfa1/vortex/demo/fakedata/ProgressBar.java @@ -13,6 +13,7 @@ final class ProgressBar { private final long total; private final long startNanos; private long lastDrawNanos; + private int maxLineLength; ProgressBar(long total) { this.total = total; @@ -45,8 +46,14 @@ void update(long done) { int filled = (int) (fraction * BAR_WIDTH); String bar = "=".repeat(filled) + " ".repeat(BAR_WIDTH - filled); - System.err.printf("\r[%s] %5.1f%% %,d/%,d rows elapsed=%s eta=%s", + String line = "[%s] %5.1f%% %,d/%,d rows elapsed=%s eta=%s".formatted( bar, fraction * 100, done, total, format(elapsed), isFinal ? format(Duration.ZERO) : format(eta)); + // '\r' only returns the cursor to column 0, it doesn't erase anything -- a shorter line + // (e.g. the row count gaining a comma-grouped digit, or the ETA's minute count dropping) + // would otherwise leave trailing characters from the previous, longer redraw stuck on + // screen. Pad to the longest line drawn so far so every redraw fully overwrites the last. + maxLineLength = Math.max(maxLineLength, line.length()); + System.err.print('\r' + line + " ".repeat(maxLineLength - line.length())); if (isFinal) { System.err.println(); } From 1d8b90b284937cdcf5d8a0074575417982e7c637 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 20:05:24 +0200 Subject: [PATCH 10/17] feat(demo): add --no-upload to query an already-uploaded object Every run re-uploaded the file via PUT, even when iterating on --filter-column/--filter-min/--filter-max/--project against a file already sitting on the server from a previous run -- wasteful and slow for exactly the "try different queries" workflow this client is meant to support. --no-upload skips the PUT and queries the existing object directly; requires --server since an embedded server always starts empty. Verified: two runs against the same file, second with --no-upload and a different filter, server log shows exactly one PUT total. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P4ijFsGW1MHEcGiu26vNzi --- .../vortex/demo/client/HttpRangeDemo.java | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java index dc604b3b..2964dc40 100644 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java +++ b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java @@ -72,6 +72,7 @@ private static void run(String[] args) throws IOException, InterruptedException long filterMin = DEFAULT_FILTER_MIN; long filterMax = DEFAULT_FILTER_MAX; String projectColumn = DEFAULT_PROJECT_COLUMN; + boolean upload = true; int i = 0; while (i < args.length) { @@ -100,6 +101,10 @@ private static void run(String[] args) throws IOException, InterruptedException projectColumn = args[++i]; i++; } + case "--no-upload" -> { + upload = false; + i++; + } default -> throw new IllegalArgumentException("unknown argument: " + args[i]); } } @@ -110,23 +115,30 @@ private static void run(String[] args) throws IOException, InterruptedException long fileSize = Files.size(file); if (serverBaseUri != null) { - runAgainst(serverBaseUri, file, fileSize, filterColumn, filterMin, filterMax, projectColumn); + runAgainst(serverBaseUri, file, fileSize, filterColumn, filterMin, filterMax, projectColumn, upload); } else { + if (!upload) { + throw new IllegalArgumentException("--no-upload requires --server (an embedded server starts empty)"); + } try (VortexServer server = VortexServer.start(Files.createTempDirectory("vortex-server"), 0)) { System.out.println("Embedded vortex-server at " + server.baseUri()); - runAgainst(server.baseUri(), file, fileSize, filterColumn, filterMin, filterMax, projectColumn); + runAgainst(server.baseUri(), file, fileSize, filterColumn, filterMin, filterMax, projectColumn, true); } } } private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize, - String filterColumn, long filterMin, long filterMax, String projectColumn) + String filterColumn, long filterMin, long filterMax, String projectColumn, boolean upload) throws IOException, InterruptedException { HttpClient client = HttpClient.newHttpClient(); URI objectUri = serverBaseUri.resolve(localFile.getFileName().toString()); - System.out.println("Uploading to " + objectUri + " ..."); - upload(client, objectUri, localFile); + if (upload) { + System.out.println("Uploading to " + objectUri + " ..."); + upload(client, objectUri, localFile); + } else { + System.out.println("Skipping upload, querying existing object at " + objectUri + " ..."); + } long bytesServedBefore = readBytesServed(client, serverBaseUri); System.out.printf("%nScanning for %d <= %s <= %d, projecting '%s' over HTTP...%n%n", @@ -234,6 +246,11 @@ private static void printUsage() { row [1000000, 1050000) window) --filter-max N inclusive upper bound --project NAME column to project (default: price) + --no-upload skip the PUT, query an object already on the server + (requires --server -- an embedded server starts empty). + Useful for trying several --filter-*/--project + combinations against the same uploaded file without + re-uploading it every time. Generate a file first with vortex-fakedata-generator, e.g.: vortex-fakedata-generator --rows 2000000 --out trades.vortex \\ From 819f72a8de7c69c49db3fc1dc01e2428d8d9c467 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 20:11:39 +0200 Subject: [PATCH 11/17] feat(demo): redesign client CLI around --upload and a positional URL/file argument Replaces --file/--server/--no-upload flags with two clear modes: --upload FILE SERVER_URL copies a file and exits; a query takes a remote object URL directly (no upload, no local file at all -- file size comes from VortexHandle#fileSize on the opened reader) or a local file path (embeds a server, uploads, then queries), dropping the separate --file/--server/--no-upload flag combination entirely. --- demo/README.md | 18 ++- .../vortex/demo/client/HttpRangeDemo.java | 140 +++++++++--------- 2 files changed, 82 insertions(+), 76 deletions(-) diff --git a/demo/README.md b/demo/README.md index 7651272d..463d696f 100644 --- a/demo/README.md +++ b/demo/README.md @@ -54,13 +54,18 @@ terminal — only really visible on larger row counts). 2,000,000 rows, real com a `series(...)`, so it's naturally in row order simply because that's the order it was generated in, exactly like a real append-only ingestion stream. -**Terminal 2 — upload and scan it:** +**Terminal 2 — upload it, then query it:** ```bash -java -jar demo/client/target/vortex-demo.jar \ - --file /tmp/trades.vortex --server http://127.0.0.1:8080/ +java -jar demo/client/target/vortex-demo.jar --upload /tmp/trades.vortex http://127.0.0.1:8080/ +java -jar demo/client/target/vortex-demo.jar http://127.0.0.1:8080/trades.vortex ``` +The first command just copies the file to the server and exits — no query. The second queries the +object directly by URL: no local file, no upload, it's already there. Run the second command again +with different `--filter-column`/`--filter-min`/`--filter-max`/`--project` values to try other +queries against the same uploaded object without re-uploading it. + By default this filters `timestamp` to a narrow window (50,000 of the 2,000,000 rows) and projects `price` — a realistic "give me this time range" query, not an equality match on some other column. While the scan runs, a live "bytes downloaded so far" counter updates in place on @@ -71,8 +76,6 @@ counts, since a small scan can finish before the first redraw. Expected output (numbers will vary slightly with row count): ``` -Uploading to http://127.0.0.1:8080/trades.vortex ... - Scanning for 1701000000000 <= timestamp <= 1701049999000, projecting 'price' over HTTP... Downloaded so far: 401,644 / 24,552,440 bytes (1.6%) @@ -85,10 +88,11 @@ Switch back to **Terminal 1** — you'll see the `PUT` (the upload) followed by ## One-terminal version (no server to manage) -Omit `--server` and the client embeds its own: +Pass a local file path instead of a URL and the client embeds its own server, uploads to it, and +queries it in one shot: ```bash -java -jar demo/client/target/vortex-demo.jar --file /tmp/trades.vortex +java -jar demo/client/target/vortex-demo.jar /tmp/trades.vortex ``` Good for a quick local check; the two-terminal version is more compelling live, since the diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java index 2964dc40..20153a38 100644 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java +++ b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java @@ -17,27 +17,27 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -/// Demo client: uploads an existing Vortex file (e.g. one produced by -/// `vortex-fakedata-generator`) to a `vortex-server` object store, then runs a -/// time-range-filtered, projected [VortexHttpReader] scan against it — printing how many bytes -/// the scan actually pulled over the wire against the object's full size. +/// Demo client: two modes, chosen by the first argument. /// -/// Filters by a numeric column range (`timestamp` by default) rather than an equality match on -/// some other column: real data is written in the order it arrives, so a time-ordered column is -/// naturally clustered by row position even without any artificial sorting -- zone-map pruning -/// narrows a time-range query to the handful of chunks that actually overlap it, no -/// `--sort-by`-style clustering needed. -/// -/// Run standalone (embeds its own [VortexServer]): +/// **Upload only** — copy a local file to a `vortex-server`, no query: /// ``` -/// java -jar client/target/vortex-demo.jar --file trades.vortex +/// java -jar vortex-demo.jar --upload trades.vortex http://127.0.0.1:8080/ /// ``` /// -/// Or against an already-running server — e.g. `java -jar server/target/vortex-server.jar` in a -/// separate terminal, for a two-process live demo: +/// **Query** — a time-range-filtered, projected [VortexHttpReader] scan, printing how many bytes +/// the scan actually pulled over the wire against the object's full size. Given a remote +/// `http(s)://` URL, queries that object directly, no upload. Given a local file path, embeds its +/// own [VortexServer], uploads the file, then queries it there: /// ``` -/// java -jar client/target/vortex-demo.jar --file trades.vortex --server http://127.0.0.1:8080/ +/// java -jar vortex-demo.jar http://127.0.0.1:8080/trades.vortex --filter-column price --filter-min 100 --filter-max 105 +/// java -jar vortex-demo.jar trades.vortex /// ``` +/// +/// Filters by a numeric column range (`timestamp` by default) rather than an equality match on +/// some other column: real data is written in the order it arrives, so a time-ordered column is +/// naturally clustered by row position even without any artificial sorting -- zone-map pruning +/// narrows a time-range query to the handful of chunks that actually overlap it, no +/// `--sort-by`-style clustering needed. public final class HttpRangeDemo { private static final String DEFAULT_FILTER_COLUMN = "timestamp"; @@ -66,25 +66,23 @@ public static void main(String[] args) { } private static void run(String[] args) throws IOException, InterruptedException { - Path file = null; - URI serverBaseUri = null; + if (args.length == 0) { + throw new IllegalArgumentException("a local file or a remote object URL is required"); + } + if (args[0].equals("--upload")) { + runUploadOnly(args); + return; + } + + String target = args[0]; String filterColumn = DEFAULT_FILTER_COLUMN; long filterMin = DEFAULT_FILTER_MIN; long filterMax = DEFAULT_FILTER_MAX; String projectColumn = DEFAULT_PROJECT_COLUMN; - boolean upload = true; - int i = 0; + int i = 1; while (i < args.length) { switch (args[i]) { - case "--file" -> { - file = Path.of(args[++i]); - i++; - } - case "--server" -> { - serverBaseUri = URI.create(args[++i]); - i++; - } case "--filter-column" -> { filterColumn = args[++i]; i++; @@ -101,51 +99,55 @@ private static void run(String[] args) throws IOException, InterruptedException projectColumn = args[++i]; i++; } - case "--no-upload" -> { - upload = false; - i++; - } default -> throw new IllegalArgumentException("unknown argument: " + args[i]); } } - if (file == null) { - throw new IllegalArgumentException("--file is required"); - } - long fileSize = Files.size(file); - - if (serverBaseUri != null) { - runAgainst(serverBaseUri, file, fileSize, filterColumn, filterMin, filterMax, projectColumn, upload); + if (target.startsWith("http://") || target.startsWith("https://")) { + runQuery(HttpClient.newHttpClient(), URI.create(target), filterColumn, filterMin, filterMax, projectColumn); } else { - if (!upload) { - throw new IllegalArgumentException("--no-upload requires --server (an embedded server starts empty)"); - } + Path file = Path.of(target); try (VortexServer server = VortexServer.start(Files.createTempDirectory("vortex-server"), 0)) { System.out.println("Embedded vortex-server at " + server.baseUri()); - runAgainst(server.baseUri(), file, fileSize, filterColumn, filterMin, filterMax, projectColumn, true); + HttpClient client = HttpClient.newHttpClient(); + URI objectUri = server.baseUri().resolve(file.getFileName().toString()); + System.out.println("Uploading to " + objectUri + " ..."); + upload(client, objectUri, file); + runQuery(client, objectUri, filterColumn, filterMin, filterMax, projectColumn); } } } - private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize, - String filterColumn, long filterMin, long filterMax, String projectColumn, boolean upload) - throws IOException, InterruptedException { - HttpClient client = HttpClient.newHttpClient(); - URI objectUri = serverBaseUri.resolve(localFile.getFileName().toString()); + private static void runUploadOnly(String[] args) throws IOException, InterruptedException { + if (args.length < 3) { + throw new IllegalArgumentException("--upload requires FILE and SERVER_URL, e.g. " + + "--upload trades.vortex http://127.0.0.1:8080/"); + } + Path file = Path.of(args[1]); + URI objectUri = URI.create(args[2]).resolve(file.getFileName().toString()); + System.out.println("Uploading to " + objectUri + " ..."); + upload(HttpClient.newHttpClient(), objectUri, file); + System.out.printf("Uploaded %,d bytes.%n", Files.size(file)); + } - if (upload) { - System.out.println("Uploading to " + objectUri + " ..."); - upload(client, objectUri, localFile); - } else { - System.out.println("Skipping upload, querying existing object at " + objectUri + " ..."); + /// Queries `objectUri` directly -- no upload, whether it's an object this run just uploaded + /// itself or one that was already there. Reports bytes fetched against the object's *actual* + /// size, read from the opened [VortexHttpReader] itself ([io.github.dfa1.vortex.reader.VortexHandle#fileSize]), + /// so no local file is needed for this path at all. + private static void runQuery(HttpClient client, URI objectUri, String filterColumn, long filterMin, + long filterMax, String projectColumn) throws IOException, InterruptedException { + URI statsUri = objectUri.resolve("/_stats"); + long fileSize; + try (VortexHttpReader vf = VortexHttpReader.open(objectUri)) { + fileSize = vf.fileSize(); } - long bytesServedBefore = readBytesServed(client, serverBaseUri); + long bytesServedBefore = readBytesServed(client, statsUri); System.out.printf("%nScanning for %d <= %s <= %d, projecting '%s' over HTTP...%n%n", filterMin, filterColumn, filterMax, projectColumn); - long rows = scanWithLiveDownloadCounter(client, serverBaseUri, objectUri, bytesServedBefore, fileSize, + long rows = scanWithLiveDownloadCounter(client, statsUri, objectUri, bytesServedBefore, fileSize, filterColumn, filterMin, filterMax, projectColumn); - long bytesServedAfter = readBytesServed(client, serverBaseUri); + long bytesServedAfter = readBytesServed(client, statsUri); long servedDuringScan = bytesServedAfter - bytesServedBefore; System.out.println("Matched rows: " + rows); @@ -160,14 +162,14 @@ private static void runAgainst(URI serverBaseUri, Path localFile, long fileSize, /// on a large enough dataset that the scan takes more than a poll interval or two; on a small /// file the whole scan finishes before the first redraw and this degrades gracefully to /// printing just the final line. - private static long scanWithLiveDownloadCounter(HttpClient client, URI serverBaseUri, URI objectUri, + private static long scanWithLiveDownloadCounter(HttpClient client, URI statsUri, URI objectUri, long bytesServedBefore, long fileSize, String filterColumn, long filterMin, long filterMax, String projectColumn) throws IOException, InterruptedException { AtomicBoolean scanning = new AtomicBoolean(true); Thread poller = Thread.ofVirtual().name("download-progress").start(() -> { while (scanning.get()) { try { - long servedSoFar = readBytesServed(client, serverBaseUri) - bytesServedBefore; + long servedSoFar = readBytesServed(client, statsUri) - bytesServedBefore; System.err.printf("\r Downloaded so far: %,d / %,d bytes (%.1f%%)", servedSoFar, fileSize, 100.0 * servedSoFar / fileSize); Thread.sleep(POLL_INTERVAL); @@ -206,9 +208,8 @@ private static void upload(HttpClient client, URI objectUri, Path localFile) } } - private static long readBytesServed(HttpClient client, URI serverBaseUri) - throws IOException, InterruptedException { - HttpRequest req = HttpRequest.newBuilder(serverBaseUri.resolve("_stats")).GET().build(); + private static long readBytesServed(HttpClient client, URI statsUri) throws IOException, InterruptedException { + HttpRequest req = HttpRequest.newBuilder(statsUri).GET().build(); HttpResponse response = client.send(req, HttpResponse.BodyHandlers.ofString()); Matcher m = BYTES_SERVED_LINE.matcher(response.body()); if (!m.find()) { @@ -236,21 +237,22 @@ private static long scan(URI objectUri, String filterColumn, long filterMin, lon private static void printUsage() { System.err.println(""" - Usage: vortex-demo --file FILE [options] + Usage: + vortex-demo --upload FILE SERVER_URL upload FILE to a vortex-server, no query + vortex-demo URL [options] query an existing remote object directly + vortex-demo FILE [options] embed a server, upload FILE, then query it - Options: - --file FILE local .vortex file to upload and scan (required) - --server URI base URI of an already-running vortex-server; omit to embed one + Options (query modes only): --filter-column NAME numeric column to range-filter on (default: timestamp) --filter-min N inclusive lower bound (default: matches the README example's row [1000000, 1050000) window) --filter-max N inclusive upper bound --project NAME column to project (default: price) - --no-upload skip the PUT, query an object already on the server - (requires --server -- an embedded server starts empty). - Useful for trying several --filter-*/--project - combinations against the same uploaded file without - re-uploading it every time. + + Examples: + vortex-demo --upload trades.vortex http://127.0.0.1:8080/ + vortex-demo http://127.0.0.1:8080/trades.vortex --filter-column price --filter-min 100 --filter-max 105 + vortex-demo trades.vortex Generate a file first with vortex-fakedata-generator, e.g.: vortex-fakedata-generator --rows 2000000 --out trades.vortex \\ From 7a98465f615e8b4430f34b097665fb3ec82aac3f Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 20:16:03 +0200 Subject: [PATCH 12/17] fix(demo): don't print usage for runtime errors, only bad invocations A 404/connection failure isn't a syntax problem -- printing the usage block after it (as if the command line were wrong) was misleading. Only IllegalArgumentException (missing/unknown argument) shows usage now; other failures print just the error message. --- .../io/github/dfa1/vortex/demo/client/HttpRangeDemo.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java index 20153a38..68393434 100644 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java +++ b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java @@ -57,11 +57,17 @@ private HttpRangeDemo() { public static void main(String[] args) { try { run(args); - } catch (RuntimeException | IOException | InterruptedException e) { + } catch (IllegalArgumentException e) { + // A bad invocation (missing/unknown argument) -- usage helps here. System.err.println("error: " + e.getMessage()); System.err.println(); printUsage(); System.exit(1); + } catch (RuntimeException | IOException | InterruptedException e) { + // A well-formed invocation that failed at runtime (network error, malformed file, + // ...) -- usage wouldn't explain anything here, just the error itself. + System.err.println("error: " + e.getMessage()); + System.exit(1); } } From a762fed1ec5c01faa5d4bed149c5c6f81365a3b7 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 20:18:49 +0200 Subject: [PATCH 13/17] fix(demo): normalize dataDir at server startup, fixing 404s on relative paths "vortex-server 8080 ." (a relative dataDir) served a correct directory listing but 404'd every single-object GET/HEAD: #resolve normalized the resolved candidate before its startsWith containment check, but never normalized dataDir itself, so a bare "." never structurally startsWith-matched a candidate that normalize() had already stripped the "." from. Normalizing dataDir to an absolute path once at start() fixes it for every relative form, not just ".". --- .../dfa1/vortex/demo/server/VortexServer.java | 8 ++++++- .../vortex/demo/server/VortexServerTest.java | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/demo/server/src/main/java/io/github/dfa1/vortex/demo/server/VortexServer.java b/demo/server/src/main/java/io/github/dfa1/vortex/demo/server/VortexServer.java index 70c171a6..e515e315 100644 --- a/demo/server/src/main/java/io/github/dfa1/vortex/demo/server/VortexServer.java +++ b/demo/server/src/main/java/io/github/dfa1/vortex/demo/server/VortexServer.java @@ -53,8 +53,14 @@ private VortexServer(HttpServer server, Path dataDir) { /// @throws IOException if `dataDir` cannot be created or the server socket cannot be bound public static VortexServer start(Path dataDir, int port) throws IOException { Files.createDirectories(dataDir); + // Absolute + normalized once here so every #resolve call's startsWith containment check + // compares like with like -- a relative dataDir (e.g. ".") left as-is would never + // startsWith-match a resolved candidate that Path#normalize stripped the "." from, + // 404-ing every GET/HEAD even though the file is right there (confirmed live: "vortex-server + // 8080 ." served a correct directory listing but 404'd every single-object request). + Path root = dataDir.toAbsolutePath().normalize(); HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", port), 0); - VortexServer instance = new VortexServer(server, dataDir); + VortexServer instance = new VortexServer(server, root); server.createContext("/", instance::handle); server.start(); return instance; diff --git a/demo/server/src/test/java/io/github/dfa1/vortex/demo/server/VortexServerTest.java b/demo/server/src/test/java/io/github/dfa1/vortex/demo/server/VortexServerTest.java index 09b83d32..5f600bd8 100644 --- a/demo/server/src/test/java/io/github/dfa1/vortex/demo/server/VortexServerTest.java +++ b/demo/server/src/test/java/io/github/dfa1/vortex/demo/server/VortexServerTest.java @@ -184,6 +184,28 @@ void listsStoredObjects() throws Exception { assertThat(result.body()).contains("data.bin\t" + CONTENT.length()); } + @Test + void servesObjectsWhenDataDirIsNotNormalized(@TempDir Path freshDir) throws Exception { + // Given a server started with an unnormalized dataDir -- a trailing "/." here reproduces + // the same shape as the real "vortex-server PORT ." invocation, whose relative "." + // previously survived un-normalized into #resolve's startsWith containment check. A + // resolved, Path#normalize-d candidate never has that trailing "." component, so it + // structurally failed startsWith against the un-normalized root and every single-object + // request 404'd -- even though #handleList (which just streams dataDir's own entries, + // no startsWith check) correctly showed the object was right there. + Files.writeString(freshDir.resolve("obj.bin"), "hello", StandardCharsets.US_ASCII); + Path unnormalized = freshDir.resolve("."); + + try (VortexServer freshServer = VortexServer.start(unnormalized, 0)) { + // When + HttpResponse result = get(freshServer.baseUri().resolve("obj.bin"), null); + + // Then + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).isEqualTo("hello"); + } + } + private HttpResponse get(URI uri, String rangeHeader) throws Exception { HttpRequest.Builder builder = HttpRequest.newBuilder(uri).GET(); if (rangeHeader != null) { From 879e240992733d0d04b6c1328a9ac6149e02529e Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 20:22:59 +0200 Subject: [PATCH 14/17] fix(demo): count rows Compute filters actually select, not raw chunk sizes ScanOptions#withFilter only prunes whole chunks via zone-map stats -- it never filters individual rows within a surviving chunk. Summing chunk.rowCount() therefore reported every row in every non-pruned chunk as "matched", regardless of whether it actually satisfied the filter. A price:range(50,150) filter of [500,501] -- outside the column's entire range -- was reporting all 200,000,000 rows as matched, live on the user's own machine. Compute#filteredAggregate(chunk, filter, null) runs the same filter row-by-row (fused single pass, COUNT(*)-style with no aggregate column) to get the true matched-row count. Verified against the user's real 2.4GB/200M-row file: a [100,102] filter over a uniform [50,150] price column now reports 3,996,842 matched rows, ~2% of 200M as expected, instead of 200,000,000. --- .../io/github/dfa1/vortex/demo/client/HttpRangeDemo.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java index 68393434..3767c5db 100644 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java +++ b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java @@ -4,6 +4,7 @@ import io.github.dfa1.vortex.reader.RowFilter; import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexHttpReader; +import io.github.dfa1.vortex.reader.compute.Compute; import java.io.IOException; import java.net.URI; @@ -233,8 +234,12 @@ private static long scan(URI objectUri, String filterColumn, long filterMin, lon try (VortexHttpReader vf = VortexHttpReader.open(objectUri); var iter = vf.scan(opts)) { while (iter.hasNext()) { + // ScanOptions#withFilter only prunes whole chunks via zone-map stats -- a surviving + // chunk can still contain rows outside the range, so chunk.rowCount() alone would + // overcount. Compute#filteredAggregate applies the same filter row-by-row (fused, + // single pass, no aggregate column needed) to get the true matched-row count. try (var chunk = iter.next()) { - rows += chunk.rowCount(); + rows += Compute.filteredAggregate(chunk, filter, null).selectedRows(); } } } From db064a32ff18e4070d0bdbf8cd960202388afd8d Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 20:33:26 +0200 Subject: [PATCH 15/17] docs(demo): note #382 (ALP-RD drops zone-map stats) in the README price never prunes regardless of filter overlap, unlike the "no natural clustering" story that explains symbol/enum columns -- worth distinguishing since it looks the same in the demo's own output (same ~54% either way) but has a different, currently-open root cause. --- demo/README.md | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/demo/README.md b/demo/README.md index 463d696f..cc8bd3e0 100644 --- a/demo/README.md +++ b/demo/README.md @@ -115,11 +115,18 @@ audience can watch the server's request log update in real time. range query; an `enum(...)` column's values are scattered uniformly across every chunk by design, so an equality filter on it (e.g. `symbol == "SYM015"`) has nothing to prune — every chunk could contain a match. +- **Why not filter `price`** — a `range()`-generated `f64` column pruning as if it were unclustered + is expected (see above), but `price` currently prunes *no better than that even for a filter + entirely outside its data range* — e.g. `[5, 6]` when every value is in `[50, 150)`. That's + [#382](https://github.com/dfa1/vortex-java/issues/382), still open: the cascade routes `f64` + through `vortex.alprd` (ALP-RD), which never computes zone-map `min`/`max` at all, so there is + no stat to prune with regardless of overlap. `timestamp`/`volume` (plain integer encodings) and + `symbol` (dict-encoded) are unaffected. -## Bugs this demo surfaced (now fixed upstream) +## Bugs this demo surfaced -Building this surfaced three real gaps in zone-map pruning, filed and since fixed in vortex-java's -`reader`/`writer` modules: +Building this surfaced four real gaps in zone-map pruning in vortex-java's `reader`/`writer` +modules. Three are fixed: - [#378](https://github.com/dfa1/vortex-java/issues/378) — `WriteOptions`'s default `globalDict=true` silently defeated zone-map pruning for `Utf8` columns. @@ -129,5 +136,12 @@ Building this surfaced three real gaps in zone-map pruning, filed and since fixe could be pruned fetched that chunk's *entire* segment first, costing as much bandwidth as just reading it. -This demo's numbers reflect the fixed behavior. If you're running against an older vortex-java -build, pruning may not work and the byte percentage will be much higher than shown above. +One is still open: + +- [#382](https://github.com/dfa1/vortex-java/issues/382) — `AlpRdEncodingEncoder` never emits + zone-map min/max stats at all, so any column the cascade routes through ALP-RD (typically `f64`) + never prunes, independent of the filter (see "Why not filter `price`" above). + +This demo's numbers reflect the fixed behavior for #378-#380. If you're running against an older +vortex-java build, pruning may not work at all and the byte percentage will be much higher than +shown above. From 7e864b8031f18698c833e5c89fd5eb80a6d8c20c Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 20:57:28 +0200 Subject: [PATCH 16/17] fix(demo): collapse --filter-column/--filter-min/--filter-max into --range One compact COLUMN:MIN:MAX argument instead of three separate flags to remember for a live demo; --project stays separate since it's optional. Fixes #383. --- demo/README.md | 10 +++--- .../vortex/demo/client/HttpRangeDemo.java | 31 +++++++++---------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/demo/README.md b/demo/README.md index cc8bd3e0..9456f539 100644 --- a/demo/README.md +++ b/demo/README.md @@ -63,8 +63,8 @@ java -jar demo/client/target/vortex-demo.jar http://127.0.0.1:8080/trades.vortex The first command just copies the file to the server and exits — no query. The second queries the object directly by URL: no local file, no upload, it's already there. Run the second command again -with different `--filter-column`/`--filter-min`/`--filter-max`/`--project` values to try other -queries against the same uploaded object without re-uploading it. +with different `--range`/`--project` values to try other queries against the same uploaded object +without re-uploading it. By default this filters `timestamp` to a narrow window (50,000 of the 2,000,000 rows) and projects `price` — a realistic "give me this time range" query, not an equality match on some @@ -100,9 +100,9 @@ audience can watch the server's request log update in real time. ## Customizing the story -- **Different filter/projection** — `vortex-demo` accepts `--filter-column`, `--filter-min`, - `--filter-max`, and `--project` (numeric-range filter, not equality — see below for why). Match - these to whatever schema you generate. +- **Different filter/projection** — `vortex-demo` accepts `--range COLUMN:MIN:MAX` and `--project` + (numeric-range filter, not equality — see below for why). Match these to whatever schema you + generate, e.g. `--range price:100:105 --project timestamp`. - **Different dataset shape** — `vortex-fakedata-generator`'s column grammar is `name:type:generator(args)`. Run it with no arguments for the full grammar reference (types, generators, an example). The `series(start,step)` generator is a direct nod to SQL's diff --git a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java index 3767c5db..40e63f36 100644 --- a/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java +++ b/demo/client/src/main/java/io/github/dfa1/vortex/demo/client/HttpRangeDemo.java @@ -30,7 +30,7 @@ /// `http(s)://` URL, queries that object directly, no upload. Given a local file path, embeds its /// own [VortexServer], uploads the file, then queries it there: /// ``` -/// java -jar vortex-demo.jar http://127.0.0.1:8080/trades.vortex --filter-column price --filter-min 100 --filter-max 105 +/// java -jar vortex-demo.jar http://127.0.0.1:8080/trades.vortex --range price:100:105 /// java -jar vortex-demo.jar trades.vortex /// ``` /// @@ -90,16 +90,15 @@ private static void run(String[] args) throws IOException, InterruptedException int i = 1; while (i < args.length) { switch (args[i]) { - case "--filter-column" -> { - filterColumn = args[++i]; - i++; - } - case "--filter-min" -> { - filterMin = Long.parseLong(args[++i]); - i++; - } - case "--filter-max" -> { - filterMax = Long.parseLong(args[++i]); + case "--range" -> { + String[] parts = args[++i].split(":", 3); + if (parts.length != 3) { + throw new IllegalArgumentException( + "--range requires COLUMN:MIN:MAX, e.g. --range price:100:105"); + } + filterColumn = parts[0]; + filterMin = Long.parseLong(parts[1]); + filterMax = Long.parseLong(parts[2]); i++; } case "--project" -> { @@ -254,15 +253,13 @@ private static void printUsage() { vortex-demo FILE [options] embed a server, upload FILE, then query it Options (query modes only): - --filter-column NAME numeric column to range-filter on (default: timestamp) - --filter-min N inclusive lower bound (default: matches the README example's - row [1000000, 1050000) window) - --filter-max N inclusive upper bound - --project NAME column to project (default: price) + --range COLUMN:MIN:MAX inclusive numeric range filter (default: matches the + README example's timestamp row [1000000, 1050000) window) + --project NAME column to project (default: price) Examples: vortex-demo --upload trades.vortex http://127.0.0.1:8080/ - vortex-demo http://127.0.0.1:8080/trades.vortex --filter-column price --filter-min 100 --filter-max 105 + vortex-demo http://127.0.0.1:8080/trades.vortex --range price:100:105 vortex-demo trades.vortex Generate a file first with vortex-fakedata-generator, e.g.: From a141f7a8655f583832588879768ba2fb93cfe605 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 12 Sep 2026 21:00:53 +0200 Subject: [PATCH 17/17] docs(demo): update README now that #382 (ALP-RD zone-map stats) is fixed Rebasing onto main pulled in the #382 fix. Verified live: a price filter entirely outside the data's range now prunes to <1% of the file instead of the ~54% it fetched when ALP-RD reported no min/max at all. A filter that overlaps price's range still touches most chunks, but that's the column's lack of natural clustering, not the stats bug. --- demo/README.md | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/demo/README.md b/demo/README.md index 9456f539..ac903371 100644 --- a/demo/README.md +++ b/demo/README.md @@ -115,18 +115,18 @@ audience can watch the server's request log update in real time. range query; an `enum(...)` column's values are scattered uniformly across every chunk by design, so an equality filter on it (e.g. `symbol == "SYM015"`) has nothing to prune — every chunk could contain a match. -- **Why not filter `price`** — a `range()`-generated `f64` column pruning as if it were unclustered - is expected (see above), but `price` currently prunes *no better than that even for a filter - entirely outside its data range* — e.g. `[5, 6]` when every value is in `[50, 150)`. That's - [#382](https://github.com/dfa1/vortex-java/issues/382), still open: the cascade routes `f64` - through `vortex.alprd` (ALP-RD), which never computes zone-map `min`/`max` at all, so there is - no stat to prune with regardless of overlap. `timestamp`/`volume` (plain integer encodings) and - `symbol` (dict-encoded) are unaffected. +- **Why filtering `price` outside its range now prunes fully, but overlapping it still doesn't** — + a `range()`-generated `f64` column has no natural clustering by row position (unlike `series(...)` + `timestamp`), so any filter that *overlaps* its data range still touches most chunks — that part + is expected, not a bug. A filter *entirely outside* the range (e.g. `[5, 6]` when every value is + in `[50, 150)`) is different: with #382 fixed, ALP-RD now computes zone-map `min`/`max`, so that + case prunes down to a couple of stray range fetches (well under 1% of the file) instead of the + ~54% it used to fetch when ALP-RD reported no stats at all. ## Bugs this demo surfaced Building this surfaced four real gaps in zone-map pruning in vortex-java's `reader`/`writer` -modules. Three are fixed: +modules — all four are now fixed: - [#378](https://github.com/dfa1/vortex-java/issues/378) — `WriteOptions`'s default `globalDict=true` silently defeated zone-map pruning for `Utf8` columns. @@ -135,13 +135,10 @@ modules. Three are fixed: - [#380](https://github.com/dfa1/vortex-java/issues/380) — checking whether an HTTP-backed chunk could be pruned fetched that chunk's *entire* segment first, costing as much bandwidth as just reading it. +- [#382](https://github.com/dfa1/vortex-java/issues/382) — `AlpRdEncodingEncoder` never emitted + zone-map min/max stats at all, so any column the cascade routed through ALP-RD (typically `f64`) + never pruned, independent of the filter (see above). -One is still open: - -- [#382](https://github.com/dfa1/vortex-java/issues/382) — `AlpRdEncodingEncoder` never emits - zone-map min/max stats at all, so any column the cascade routes through ALP-RD (typically `f64`) - never prunes, independent of the filter (see "Why not filter `price`" above). - -This demo's numbers reflect the fixed behavior for #378-#380. If you're running against an older -vortex-java build, pruning may not work at all and the byte percentage will be much higher than -shown above. +This demo's numbers reflect the fixed behavior for #378-#380 and #382. If you're running against +an older vortex-java build, pruning may not work at all and the byte percentage will be much higher +than shown above.