From 60eed0b85edd9cbc8cf05c18aed354325a83a0a7 Mon Sep 17 00:00:00 2001 From: czyt Date: Fri, 16 Oct 2026 14:49:43 +0800 Subject: [PATCH 1/3] Define Data query pre scripts and saved editor contract --- crates/core/src/data.rs | 17 +++++ crates/formats/src/redact.rs | 2 + crates/formats/tests/interchange.rs | 2 +- docs/specs/data-query-pre-scripts.md | 24 +++++++ .../data-query-pre-scripts-plan.md | 68 +++++++++++++++++++ web/src/features/data/DataWorkbench.test.tsx | 9 +++ web/src/features/data/DataWorkbench.tsx | 4 +- web/src/features/data/model.ts | 2 +- web/src/features/data/types.ts | 2 +- web/src/shared/i18n/en.json | 3 + web/src/shared/i18n/zh-CN.json | 3 + 11 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 docs/specs/data-query-pre-scripts.md create mode 100644 tasks/implementation/data-query-pre-scripts-plan.md diff --git a/crates/core/src/data.rs b/crates/core/src/data.rs index 419f8c8c..10467af3 100644 --- a/crates/core/src/data.rs +++ b/crates/core/src/data.rs @@ -29,6 +29,7 @@ pub enum DataFileFormat { pub struct DataConfig { pub source: DataSource, pub sql: String, + pub query_pre_script: String, pub file_name: String, pub file_format: DataFileFormat, pub sheet_name: String, @@ -44,6 +45,7 @@ impl Default for DataConfig { Self { source: DataSource::Postgresql, sql: "SELECT 1 AS value".into(), + query_pre_script: String::new(), file_name: String::new(), file_format: DataFileFormat::Csv, sheet_name: String::new(), @@ -57,6 +59,7 @@ impl Default for DataConfig { } } pub fn validate_data_config(config: &DataConfig) -> Result<()> { + crate::validate_script(&config.query_pre_script)?; ensure!( config.sql.len() <= MAX_SQL_SOURCE && !config.sql.contains('\0'), "Data SQL source exceeds limits" @@ -221,6 +224,20 @@ pub fn data_url(raw: &str, source: DataSource) -> Result { mod tests { use super::*; #[test] + fn query_pre_script_defaults_and_obeys_shared_script_limit() { + let legacy: DataConfig = + serde_json::from_value(serde_json::json!({"sql":"SELECT 1"})).unwrap(); + assert!(legacy.query_pre_script.is_empty()); + let mut configured = legacy; + configured.query_pre_script = "x".repeat(crate::MAX_SCRIPT_BYTES); + validate_data_config(&configured).unwrap(); + let copied: DataConfig = + serde_json::from_value(serde_json::to_value(&configured).unwrap()).unwrap(); + assert_eq!(copied.query_pre_script, configured.query_pre_script); + configured.query_pre_script.push('x'); + assert!(validate_data_config(&configured).is_err()); + } + #[test] fn retains_selected_file_and_sql_but_rejects_hidden_network_options() { let config = DataConfig { source: DataSource::LocalFile, diff --git a/crates/formats/src/redact.rs b/crates/formats/src/redact.rs index d616c6d3..1026b1e3 100644 --- a/crates/formats/src/redact.rs +++ b/crates/formats/src/redact.rs @@ -522,6 +522,8 @@ pub(super) fn workspace(source: &Workspace) -> Workspace { request.url = data_privacy.screen_text(&redact_url(&request.url)); config.file_base64.clear(); config.sql = data_privacy.screen_generation_text(&config.sql); + config.query_pre_script = data_privacy + .screen_bounded(&config.query_pre_script, moleapi_core::MAX_SCRIPT_BYTES); config.file_name = data_privacy.screen_text(&config.file_name); config.table_name = data_privacy.screen_text(&config.table_name); } diff --git a/crates/formats/tests/interchange.rs b/crates/formats/tests/interchange.rs index 069ce2c0..41c4a5ab 100644 --- a/crates/formats/tests/interchange.rs +++ b/crates/formats/tests/interchange.rs @@ -761,7 +761,7 @@ fn data_request_exports_screen_credentials_copied_sql_and_keep_explicit_original sql: "SELECT 'it''s-private' AS copied".into(), ..Default::default() }; - let request:moleapi_core::RequestSpec=serde_json::from_value(json!({"id":"data","name":"Data request","protocol":{"kind":"data","source":"postgresql","sql":config.sql},"method":"GET","url":"postgresql://user:it%27s-private@db.example/main","description":"","query":[],"headers":[],"body_kind":"none","body":"","auth":{"kind":"none","token":"","username":"","password":""},"timeout_ms":1000,"verify_tls":true,"follow_redirects":false,"assertions":[],"examples":[]})).unwrap(); + let request:moleapi_core::RequestSpec=serde_json::from_value(json!({"id":"data","name":"Data request","protocol":{"kind":"data","source":"postgresql","sql":config.sql,"query_pre_script":"pm.request.body.update(\"SELECT 'it''s-private'\")"},"method":"GET","url":"postgresql://user:it%27s-private@db.example/main","description":"","query":[],"headers":[],"body_kind":"none","body":"","auth":{"kind":"none","token":"","username":"","password":""},"timeout_ms":1000,"verify_tls":true,"follow_redirects":false,"assertions":[],"examples":[]})).unwrap(); let mut w=workspace(serde_json::from_value(json!({"schema_version":1,"collections":[{"id":"c","name":"Data","description":"","requests":[request]}],"environments":[],"active_environment_id":null})).unwrap()); let format = "moleapi"; { diff --git a/docs/specs/data-query-pre-scripts.md b/docs/specs/data-query-pre-scripts.md new file mode 100644 index 00000000..cb4e28b3 --- /dev/null +++ b/docs/specs/data-query-pre-scripts.md @@ -0,0 +1,24 @@ +# Data per-query preparation + +Authority: `data-client.md`, `data-query-completion.md`, remaining-work milestone 3, and the shared `moleapi-pm/1` script and variable contracts. + +## Objective + +The Data workbench currently runs the inherited pre-request script once when it opens a connection. Subsequent SQL submissions use the editor text directly. Add a separate saved **query pre script** so each explicit Data query can prepare scoped variables, edit its SQL and report its own tests/logs before the SQL operation starts. Connection preparation retains its current behavior; an existing saved Data request with no query script behaves exactly as before. + +## User and API contract + +- `DataConfig.query_pre_script` is an optional string, defaulting to empty, limited to the same 256 KiB script budget as other request scripts. Existing imports and saved requests without the field continue to load. The Data editor shows it in a dedicated tab while disconnected and connected; edits are saved with the request but a running session uses the connection-time snapshot until reconnect. +- Each accepted `data_query` carries a query ID, selected SQL, and read-only flag. The server presents the selected SQL to the isolated script worker as `pm.request.body.raw` in text mode. The script can set variables through existing scope APIs and replace SQL with `pm.request.body.update(...)`. It can log and test. `pm.response` is absent. The output SQL is resolved against the updated scoped values and then checked against the existing 64 KiB, single-statement, dialect, read-only/write and execution budgets. The script cannot change connection URL/auth, headers, method, protocol/source, file selection or read-only flag. Reject an attempted unsupported mutation before dispatching SQL. +- Only one query preparation or execution is active per Data session. The next pre script waits for the previous query's response postprocessing to finish so it sees the prior script/extraction variable updates. An explicit cancel during preparation stops the preparation and emits a query-scoped cancelled/preparation error outcome without dispatching SQL. Failed/oversized/invalid pre script or SQL resolution emits `data_preprocess_error` with query ID and masked details, and must not run or retry SQL. `pm.skipRequest()` means no SQL and an explicit skipped result. No history record is created for a query that never reached the Data worker. +- Successful preparation emits a `data_prepared` event with the query ID, safe SQL preview or change indicator, tests/logs and non-private variable update metadata. The Data workbench displays preparation status before live rows and keeps it attached to the correct query; it does not silently edit the saved SQL draft. The actual resolved SQL source is passed to the protocol worker and retained by the existing completion/history path. + +## Ownership, privacy and limits + +- Recheck session, account, workspace/request identity and owner admission generation before running the script and before dispatch. Logout, session deletion, workspace deletion or a changed owner generation prevents SQL dispatch and later variable updates. The script worker has its existing 250 ms deadline, 64 MiB heap, disabled network/filesystem and 8 MiB output cap. The prepared SQL is revalidated by the protocol runtime; neither the script nor the API bypasses its SQL AST policy. +- Script output and pre error details are masked using the captured private values. New private values from the script are added to the session scope before later postprocessing and history screening. A script failure with incomplete privacy metadata withholds diagnostic text. The bounded Data session event budget remains authoritative. +- Busy or full Data sessions reject before script side effects. Each accepted query ID produces at most one SQL dispatch and one terminal preparation result. SQL writes are never retried or called rolled back on cancellation. + +## Evidence + +Use actual local file and hosted PostgreSQL/MySQL workers to prove variable substitution, SQL mutation, one pre run per query, ordering with post updates, skip/failure/cancel with zero SQL dispatches, read-only policy, privacy masking, owner/logout/delete fencing, script budget, and history SQL source. Verify the bilingual editor and preparation status at 390px. Run Rust and frontend suites, strict Clippy, formatting, production/embedded build, then branch CI and platform gates. Saved datasets, extra Data sources and EDAS remain separate capabilities. diff --git a/tasks/implementation/data-query-pre-scripts-plan.md b/tasks/implementation/data-query-pre-scripts-plan.md new file mode 100644 index 00000000..ab167718 --- /dev/null +++ b/tasks/implementation/data-query-pre-scripts-plan.md @@ -0,0 +1,68 @@ +# Data per-query pre scripts implementation plan + +> **For agentic workers:** Execute each numbered task in order with a failing integration test before its implementation. Use `executing-plans` for inline work. Keep the one final independent branch review. + +**Goal:** Run a saved isolated script before every accepted Data SQL query, with safe SQL mutation and scoped variables. + +**Architecture:** Keep connection-time scripts as they are. Add a bounded query-pre script field to `DataConfig`, and a server-owned per-session preparation context shared by the send handler and Data completion consumer. Split the protocol's Data query admission from queue dispatch, so a script runs only after one query ID has reserved the active slot; cancellation can abort preparation without SQL. The output SQL is resolved and checked again by the existing Data worker. + +**Tech Stack:** Rust/Tokio/Axum, `moleapi-pm/1` QuickJS worker, React/TypeScript, Vitest, local DataFusion file worker and hosted PostgreSQL/MySQL. + +**Spec:** `docs/specs/data-query-pre-scripts.md` + +## Global constraints + +- Preserve the old connection-time pre script and empty-field request behavior. +- Keep SQL at 64 KiB, scripts at 256 KiB and the 250 ms isolated worker deadline. +- Never dispatch SQL before owner/session/admission checks or after a failed script. Never retry writes. +- Reuse Data AST/read-only checks and shared variable/privacy helpers. + +## Tasks + +### Task 1: Saved contract and editor + +**Files:** `crates/core/src/data.rs`, `web/src/features/data/types.ts`, `web/src/features/data/DataWorkbench.tsx`, bilingual dictionaries, core and web Data tests. + +- [ ] Add a core serialization/validation test: old Data JSON defaults to an empty script, and a script over 256 KiB is rejected before persistence. Run it red. +- [ ] Add `query_pre_script` to `DataConfig` and the frontend type with the empty default and cap; run the test green. +- [ ] Add a Data workbench script tab with a labelled editor that stays editable while connected but documents reconnect-to-apply. Add a saved/reloaded editor test and inspect 390px Chinese/English. +- [ ] Run `cargo test -p moleapi-core data` and the focused Vitest files; commit the contract and editor. + +### Task 2: Protocol reservation + +**Files:** `crates/protocols/src/data.rs`, `crates/protocols/src/lib.rs`, `crates/protocols/src/models.rs`, `crates/server/tests/data_client.rs`. + +- [ ] Add a failing real-worker test where two sends race, one preparation is cancelled, and a rejected query never emits `data_started` or SQL rows. +- [ ] Expose owner-scoped `reserve_data_query` and `dispatch_reserved_data_query` around the current one-active-query, 512-command and 20 MiB limits. Cancellation and failure release a reservation exactly once. +- [ ] Keep the worker's SQL validation after dispatch and its existing terminal completion behavior. Run the new test and all Data client tests; commit. + +### Task 3: Server preparation and privacy + +**Files:** `crates/server/src/protocols.rs`, `crates/server/src/execution.rs`, `crates/server/src/lib.rs`, Data client and database integration tests. + +- [ ] Add a failing local CSV test: per-query script changes SQL from the selected text, runs once on each query, and query N+1 reads N's post-script variable update. +- [ ] Share session scopes with an async lock between the send handler and completion consumer. Capture request/collection/workspace and owner generation at connect, then run the isolated script on a synthetic text-body request. +- [ ] Reject changes to URL/auth, method, headers, protocol/source, file selection and read-only mode; accept only SQL/body and scoped variable updates. Validate/resolve the output SQL before dispatch. Return query-scoped prepared, skipped or masked error events without history for unexecuted SQL. +- [ ] Add failing tests for script deadline/privacy failure, invalid SQL/mutation, owner logout/workspace deletion and cancellation during preparation. Make them pass, then run the Data and database suites; commit. + +### Task 4: Workbench result flow + +**Files:** `web/src/features/data/useDataEvents.ts`, `web/src/features/data/DataWorkbench.tsx`, `web/src/features/data/types.ts`, focused Vitest files and dictionaries. + +- [ ] Add a failing hook test for stale preparation events and cancel/error releasing the run button. +- [ ] Display query preparation status/tests/logs before rows, retain them with the matching result and never rewrite the saved SQL draft. +- [ ] Run focused Vitest and browser QA at 390px in both languages against a real CSV fixture; commit. + +### Task 5: Gates and review + +- [ ] Run `cargo fmt -- --check`, `cargo test --locked`, `cargo clippy --locked --all-targets -- -D warnings`, `npm --prefix web test`, and `npm --prefix web run build`. +- [ ] Verify hosted PostgreSQL/MySQL SQL mutation and read-only enforcement in Application CI, then track the current platform build and perform one independent whole-branch review. +- [ ] Update `docs/FEATURE-MATRIX.md`, `docs/IMPLEMENTATION-STATUS.md` and the remaining-work plan with exact supported combinations and still-open saved dataset/source/EDAS gates. + +## Review focus + +- A script changes the Data source, URL/auth, headers or read-only flag: fail before SQL and mask private diagnostics. +- Two simultaneous sends or a cancel while the worker is starting: at most one SQL dispatch, no stranded active query. +- Postprocessing of query N overlaps preparation of N+1: N+1 sees N's scoped updates without leaking them into saved drafts. +- A script worker times out or loses privacy metadata: no SQL dispatch and no unmasked output. +- Logout, workspace deletion or session removal during preparation: no late dispatch, history or cross-owner event. diff --git a/web/src/features/data/DataWorkbench.test.tsx b/web/src/features/data/DataWorkbench.test.tsx index c2ec8a0f..e1671b63 100644 --- a/web/src/features/data/DataWorkbench.test.tsx +++ b/web/src/features/data/DataWorkbench.test.tsx @@ -8,7 +8,16 @@ vi.mock("../workbench/context",()=>({useWorkbench:vi.fn()})); vi.mock("./DataSourceFields",()=>({DataSourceFields:()=>null})); vi.mock("./DataResults",()=>({DataResults:()=>null})); vi.mock("./DataSqlEditor",()=>({DataSqlEditor:()=>null})); +vi.mock("../../shared/ui",()=>({Editor:({label,value,onChange}:{label:string;value:string;onChange:(value:string)=>void})=>