From daf744abe971f2c7d169d14a578fe7ef94abed55 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 18:19:16 +0000 Subject: [PATCH 1/3] goldeneye: generate the SQLite dialect from the official sqlite3 shell Add a sqlite package to goldeneye that reads pragma_function_list from the sqlite3 shell sqlite.org publishes, run against an in-memory database, and writes internal/engine/sqlite/dialect/functions.jsonl from it. `install sqlite` downloads the pinned release (3.53.4, the one the ncruces/go-sqlite3 driver embeds) into the user cache directory and checks it against the SHA3-256 the download page lists, the same way the clickhouse installer works; SQLITE3 names a shell to use instead. SQLite records a function's name, kind and argument count and nothing about types, so the return and argument types come from a table in sqlite/signatures.go. A built-in function the shell reports that the table lacks fails the run, as does a table entry the shell does not report. Aggregates are told apart from window functions by calling each without OVER, since SQLite marks every aggregate 'w'. SQLite has no catalog of types or operators, so those files stay hand-written. Regenerating the committed file adds the JSON, date and time, window and percentile functions the hand-written list lacked, corrects the misspelled random and randomblob entries, marks group_concat nullable as it is over no rows, and drops soundex, which the official build does not compile in. The goldens that move follow from those. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RYkxvtwH6GgysuWYP87vyj --- .github/workflows/gen.yml | 2 + CLAUDE.md | 1 + .../builtins/sqlite/go/aggfunc.sql.go | 16 +- .../builtins/sqlite/go/scalarfunc.sql.go | 12 +- .../table_function/sqlite/go/query.sql.go | 2 +- .../engine/sqlite/dialect/functions.jsonl | 258 +++++++++++------- internal/engine/sqlite/stdlib.go | 9 +- internal/goldeneye/README.md | 20 +- internal/goldeneye/cmd/goldeneye/main.go | 35 ++- internal/goldeneye/sqlite/install.go | 206 ++++++++++++++ internal/goldeneye/sqlite/signatures.go | 243 +++++++++++++++++ internal/goldeneye/sqlite/sqlite.go | 221 +++++++++++++++ internal/goldeneye/sqlite/sqlite_test.go | 37 +++ 13 files changed, 938 insertions(+), 124 deletions(-) create mode 100644 internal/goldeneye/sqlite/install.go create mode 100644 internal/goldeneye/sqlite/signatures.go create mode 100644 internal/goldeneye/sqlite/sqlite.go create mode 100644 internal/goldeneye/sqlite/sqlite_test.go diff --git a/.github/workflows/gen.yml b/.github/workflows/gen.yml index c3b6bdd4ad..691c06d4da 100644 --- a/.github/workflows/gen.yml +++ b/.github/workflows/gen.yml @@ -24,6 +24,8 @@ jobs: check-latest: true - run: go run ./cmd/goldeneye install clickhouse working-directory: internal/goldeneye + - run: go run ./cmd/goldeneye install sqlite + working-directory: internal/goldeneye - run: go run ./cmd/goldeneye generate working-directory: internal/goldeneye env: diff --git a/CLAUDE.md b/CLAUDE.md index dfce6cfb08..0797a3183d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -153,6 +153,7 @@ is not available skip. ```bash cd internal/goldeneye go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary once +go run ./cmd/goldeneye install sqlite # download the pinned sqlite3 shell once POSTGRESQL_SERVER_URI="postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable" go test ./... go run ./cmd/goldeneye generate postgresql # rewrite the files after a change ``` diff --git a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go index 29f345be04..de2a1fa89c 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go @@ -47,9 +47,9 @@ const getGroupConcatInt = `-- name: GetGroupConcatInt :one SELECT group_concat(int_val) FROM test ` -func (q *Queries) GetGroupConcatInt(ctx context.Context) (string, error) { +func (q *Queries) GetGroupConcatInt(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getGroupConcatInt) - var group_concat string + var group_concat sql.NullString err := row.Scan(&group_concat) return group_concat, err } @@ -58,9 +58,9 @@ const getGroupConcatInt2 = `-- name: GetGroupConcatInt2 :one SELECT group_concat(1, ':') FROM test ` -func (q *Queries) GetGroupConcatInt2(ctx context.Context) (string, error) { +func (q *Queries) GetGroupConcatInt2(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getGroupConcatInt2) - var group_concat string + var group_concat sql.NullString err := row.Scan(&group_concat) return group_concat, err } @@ -69,9 +69,9 @@ const getGroupConcatText = `-- name: GetGroupConcatText :one SELECT group_concat(text_val) FROM test ` -func (q *Queries) GetGroupConcatText(ctx context.Context) (string, error) { +func (q *Queries) GetGroupConcatText(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getGroupConcatText) - var group_concat string + var group_concat sql.NullString err := row.Scan(&group_concat) return group_concat, err } @@ -80,9 +80,9 @@ const getGroupConcatText2 = `-- name: GetGroupConcatText2 :one SELECT group_concat(text_val, ':') FROM test ` -func (q *Queries) GetGroupConcatText2(ctx context.Context) (string, error) { +func (q *Queries) GetGroupConcatText2(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getGroupConcatText2) - var group_concat string + var group_concat sql.NullString err := row.Scan(&group_concat) return group_concat, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go index 47989b287c..adaf3985f7 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go @@ -289,9 +289,9 @@ const getRandom = `-- name: GetRandom :one SELECT random() ` -func (q *Queries) GetRandom(ctx context.Context) (any, error) { +func (q *Queries) GetRandom(ctx context.Context) (int64, error) { row := q.db.QueryRowContext(ctx, getRandom) - var random any + var random int64 err := row.Scan(&random) return random, err } @@ -300,9 +300,9 @@ const getRandomBlob = `-- name: GetRandomBlob :one SELECT randomblob(16) ` -func (q *Queries) GetRandomBlob(ctx context.Context) (any, error) { +func (q *Queries) GetRandomBlob(ctx context.Context) ([]byte, error) { row := q.db.QueryRowContext(ctx, getRandomBlob) - var randomblob any + var randomblob []byte err := row.Scan(&randomblob) return randomblob, err } @@ -421,9 +421,9 @@ const getSoundex = `-- name: GetSoundex :one SELECT soundex('abc') ` -func (q *Queries) GetSoundex(ctx context.Context) (string, error) { +func (q *Queries) GetSoundex(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getSoundex) - var soundex string + var soundex any err := row.Scan(&soundex) return soundex, err } diff --git a/internal/endtoend/testdata/table_function/sqlite/go/query.sql.go b/internal/endtoend/testdata/table_function/sqlite/go/query.sql.go index 2ee3387a50..345333f3f7 100644 --- a/internal/endtoend/testdata/table_function/sqlite/go/query.sql.go +++ b/internal/endtoend/testdata/table_function/sqlite/go/query.sql.go @@ -29,7 +29,7 @@ type GetTransactionParams struct { type GetTransactionRow struct { JsonExtract any - JsonGroupArray any + JsonGroupArray string } func (q *Queries) GetTransaction(ctx context.Context, arg GetTransactionParams) ([]GetTransactionRow, error) { diff --git a/internal/engine/sqlite/dialect/functions.jsonl b/internal/engine/sqlite/dialect/functions.jsonl index 44a60e4dac..3304b6a55f 100644 --- a/internal/engine/sqlite/dialect/functions.jsonl +++ b/internal/engine/sqlite/dialect/functions.jsonl @@ -1,92 +1,166 @@ -{"name":"AVG","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"COUNT","returns":"integer"} -{"name":"COUNT","args":[{"type":"any"}],"returns":"integer"} -{"name":"GROUP_CONCAT","args":[{"type":"any"}],"returns":"text"} -{"name":"GROUP_CONCAT","args":[{"type":"any"},{"type":"text"}],"returns":"text"} -{"name":"MAX","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"MIN","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"SUM","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"TOTAL","args":[{"type":"any"}],"returns":"real"} -{"name":"ACOS","args":[{"type":"any"}],"returns":"real"} -{"name":"ACOSH","args":[{"type":"any"}],"returns":"real"} -{"name":"ASIN","args":[{"type":"any"}],"returns":"real"} -{"name":"ASINH","args":[{"type":"any"}],"returns":"real"} -{"name":"ATAN","args":[{"type":"any"}],"returns":"real"} -{"name":"ATAN2","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"ATANH","args":[{"type":"any"}],"returns":"real"} -{"name":"CEIL","args":[{"type":"any"}],"returns":"integer"} -{"name":"CEILING","args":[{"type":"any"}],"returns":"integer"} -{"name":"COS","args":[{"type":"any"}],"returns":"real"} -{"name":"COSH","args":[{"type":"any"}],"returns":"real"} -{"name":"DEGREES","args":[{"type":"any"}],"returns":"real"} -{"name":"EXP","args":[{"type":"any"}],"returns":"real"} -{"name":"FLOOR","args":[{"type":"any"}],"returns":"integer"} -{"name":"LN","args":[{"type":"any"}],"returns":"real"} -{"name":"LOG","args":[{"type":"any"}],"returns":"real"} -{"name":"LOG10","args":[{"type":"any"}],"returns":"real"} -{"name":"LOG","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"LOG2","args":[{"type":"any"}],"returns":"real"} -{"name":"MOD","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"PI","returns":"real"} -{"name":"POW","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"POWER","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"RADIANS","args":[{"type":"any"}],"returns":"real"} -{"name":"SIN","args":[{"type":"any"}],"returns":"real"} -{"name":"SINH","args":[{"type":"any"}],"returns":"real"} -{"name":"SQRT","args":[{"type":"any"}],"returns":"real"} -{"name":"TAN","args":[{"type":"any"}],"returns":"real"} -{"name":"TANH","args":[{"type":"any"}],"returns":"real"} -{"name":"TRUNC","args":[{"type":"any"}],"returns":"integer"} -{"name":"ABS","args":[{"type":"any"}],"returns":"real"} -{"name":"CHANGES","returns":"integer"} -{"name":"CHAR","args":[{"type":"int"},{"type":"int","mode":"v"}],"returns":"text"} -{"name":"COALESCE","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"FORMAT","args":[{"type":"text"},{"type":"any","mode":"v"}],"returns":"text","nullable":true} -{"name":"GLOB","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} -{"name":"HEX","args":[{"type":"any"}],"returns":"text"} -{"name":"IFNULL","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"IIF","args":[{"type":"any"},{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"INSTR","args":[{"type":"text"},{"type":"text"}],"returns":"integer","nullable":true} -{"name":"LAST_INSERT_ROWID","returns":"integer"} -{"name":"LENGTH","args":[{"type":"any"}],"returns":"integer","nullable":true} -{"name":"LIKE","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} -{"name":"LIKE","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"integer"} -{"name":"LIKELIHOOD","args":[{"type":"any"},{"type":"real"}],"returns":"any","nullable":true} -{"name":"LIKELY","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"LOWER","args":[{"type":"text"}],"returns":"text"} -{"name":"LTRIM","args":[{"type":"text"}],"returns":"text"} -{"name":"LTRIM","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"MAX","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"MIN","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"NULLIF","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"PRINTF","args":[{"type":"text"},{"type":"any","mode":"v"}],"returns":"text","nullable":true} -{"name":"QUOTE","args":[{"type":"any"}],"returns":"text"} -{"name":"RAMDOM","returns":"integer"} -{"name":"RAMDOMBLOB","args":[{"type":"integer"}],"returns":"blob"} -{"name":"REPLACE","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"ROUND","args":[{"type":"real"}],"returns":"real"} -{"name":"ROUND","args":[{"type":"real"},{"type":"real"}],"returns":"real"} -{"name":"RTRIM","args":[{"type":"text"}],"returns":"text"} -{"name":"RTRIM","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"SIGN","args":[{"type":"any"}],"returns":"integer","nullable":true} -{"name":"SOUNDEX","args":[{"type":"text"}],"returns":"text"} -{"name":"SQLITE_COMPILEOPTION_GET","args":[{"type":"integer"}],"returns":"text","nullable":true} -{"name":"SQLITE_COMPILEOPTION_USED","args":[{"type":"text"}],"returns":"integer"} -{"name":"SQLITE_OFFSET","args":[{"type":"any"}],"returns":"integer","nullable":true} -{"name":"SQLITE_SOURCE_ID","returns":"text"} -{"name":"SQLITE_VERSION","returns":"text"} -{"name":"SUBSTR","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} -{"name":"SUBSTR","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} -{"name":"SUBSTRING","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} -{"name":"SUBSTRING","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} -{"name":"TOTAL_CHANGES","returns":"integer"} -{"name":"TRIM","args":[{"type":"text"}],"returns":"text"} -{"name":"TRIM","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"TYPEOF","args":[{"type":"any"}],"returns":"text"} -{"name":"UNICODE","args":[{"type":"any"}],"returns":"integer"} -{"name":"UNLIKELY","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"UPPER","args":[{"type":"text"}],"returns":"text"} -{"name":"ZEROBLOB","args":[{"type":"integer"}],"returns":"blob"} -{"name":"HIGHLIGHT","args":[{"type":"text"},{"type":"integer"},{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"SNIPPET","args":[{"type":"text"},{"type":"integer"},{"type":"text"},{"type":"text"},{"type":"text"},{"type":"integer"}],"returns":"text"} -{"name":"bm25","args":[{"type":"text"},{"type":"real","mode":"v"}],"returns":"real"} +{"name":"-\u003e","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"-\u003e\u003e","args":[{"type":"any"},{"type":"text"}],"returns":"any","nullable":true} +{"name":"abs","args":[{"type":"any"}],"returns":"real"} +{"name":"acos","args":[{"type":"any"}],"returns":"real"} +{"name":"acosh","args":[{"type":"any"}],"returns":"real"} +{"name":"asin","args":[{"type":"any"}],"returns":"real"} +{"name":"asinh","args":[{"type":"any"}],"returns":"real"} +{"name":"atan","args":[{"type":"any"}],"returns":"real"} +{"name":"atan2","args":[{"type":"any"},{"type":"any"}],"returns":"real"} +{"name":"atanh","args":[{"type":"any"}],"returns":"real"} +{"name":"avg","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"bm25","args":[{"type":"text","has_default":true},{"type":"real","mode":"v"}],"returns":"real"} +{"name":"ceil","args":[{"type":"any"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"any"}],"returns":"integer"} +{"name":"changes","returns":"integer"} +{"name":"char","args":[{"type":"integer","mode":"v"}],"returns":"text"} +{"name":"coalesce","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"concat","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"concat_ws","args":[{"type":"text"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"cos","args":[{"type":"any"}],"returns":"real"} +{"name":"cosh","args":[{"type":"any"}],"returns":"real"} +{"name":"count","kind":"a","returns":"integer"} +{"name":"count","kind":"a","args":[{"type":"any"}],"returns":"integer"} +{"name":"cume_dist","kind":"w","returns":"real"} +{"name":"current_date","returns":"text"} +{"name":"current_time","returns":"text"} +{"name":"current_timestamp","returns":"text"} +{"name":"date","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"datetime","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"degrees","args":[{"type":"any"}],"returns":"real"} +{"name":"dense_rank","kind":"w","returns":"integer"} +{"name":"exp","args":[{"type":"any"}],"returns":"real"} +{"name":"first_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"floor","args":[{"type":"any"}],"returns":"integer"} +{"name":"format","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"glob","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} +{"name":"group_concat","kind":"a","args":[{"type":"any"}],"returns":"text","nullable":true} +{"name":"group_concat","kind":"a","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"hex","args":[{"type":"any"}],"returns":"text"} +{"name":"highlight","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"if","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"ifnull","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"iif","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"instr","args":[{"type":"text"},{"type":"text"}],"returns":"integer","nullable":true} +{"name":"json","args":[{"type":"any"}],"returns":"text"} +{"name":"json_array","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_array_insert","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_array_length","args":[{"type":"any"}],"returns":"integer","nullable":true} +{"name":"json_array_length","args":[{"type":"any"},{"type":"text"}],"returns":"integer","nullable":true} +{"name":"json_error_position","args":[{"type":"any"}],"returns":"integer"} +{"name":"json_extract","args":[{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"json_group_array","kind":"a","args":[{"type":"any"}],"returns":"text"} +{"name":"json_group_object","kind":"a","args":[{"type":"text"},{"type":"any"}],"returns":"text"} +{"name":"json_insert","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_object","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_patch","args":[{"type":"any"},{"type":"any"}],"returns":"text"} +{"name":"json_pretty","args":[{"type":"any"}],"returns":"text"} +{"name":"json_pretty","args":[{"type":"any"},{"type":"text"}],"returns":"text"} +{"name":"json_quote","args":[{"type":"any"}],"returns":"text"} +{"name":"json_remove","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_replace","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_set","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_type","args":[{"type":"any"}],"returns":"text","nullable":true} +{"name":"json_type","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"json_valid","args":[{"type":"any"}],"returns":"integer"} +{"name":"json_valid","args":[{"type":"any"},{"type":"integer"}],"returns":"integer"} +{"name":"jsonb","args":[{"type":"any"}],"returns":"blob"} +{"name":"jsonb_array","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_array_insert","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_extract","args":[{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"jsonb_group_array","kind":"a","args":[{"type":"any"}],"returns":"blob"} +{"name":"jsonb_group_object","kind":"a","args":[{"type":"text"},{"type":"any"}],"returns":"blob"} +{"name":"jsonb_insert","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_object","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_patch","args":[{"type":"any"},{"type":"any"}],"returns":"blob"} +{"name":"jsonb_remove","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_replace","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_set","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"julianday","args":[{"type":"any","mode":"v"}],"returns":"real","nullable":true} +{"name":"lag","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"lag","kind":"w","args":[{"type":"any"},{"type":"integer"}],"returns":"any","nullable":true} +{"name":"lag","kind":"w","args":[{"type":"any"},{"type":"integer"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"last_insert_rowid","returns":"integer"} +{"name":"last_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"lead","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"lead","kind":"w","args":[{"type":"any"},{"type":"integer"}],"returns":"any","nullable":true} +{"name":"lead","kind":"w","args":[{"type":"any"},{"type":"integer"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"length","args":[{"type":"any"}],"returns":"integer","nullable":true} +{"name":"like","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} +{"name":"like","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"integer"} +{"name":"likelihood","args":[{"type":"any"},{"type":"real"}],"returns":"any","nullable":true} +{"name":"likely","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"ln","args":[{"type":"any"}],"returns":"real"} +{"name":"log","args":[{"type":"any"}],"returns":"real"} +{"name":"log","args":[{"type":"any"},{"type":"any"}],"returns":"real"} +{"name":"log10","args":[{"type":"any"}],"returns":"real"} +{"name":"log2","args":[{"type":"any"}],"returns":"real"} +{"name":"lower","args":[{"type":"text"}],"returns":"text"} +{"name":"ltrim","args":[{"type":"text"}],"returns":"text"} +{"name":"ltrim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"max","kind":"a","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"max","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"median","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"min","kind":"a","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"min","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"mod","args":[{"type":"any"},{"type":"any"}],"returns":"real"} +{"name":"nth_value","kind":"w","args":[{"type":"any"},{"type":"integer"}],"returns":"any","nullable":true} +{"name":"ntile","kind":"w","args":[{"type":"integer"}],"returns":"integer"} +{"name":"nullif","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"octet_length","args":[{"type":"any"}],"returns":"integer","nullable":true} +{"name":"percent_rank","kind":"w","returns":"real"} +{"name":"percentile","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile_cont","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile_disc","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"any","nullable":true} +{"name":"pi","returns":"real"} +{"name":"pow","args":[{"type":"any"},{"type":"any"}],"returns":"real"} +{"name":"power","args":[{"type":"any"},{"type":"any"}],"returns":"real"} +{"name":"printf","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"quote","args":[{"type":"any"}],"returns":"text"} +{"name":"radians","args":[{"type":"any"}],"returns":"real"} +{"name":"random","returns":"integer"} +{"name":"randomblob","args":[{"type":"integer"}],"returns":"blob"} +{"name":"rank","kind":"w","returns":"integer"} +{"name":"replace","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"round","args":[{"type":"real"}],"returns":"real"} +{"name":"round","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"row_number","kind":"w","returns":"integer"} +{"name":"rtrim","args":[{"type":"text"}],"returns":"text"} +{"name":"rtrim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"sign","args":[{"type":"any"}],"returns":"integer","nullable":true} +{"name":"sin","args":[{"type":"any"}],"returns":"real"} +{"name":"sinh","args":[{"type":"any"}],"returns":"real"} +{"name":"snippet","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"sqlite_compileoption_get","args":[{"type":"integer"}],"returns":"text","nullable":true} +{"name":"sqlite_compileoption_used","args":[{"type":"text"}],"returns":"integer"} +{"name":"sqlite_offset","args":[{"type":"any"}],"returns":"integer","nullable":true} +{"name":"sqlite_source_id","returns":"text"} +{"name":"sqlite_version","returns":"text"} +{"name":"sqrt","args":[{"type":"any"}],"returns":"real"} +{"name":"strftime","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"string_agg","kind":"a","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"substr","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} +{"name":"substr","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} +{"name":"substring","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} +{"name":"substring","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} +{"name":"subtype","args":[{"type":"any"}],"returns":"integer"} +{"name":"sum","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"tan","args":[{"type":"any"}],"returns":"real"} +{"name":"tanh","args":[{"type":"any"}],"returns":"real"} +{"name":"time","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"timediff","args":[{"type":"any"},{"type":"any"}],"returns":"text","nullable":true} +{"name":"total","kind":"a","args":[{"type":"any"}],"returns":"real"} +{"name":"total_changes","returns":"integer"} +{"name":"trim","args":[{"type":"text"}],"returns":"text"} +{"name":"trim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"trunc","args":[{"type":"any"}],"returns":"integer"} +{"name":"typeof","args":[{"type":"any"}],"returns":"text"} +{"name":"unhex","args":[{"type":"text"}],"returns":"blob","nullable":true} +{"name":"unhex","args":[{"type":"text"},{"type":"text"}],"returns":"blob","nullable":true} +{"name":"unicode","args":[{"type":"text"}],"returns":"integer"} +{"name":"unistr","args":[{"type":"text"}],"returns":"text"} +{"name":"unistr_quote","args":[{"type":"text"}],"returns":"text"} +{"name":"unixepoch","args":[{"type":"any","mode":"v"}],"returns":"integer","nullable":true} +{"name":"unlikely","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"upper","args":[{"type":"text"}],"returns":"text"} +{"name":"zeroblob","args":[{"type":"integer"}],"returns":"blob"} diff --git a/internal/engine/sqlite/stdlib.go b/internal/engine/sqlite/stdlib.go index 67e1f12dae..0a4c4960e4 100644 --- a/internal/engine/sqlite/stdlib.go +++ b/internal/engine/sqlite/stdlib.go @@ -5,13 +5,8 @@ import ( ) // defaultSchema is SQLite's standard library, read from the dialect -// directory's functions.jsonl. -// -// The functions are drawn from: -// -// https://www.sqlite.org/lang_aggfunc.html -// https://www.sqlite.org/lang_mathfunc.html -// https://www.sqlite.org/lang_corefunc.html +// directory's functions.jsonl, which internal/goldeneye generates from the +// sqlite3 shell's pragma_function_list. func defaultSchema(name string) *catalog.Schema { return &catalog.Schema{Name: name, Funcs: stdlib()} } diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index 3bd0a35787..db6976a205 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -15,6 +15,7 @@ reads the files: the files are the contract. Run it from this directory: ```bash go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary once +go run ./cmd/goldeneye install sqlite # download the pinned sqlite3 shell once go run ./cmd/goldeneye check # check every engine whose database is available go run ./cmd/goldeneye check postgresql # check one engine go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database @@ -54,14 +55,29 @@ the hand-written files alone, and the checks do not look at them. `clickhouse/install.go`, and a download that does not match is discarded. ClickHouse describes its functions no further than their names, so `functions.jsonl` is hand-written. +- **`sqlite`** needs no server either: `functions.jsonl` comes from + `pragma_function_list` of the `sqlite3` shell sqlite.org publishes, run + against an in-memory database. SQLite describes its functions as far as + their names, their kinds and the number of arguments each overload takes, + and no further — it types values, not functions — so what each returns and + what its arguments hold comes from the table in `sqlite/signatures.go`, + and a built-in function the shell reports that the table does not know + fails the run rather than being guessed at. The shell is downloaded once + per pinned release by `install` into the user cache directory, or supplied + through the `SQLITE3` environment variable; the pinned release, which is + the one the main module's driver embeds, and the SHA3-256 of each + platform's download live in `sqlite/install.go`. SQLite has no catalog of + types or operators, so `types.jsonl` and `operators.jsonl` are + hand-written. ## Layout - `dialect/` — the record types the files are made of, mirrored from `internal/core/seed`, and the helpers that write a generated set of files into an engine directory or diff it against what is committed. -- `postgresql/`, `duckdb/`, `clickhouse/` — one package per engine, each - exposing `Locate`, `Version` and `Generate`, and a test that runs the check. +- `postgresql/`, `duckdb/`, `clickhouse/`, `sqlite/` — one package per + engine, each exposing `Locate`, `Version` and `Generate`, and a test that + runs the check. - `cmd/goldeneye/` — the command. The analysis checks — verifying the `analyze_*` cases under diff --git a/internal/goldeneye/cmd/goldeneye/main.go b/internal/goldeneye/cmd/goldeneye/main.go index c3af78df38..438f2c36bf 100644 --- a/internal/goldeneye/cmd/goldeneye/main.go +++ b/internal/goldeneye/cmd/goldeneye/main.go @@ -5,6 +5,7 @@ // Usage, from internal/goldeneye: // // go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary +// go run ./cmd/goldeneye install sqlite # download the pinned sqlite3 shell // go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database // go run ./cmd/goldeneye check [engine] # compare the committed files with the database // @@ -26,6 +27,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" "github.com/sqlc-dev/sqlc/internal/goldeneye/duckdb" "github.com/sqlc-dev/sqlc/internal/goldeneye/postgresql" + "github.com/sqlc-dev/sqlc/internal/goldeneye/sqlite" ) func main() { @@ -36,14 +38,14 @@ func main() { } const usage = `usage: - goldeneye install clickhouse [-version V] - download the pinned clickhouse binary into the user cache directory + goldeneye install clickhouse|sqlite [-version V] + download the pinned binary for an engine into the user cache directory goldeneye generate [engine] rewrite the generated dialect files from the database, for every available engine or one goldeneye check [engine] compare the committed dialect files with the database, for every available engine or one -engines: clickhouse, duckdb, postgresql` +engines: clickhouse, duckdb, postgresql, sqlite` // engine is one database goldeneye knows how to read a dialect from. type engine struct { @@ -61,6 +63,19 @@ var engines = []engine{ {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate}, {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate}, {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate}, + {sqlite.Engine, sqlite.Locate, sqlite.Version, sqlite.Generate}, +} + +// installer downloads the binary an engine is read through, for the engines +// that need no server and whose release is pinned in their package. +type installer struct { + defaultVersion string + install func(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) +} + +var installers = map[string]installer{ + clickhouse.Engine: {clickhouse.DefaultVersion, clickhouse.Install}, + sqlite.Engine: {sqlite.DefaultVersion, sqlite.Install}, } func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { @@ -84,16 +99,20 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { } func install(ctx context.Context, args []string, stdout, stderr io.Writer) error { - if len(args) == 0 || args[0] != clickhouse.Engine { - return errors.New("install takes the engine to install: clickhouse") + if len(args) == 0 { + return errors.New("install takes the engine to install: clickhouse or sqlite") + } + inst, ok := installers[args[0]] + if !ok { + return fmt.Errorf("install takes the engine to install, clickhouse or sqlite, not %q", args[0]) } - fs := flag.NewFlagSet("install", flag.ContinueOnError) + fs := flag.NewFlagSet("install "+args[0], flag.ContinueOnError) fs.SetOutput(stderr) - version := fs.String("version", clickhouse.DefaultVersion, "ClickHouse release to install") + version := fs.String("version", inst.defaultVersion, args[0]+" release to install") if err := fs.Parse(args[1:]); err != nil { return err } - path, err := clickhouse.Install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) + path, err := inst.install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) if err != nil { return err } diff --git a/internal/goldeneye/sqlite/install.go b/internal/goldeneye/sqlite/install.go new file mode 100644 index 0000000000..3bc14c000f --- /dev/null +++ b/internal/goldeneye/sqlite/install.go @@ -0,0 +1,206 @@ +package sqlite + +import ( + "archive/zip" + "context" + "crypto/sha3" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" +) + +// DefaultVersion is the SQLite release the dialect is generated from. It is +// the release the ncruces/go-sqlite3 driver in the main module embeds, so +// the functions the dialect knows are the ones the tests run against. +// Bumping it is a deliberate change: releases add functions and overloads, +// so regenerate and review the dialect after changing it, and add the new +// release's downloads to the table below. +const DefaultVersion = "3.53.4" + +// asset is one downloadable build of the SQLite command-line tools: a zip +// published on sqlite.org holding the sqlite3 shell (sqlite3.exe on +// Windows) next to sqldiff, sqlite3_analyzer and sqlite3_rsync. +type asset struct { + Version string + OS string + Arch string + // Path is the download's address under https://sqlite.org/, the + // release year included, as the download page's index lists it. + Path string + // SHA3 is the SHA3-256 of the zip, as the download page lists it. + SHA3 string +} + +// assets lists every build Install knows how to fetch, with the SHA3-256 of +// the download. A version that is not in this table cannot be installed: +// verifying the download is the point of the table. +// +// The checksums are the ones in the index at the foot of +// https://sqlite.org/download.html. sqlite.org builds the tools for x64 +// Linux, both macOS architectures and both Windows architectures; there is +// no Linux arm64 build to list. +var assets = []asset{ + {"3.53.4", "linux", "amd64", "2026/sqlite-tools-linux-x64-3530400.zip", "6eeb57e8f2aef7687f9f016a980992cf2799c8c07a87c5e21495530f91915047"}, + {"3.53.4", "darwin", "amd64", "2026/sqlite-tools-osx-x64-3530400.zip", "3353bb4e5ac54f85c5b82012d30476d20539a9495abdd11a1707df578cff2d7e"}, + {"3.53.4", "darwin", "arm64", "2026/sqlite-tools-osx-arm64-3530400.zip", "58d53e0eb69c17cabebed2754bf399e4d44939be42dcf194769c00078bfd776d"}, + {"3.53.4", "windows", "amd64", "2026/sqlite-tools-win-x64-3530400.zip", "88b4659fe747896b853af10157316b4ade143553efb89c1c8ca7423a278dcc8b"}, + {"3.53.4", "windows", "arm64", "2026/sqlite-tools-win-arm64-3530400.zip", "0c99da3702b2517c1d738207db7e945e5c55be7748141a192a1c8f3b4455c44b"}, +} + +// releaseAsset finds the build for a platform in the table. +func releaseAsset(version, goos, goarch string) (asset, error) { + for _, a := range assets { + if a.Version == version && a.OS == goos && a.Arch == goarch { + return a, nil + } + } + for _, a := range assets { + if a.Version == version { + return asset{}, fmt.Errorf("no SQLite %s build is listed for %s/%s", version, goos, goarch) + } + } + return asset{}, fmt.Errorf("SQLite %s is not in the asset table; add its downloads and checksums to install.go", version) +} + +// url is the asset's download address. +func (a asset) url() string { + return "https://sqlite.org/" + a.Path +} + +// binaryName is what the shell is called inside the zip and in the cache. +func binaryName(goos string) string { + if goos == "windows" { + return "sqlite3.exe" + } + return "sqlite3" +} + +// cachedBinary is where Install puts the shell for a version. +func cachedBinary(version, goos string) (string, error) { + dir, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(dir, "sqlc-sqlite", version, binaryName(goos)), nil +} + +// Locate finds a sqlite3 shell: the SQLITE3 environment variable wins, then +// the cached copy of DefaultVersion. +func Locate() (string, error) { + if path := os.Getenv("SQLITE3"); path != "" { + return path, nil + } + path, err := cachedBinary(DefaultVersion, runtime.GOOS) + if err != nil { + return "", err + } + if _, err := os.Stat(path); err != nil { + return "", fmt.Errorf("sqlite %s is not installed: run `go run ./cmd/goldeneye install sqlite` in internal/goldeneye, or set SQLITE3 to a sqlite3 shell", DefaultVersion) + } + return path, nil +} + +// Install downloads the sqlite3 shell for a version into the cache and +// returns its path. It is a no-op when the version is already cached. The +// zip is checked against the table's SHA3-256 before the shell is taken +// out of it. +func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) { + dest, err := cachedBinary(version, goos) + if err != nil { + return "", err + } + if _, err := os.Stat(dest); err == nil { + return dest, nil + } + a, err := releaseAsset(version, goos, goarch) + if err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return "", err + } + + url := a.url() + fmt.Fprintf(progress, "downloading %s\n", url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("downloading %s: %s", url, resp.Status) + } + + // The zip has to land on disk before it can be read — its directory is + // at the end — so download it whole, next to the destination, and hash + // every byte on the way. + archive, err := os.CreateTemp(filepath.Dir(dest), "sqlite-tools-*.partial") + if err != nil { + return "", err + } + defer os.Remove(archive.Name()) + sum := sha3.New256() + if _, err := io.Copy(io.MultiWriter(archive, sum), resp.Body); err != nil { + archive.Close() + return "", err + } + if err := archive.Close(); err != nil { + return "", err + } + if got := hex.EncodeToString(sum.Sum(nil)); got != a.SHA3 { + return "", fmt.Errorf("downloading %s: SHA3-256 mismatch: got %s, want %s", url, got, a.SHA3) + } + + // Write the shell next to the destination and rename so a partial + // extraction never masquerades as an installed binary. + tmp, err := os.CreateTemp(filepath.Dir(dest), "sqlite3-*.partial") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + if err := extractBinary(archive.Name(), binaryName(goos), tmp); err != nil { + tmp.Close() + return "", fmt.Errorf("downloading %s: %w", url, err) + } + if err := tmp.Close(); err != nil { + return "", err + } + if err := os.Chmod(tmp.Name(), 0o755); err != nil { + return "", err + } + if err := os.Rename(tmp.Name(), dest); err != nil { + return "", err + } + return dest, nil +} + +// extractBinary copies the named file out of a zip. The tools zips are +// flat, so the name is the whole path. +func extractBinary(archive, name string, dst io.Writer) error { + zr, err := zip.OpenReader(archive) + if err != nil { + return err + } + defer zr.Close() + for _, f := range zr.File { + if f.Name != name { + continue + } + rc, err := f.Open() + if err != nil { + return err + } + defer rc.Close() + _, err = io.Copy(dst, rc) + return err + } + return fmt.Errorf("zip does not contain %s", name) +} diff --git a/internal/goldeneye/sqlite/signatures.go b/internal/goldeneye/sqlite/signatures.go new file mode 100644 index 0000000000..d3da84e7c0 --- /dev/null +++ b/internal/goldeneye/sqlite/signatures.go @@ -0,0 +1,243 @@ +package sqlite + +import "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" + +// signature is what SQLite does not record about a function: the type it +// returns, whether that can be NULL when its arguments are not, and the +// types its arguments are meant to hold. Args types the leading arguments +// in order; a position it does not cover is "any", which the analyzer +// resolves to the argument's own type. Variadic types the arguments an +// overload of variable arity repeats, "any" when empty. How many arguments +// an overload takes, and how many of them it requires, comes from the +// shell, not from here. +type signature struct { + Args []string + Variadic string + Returns string + Nullable bool +} + +// args builds an overload's parameters from the shell's argument count. A +// fixed count is that many parameters; a negative count is the required +// leading parameters, any further ones the signature types marked as +// having a default, and a variadic tail. +func (s signature) args(narg int) []dialect.Arg { + if narg >= 0 { + args := make([]dialect.Arg, narg) + for i := range args { + args[i] = dialect.Arg{Type: s.argType(i)} + } + return args + } + required := minArgs(narg) + n := max(required, len(s.Args)) + args := make([]dialect.Arg, 0, n+1) + for i := 0; i < n; i++ { + args = append(args, dialect.Arg{Type: s.argType(i), HasDefault: i >= required}) + } + tail := s.Variadic + if tail == "" { + tail = "any" + } + return append(args, dialect.Arg{Type: tail, Mode: "v"}) +} + +func (s signature) argType(i int) string { + if i < len(s.Args) { + return s.Args[i] + } + return "any" +} + +// omitted are built-in functions the dialect leaves out: ones that exist +// for their side effect and return nothing a query can use, and one the +// shell's build adds that the library does not have. +var omitted = map[string]bool{ + // Loads a shared library and returns NULL. + "load_extension": true, + // Writes to the error log and returns NULL. + "sqlite_log": true, + // SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION's stand-in for any function the + // shell does not know, so that EXPLAIN works on queries that use one. + "unknown": true, +} + +// signatures covers every function the shell builds in, by the SQLite +// documentation of each — lang_corefunc, lang_mathfunc, lang_datefunc, +// lang_aggfunc, windowfunctions and json1 — and the auxiliary functions of +// FTS5, which the shell bundles as an extension. A function marked +// Nullable can return NULL for arguments that are not: an aggregate over +// no rows, a lookup that finds nothing, an input that does not parse. +// +// SQLite's values carry their own types, so a function's result type is +// what it typically produces. abs of an integer is an integer, but abs +// returns real here, as sum does, because the analyzer wants one answer; +// the polymorphic "any" is for functions that hand back one of their +// arguments. +var signatures = map[string]signature{ + // Aggregates. + "avg": {Returns: "real", Nullable: true}, + "count": {Returns: "integer"}, + "group_concat": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, + "max": {Returns: "any", Nullable: true}, + "min": {Returns: "any", Nullable: true}, + "string_agg": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, + "sum": {Returns: "real", Nullable: true}, + "total": {Returns: "real"}, + + // Percentiles, built in by SQLITE_ENABLE_PERCENTILE. + "median": {Returns: "real", Nullable: true}, + "percentile": {Args: []string{"any", "real"}, Returns: "real", Nullable: true}, + "percentile_cont": {Args: []string{"any", "real"}, Returns: "real", Nullable: true}, + "percentile_disc": {Args: []string{"any", "real"}, Returns: "any", Nullable: true}, + + // Window functions. + "cume_dist": {Returns: "real"}, + "dense_rank": {Returns: "integer"}, + "first_value": {Returns: "any", Nullable: true}, + "lag": {Args: []string{"any", "integer", "any"}, Returns: "any", Nullable: true}, + "last_value": {Returns: "any", Nullable: true}, + "lead": {Args: []string{"any", "integer", "any"}, Returns: "any", Nullable: true}, + "nth_value": {Args: []string{"any", "integer"}, Returns: "any", Nullable: true}, + "ntile": {Args: []string{"integer"}, Returns: "integer"}, + "percent_rank": {Returns: "real"}, + "rank": {Returns: "integer"}, + "row_number": {Returns: "integer"}, + + // Core functions. + "changes": {Returns: "integer"}, + "char": {Variadic: "integer", Returns: "text"}, + "coalesce": {Returns: "any", Nullable: true}, + "concat": {Returns: "text"}, + "concat_ws": {Args: []string{"text"}, Returns: "text"}, + "format": {Args: []string{"text"}, Returns: "text", Nullable: true}, + "glob": {Args: []string{"text", "text"}, Returns: "integer"}, + "hex": {Returns: "text"}, + "if": {Returns: "any", Nullable: true}, + "ifnull": {Returns: "any", Nullable: true}, + "iif": {Returns: "any", Nullable: true}, + "instr": {Args: []string{"text", "text"}, Returns: "integer", Nullable: true}, + "last_insert_rowid": {Returns: "integer"}, + "length": {Returns: "integer", Nullable: true}, + "like": {Args: []string{"text", "text", "text"}, Returns: "integer"}, + "likelihood": {Args: []string{"any", "real"}, Returns: "any", Nullable: true}, + "likely": {Returns: "any", Nullable: true}, + "lower": {Args: []string{"text"}, Returns: "text"}, + "ltrim": {Args: []string{"text", "text"}, Returns: "text"}, + "nullif": {Returns: "any", Nullable: true}, + "octet_length": {Returns: "integer", Nullable: true}, + "printf": {Args: []string{"text"}, Returns: "text", Nullable: true}, + "quote": {Returns: "text"}, + "random": {Returns: "integer"}, + "randomblob": {Args: []string{"integer"}, Returns: "blob"}, + "replace": {Args: []string{"text", "text", "text"}, Returns: "text"}, + "round": {Args: []string{"real", "real"}, Returns: "real"}, + "rtrim": {Args: []string{"text", "text"}, Returns: "text"}, + "sign": {Returns: "integer", Nullable: true}, + "sqlite_compileoption_get": {Args: []string{"integer"}, Returns: "text", Nullable: true}, + "sqlite_compileoption_used": {Args: []string{"text"}, Returns: "integer"}, + "sqlite_offset": {Returns: "integer", Nullable: true}, + "sqlite_source_id": {Returns: "text"}, + "sqlite_version": {Returns: "text"}, + "substr": {Args: []string{"any", "integer", "integer"}, Returns: "text"}, + "substring": {Args: []string{"any", "integer", "integer"}, Returns: "text"}, + "subtype": {Returns: "integer"}, + "total_changes": {Returns: "integer"}, + "trim": {Args: []string{"text", "text"}, Returns: "text"}, + "typeof": {Returns: "text"}, + "unhex": {Args: []string{"text", "text"}, Returns: "blob", Nullable: true}, + "unicode": {Args: []string{"text"}, Returns: "integer"}, + "unistr": {Args: []string{"text"}, Returns: "text"}, + "unistr_quote": {Args: []string{"text"}, Returns: "text"}, + "unlikely": {Returns: "any", Nullable: true}, + "upper": {Args: []string{"text"}, Returns: "text"}, + "zeroblob": {Args: []string{"integer"}, Returns: "blob"}, + + // Math functions. + "abs": {Returns: "real"}, + "acos": {Returns: "real"}, + "acosh": {Returns: "real"}, + "asin": {Returns: "real"}, + "asinh": {Returns: "real"}, + "atan": {Returns: "real"}, + "atan2": {Returns: "real"}, + "atanh": {Returns: "real"}, + "ceil": {Returns: "integer"}, + "ceiling": {Returns: "integer"}, + "cos": {Returns: "real"}, + "cosh": {Returns: "real"}, + "degrees": {Returns: "real"}, + "exp": {Returns: "real"}, + "floor": {Returns: "integer"}, + "ln": {Returns: "real"}, + "log": {Returns: "real"}, + "log10": {Returns: "real"}, + "log2": {Returns: "real"}, + "mod": {Returns: "real"}, + "pi": {Returns: "real"}, + "pow": {Returns: "real"}, + "power": {Returns: "real"}, + "radians": {Returns: "real"}, + "sin": {Returns: "real"}, + "sinh": {Returns: "real"}, + "sqrt": {Returns: "real"}, + "tan": {Returns: "real"}, + "tanh": {Returns: "real"}, + "trunc": {Returns: "integer"}, + + // Date and time functions, which return text in ISO-8601 form and NULL + // for a time value they cannot parse. + "current_date": {Returns: "text"}, + "current_time": {Returns: "text"}, + "current_timestamp": {Returns: "text"}, + "date": {Returns: "text", Nullable: true}, + "datetime": {Returns: "text", Nullable: true}, + "julianday": {Returns: "real", Nullable: true}, + "strftime": {Args: []string{"text"}, Returns: "text", Nullable: true}, + "time": {Returns: "text", Nullable: true}, + "timediff": {Returns: "text", Nullable: true}, + "unixepoch": {Returns: "integer", Nullable: true}, + + // JSON functions. A JSON argument is text or a JSONB blob, so it is + // "any"; a path is text. The json_ forms return JSON text and the + // jsonb_ forms JSONB blobs, and the extracting forms return whatever + // the path leads to, or NULL when it leads nowhere. + "->": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, + "->>": {Args: []string{"any", "text"}, Returns: "any", Nullable: true}, + "json": {Returns: "text"}, + "json_array": {Returns: "text"}, + "json_array_insert": {Args: []string{"any"}, Returns: "text"}, + "json_array_length": {Args: []string{"any", "text"}, Returns: "integer", Nullable: true}, + "json_error_position": {Returns: "integer"}, + "json_extract": {Args: []string{"any", "text"}, Returns: "any", Nullable: true}, + "json_group_array": {Returns: "text"}, + "json_group_object": {Args: []string{"text", "any"}, Returns: "text"}, + "json_insert": {Args: []string{"any"}, Returns: "text"}, + "json_object": {Returns: "text"}, + "json_patch": {Returns: "text"}, + "json_pretty": {Args: []string{"any", "text"}, Returns: "text"}, + "json_quote": {Returns: "text"}, + "json_remove": {Args: []string{"any"}, Returns: "text"}, + "json_replace": {Args: []string{"any"}, Returns: "text"}, + "json_set": {Args: []string{"any"}, Returns: "text"}, + "json_type": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, + "json_valid": {Args: []string{"any", "integer"}, Returns: "integer"}, + "jsonb": {Returns: "blob"}, + "jsonb_array": {Returns: "blob"}, + "jsonb_array_insert": {Args: []string{"any"}, Returns: "blob"}, + "jsonb_extract": {Args: []string{"any", "text"}, Returns: "any", Nullable: true}, + "jsonb_group_array": {Returns: "blob"}, + "jsonb_group_object": {Args: []string{"text", "any"}, Returns: "blob"}, + "jsonb_insert": {Args: []string{"any"}, Returns: "blob"}, + "jsonb_object": {Returns: "blob"}, + "jsonb_patch": {Returns: "blob"}, + "jsonb_remove": {Args: []string{"any"}, Returns: "blob"}, + "jsonb_replace": {Args: []string{"any"}, Returns: "blob"}, + "jsonb_set": {Args: []string{"any"}, Returns: "blob"}, + + // FTS5's auxiliary functions, registered by the extension rather than + // built in, and the only ones of its functions a query calls. + "bm25": {Args: []string{"text"}, Variadic: "real", Returns: "real"}, + "highlight": {Args: []string{"text", "integer", "text", "text"}, Returns: "text"}, + "snippet": {Args: []string{"text", "integer", "text", "text", "text", "integer"}, Returns: "text"}, +} diff --git a/internal/goldeneye/sqlite/sqlite.go b/internal/goldeneye/sqlite/sqlite.go new file mode 100644 index 0000000000..56a1953bed --- /dev/null +++ b/internal/goldeneye/sqlite/sqlite.go @@ -0,0 +1,221 @@ +// Package sqlite generates the SQLite dialect seed under +// internal/engine/sqlite/dialect from the sqlite3 shell sqlite.org +// publishes, run against an in-memory database that needs no server. +// +// SQLite describes its functions as far as their names, their kinds and the +// number of arguments each takes — pragma_function_list — and no further: +// it types values rather than columns or functions, so nothing in the +// database says what a function returns or what it expects. functions.jsonl +// is therefore built from both sides. The shell says which functions exist, +// how many arguments each overload takes and whether it aggregates, and the +// signatures table in this package says what each returns and what its +// arguments are meant to hold. A built-in function the shell reports that +// the table does not know fails generation rather than being guessed at, +// and so does a table entry the shell does not report. SQLite has no +// catalog of types or operators, so types.jsonl and operators.jsonl are +// hand-written. +// +// The shell is downloaded once per pinned version by Install, or supplied +// through the SQLITE3 environment variable. +package sqlite + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os/exec" + "sort" + "strconv" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// Engine is the name of the engine directory the dialect lives under. +const Engine = "sqlite" + +// functionList is every function the connection knows, one row per +// overload: name, whether the library builds it in or the shell or an +// extension registered it, 's', 'a' or 'w' for how it is called, and how +// many arguments it takes. Fixed arities come before the variable one so +// that an exact match is found first by anything reading the file in +// order. +const functionList = ` +SELECT name, builtin, type, narg +FROM pragma_function_list +ORDER BY name, narg < 0, narg` + +type functionRow struct { + Name string `json:"name"` + Builtin int `json:"builtin"` + Type string `json:"type"` + NArg int `json:"narg"` +} + +// Version reports the release a shell is. +func Version(ctx context.Context, binary string) (string, error) { + out, err := exec.CommandContext(ctx, binary, "--version").Output() + if err != nil { + return "", fmt.Errorf("sqlite3 --version: %w", err) + } + return "SQLite " + strings.TrimSpace(string(out)), nil +} + +// query runs a SQL statement against an in-memory database and decodes the +// shell's JSON output into rows. +func query(ctx context.Context, binary, sql string, rows any) error { + cmd := exec.CommandContext(ctx, binary, "-json", "-bail", ":memory:", sql) + out, err := cmd.Output() + if err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) { + return fmt.Errorf("sqlite3: %s: %s", err, bytes.TrimSpace(exit.Stderr)) + } + return fmt.Errorf("sqlite3: %w", err) + } + // An empty result set prints nothing rather than []. + if len(bytes.TrimSpace(out)) == 0 { + return nil + } + return json.Unmarshal(out, rows) +} + +// minArgs decodes a negative argument count from pragma_function_list, +// which prints the count SQLite keeps for the function as it is: -1 for any +// number of arguments, and the two values reserved for built-ins, -3 for +// one or more and -4 for two or more — see matchQuality in the SQLite +// source. No function is counted as -2. +func minArgs(narg int) int { + if narg < -2 { + return -2 - narg + } + return 0 +} + +// functionKind maps the type the shell reports onto the seed's letters. +// The shell says 's' for a scalar function, 'a' for an aggregate and 'w' +// for one with a window implementation — but every one of SQLite's +// aggregates has one, so 'w' covers avg and count as well as row_number. +// The two are told apart by calling the function without an OVER clause, +// which only a window function refuses. +func functionKind(ctx context.Context, binary string, row functionRow) (string, error) { + switch row.Type { + case "s": + return "", nil + case "a": + return "a", nil + case "w": + window, err := isWindowFunction(ctx, binary, row) + if err != nil { + return "", err + } + if window { + return "w", nil + } + return "a", nil + } + return "", fmt.Errorf("sqlite: function %s has unknown type %q", row.Name, row.Type) +} + +// isWindowFunction reports whether the shell refuses to call a function +// without an OVER clause. Any other complaint — an aggregate objecting to +// the NULLs it is handed — means the call was accepted as an aggregate. +func isWindowFunction(ctx context.Context, binary string, row functionRow) (bool, error) { + n := row.NArg + if n < 0 { + n = minArgs(n) + } + args := strings.TrimSuffix(strings.Repeat("NULL, ", n), ", ") + cmd := exec.CommandContext(ctx, binary, ":memory:", fmt.Sprintf("SELECT %s(%s)", row.Name, args)) + var stderr bytes.Buffer + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + err := cmd.Run() + if err == nil { + return false, nil + } + if strings.Contains(stderr.String(), "misuse of window function") { + return true, nil + } + var exit *exec.ExitError + if errors.As(err, &exit) { + return false, nil + } + return false, fmt.Errorf("sqlite3: %w", err) +} + +// readFunctions turns the shell's list into the dialect's, one record per +// overload of every function the signatures table knows. A built-in +// function without a signature is an error; a function without one that +// the shell or a bundled extension registered is not the dialect's +// business. +func readFunctions(ctx context.Context, binary string, rows []functionRow) ([]dialect.Function, error) { + var funcs []dialect.Function + seen := map[string]bool{} + reported := map[string]bool{} + var missing []string + for _, row := range rows { + reported[row.Name] = true + if omitted[row.Name] { + continue + } + sig, ok := signatures[row.Name] + if !ok { + if row.Builtin != 0 && !seen[row.Name] { + seen[row.Name] = true + missing = append(missing, row.Name) + } + continue + } + key := row.Name + "\x00" + strconv.Itoa(row.NArg) + if seen[key] { + continue + } + seen[key] = true + kind, err := functionKind(ctx, binary, row) + if err != nil { + return nil, err + } + funcs = append(funcs, dialect.Function{ + Name: row.Name, + Kind: kind, + Args: sig.args(row.NArg), + Returns: sig.Returns, + Nullable: sig.Nullable, + }) + } + if len(missing) > 0 { + return nil, fmt.Errorf("sqlite: no signature for built-in function(s) %s: add them to signatures.go", strings.Join(missing, ", ")) + } + var stale []string + for name := range signatures { + if !reported[name] { + stale = append(stale, name) + } + } + if len(stale) > 0 { + sort.Strings(stale) + return nil, fmt.Errorf("sqlite: signatures.go lists function(s) the shell does not report: %s", strings.Join(stale, ", ")) + } + return funcs, nil +} + +// Generate reads the dialect from the shell. +func Generate(ctx context.Context, binary string) (dialect.Files, error) { + var rows []functionRow + if err := query(ctx, binary, functionList, &rows); err != nil { + return nil, err + } + funcs, err := readFunctions(ctx, binary, rows) + if err != nil { + return nil, err + } + functions, err := dialect.JSONL(funcs) + if err != nil { + return nil, err + } + return dialect.Files{dialect.FunctionsFile: functions}, nil +} diff --git a/internal/goldeneye/sqlite/sqlite_test.go b/internal/goldeneye/sqlite/sqlite_test.go new file mode 100644 index 0000000000..028f015fbf --- /dev/null +++ b/internal/goldeneye/sqlite/sqlite_test.go @@ -0,0 +1,37 @@ +package sqlite + +import ( + "context" + "testing" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// TestDialect verifies the committed SQLite dialect against what the pinned +// sqlite3 shell reports. It skips unless the shell is installed. +func TestDialect(t *testing.T) { + binary, err := Locate() + if err != nil { + t.Skip(err) + } + ctx := context.Background() + version, err := Version(ctx, binary) + if err != nil { + t.Fatal(err) + } + files, err := Generate(ctx, binary) + if err != nil { + t.Fatal(err) + } + dir, err := dialect.Dir(Engine) + if err != nil { + t.Fatal(err) + } + report, err := dialect.Check(dir, files) + if err != nil { + t.Fatal(err) + } + if report != "" { + t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) + } +} From ea6e34158991fc35875e1ce02877a5517d50e87f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 03:24:16 +0000 Subject: [PATCH 2/3] goldeneye: build SQLite from source and treat compile options as extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Which functions a SQLite has is decided when it is compiled, and the official sqlite3 binary's option set matches neither the driver sqlc's tests run against nor any other in particular. So instead of downloading that binary, `install sqlite` downloads the pinned release's amalgamation, checked against the SHA3-256 sqlite.org lists, and compiles the shell from it with cc or $CC: once with the options sqlite.org's own configure turns on by default, which gives functions.jsonl, and once per option in the extension list, each of which gets a directory under extensions/ holding the functions its build adds over the default one — found the way the PostgreSQL generator finds what CREATE EXTENSION adds, by comparing the catalog with and without. GEOPOLY lives inside the RTREE module, so its build carries both options. Each shell is checked against pragma compile_options before it is read, so a stale build is caught. The eight builds take about fifteen seconds unoptimised. A schema says which options it needs the way SQLite itself does, with CREATE VIRTUAL TABLE ... USING fts5. dialect.json gains a modules map from a virtual table module to its extension, the seed resolves a module or extension name through it, and both the legacy catalog and the core load the extension when a virtual table is declared, so the fts5 case keeps its typed bm25, highlight and snippet results. The legacy catalog's createExtension also now records what it loaded, which it never did. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RYkxvtwH6GgysuWYP87vyj --- CLAUDE.md | 2 +- internal/core/schema/schema.go | 7 + internal/core/seed/extension.go | 17 +- internal/core/seed/seed.go | 63 +++- internal/engine/sqlite/catalog.go | 3 +- internal/engine/sqlite/dialect/dialect.json | 10 +- .../extensions/enable_fts3/functions.jsonl | 5 + .../extensions/enable_fts5/functions.jsonl | 7 + .../extensions/enable_geopoly/functions.jsonl | 15 + .../enable_offset_sql_func/functions.jsonl | 1 + .../enable_percentile/functions.jsonl | 4 + .../extensions/enable_rtree/functions.jsonl | 3 + .../extensions/soundex/functions.jsonl | 1 + .../engine/sqlite/dialect/functions.jsonl | 8 - internal/engine/sqlite/extension.go | 26 ++ internal/engine/sqlite/stdlib.go | 5 +- internal/goldeneye/README.md | 34 +- internal/goldeneye/cmd/goldeneye/main.go | 9 +- internal/goldeneye/sqlite/install.go | 308 ++++++++++++------ internal/goldeneye/sqlite/signatures.go | 67 +++- internal/goldeneye/sqlite/sqlite.go | 194 ++++++++--- internal/sql/catalog/extension.go | 16 +- internal/sql/catalog/table.go | 7 + 23 files changed, 604 insertions(+), 208 deletions(-) create mode 100644 internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl create mode 100644 internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl create mode 100644 internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl create mode 100644 internal/engine/sqlite/dialect/extensions/enable_offset_sql_func/functions.jsonl create mode 100644 internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl create mode 100644 internal/engine/sqlite/dialect/extensions/enable_rtree/functions.jsonl create mode 100644 internal/engine/sqlite/dialect/extensions/soundex/functions.jsonl create mode 100644 internal/engine/sqlite/extension.go diff --git a/CLAUDE.md b/CLAUDE.md index 0797a3183d..facde359b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -153,7 +153,7 @@ is not available skip. ```bash cd internal/goldeneye go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary once -go run ./cmd/goldeneye install sqlite # download the pinned sqlite3 shell once +go run ./cmd/goldeneye install sqlite # build the pinned sqlite3 shells once; needs a C compiler POSTGRESQL_SERVER_URI="postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable" go test ./... go run ./cmd/goldeneye generate postgresql # rewrite the files after a change ``` diff --git a/internal/core/schema/schema.go b/internal/core/schema/schema.go index 19dc3cc827..19cd4c7c61 100644 --- a/internal/core/schema/schema.go +++ b/internal/core/schema/schema.go @@ -133,6 +133,13 @@ func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error { if stmt.Name == nil { return fmt.Errorf("create table with nil name") } + // A virtual table's module is the dialect's word for the extension it + // needs, the way CREATE EXTENSION is PostgreSQL's. + if stmt.Using != "" { + if err := cat.LoadExtension(stmt.Using); err != nil { + return err + } + } nsOID, err := resolveOrCreateNamespace(cat, stmt.Name.Schema) if err != nil { return err diff --git a/internal/core/seed/extension.go b/internal/core/seed/extension.go index f86fa4929e..ac7eacdde8 100644 --- a/internal/core/seed/extension.go +++ b/internal/core/seed/extension.go @@ -9,17 +9,12 @@ import ( "github.com/sqlc-dev/sqlc/internal/core" ) -// applyExtension applies the named extension's directory to a catalog that -// has already been seeded. Unlike the dialect's own seed, an extension lands -// in a catalog full of types, so everything it names is resolved against what -// is there before being created. -func applyExtension(cat *core.Catalog, fsys fs.FS, name string) error { - dir := path.Join(ExtensionsDir, name) - if _, err := fs.Stat(fsys, dir); err != nil { - // An extension sqlc has no data for adds nothing, the way the legacy - // catalog has always treated one. - return nil - } +// applyExtension applies the extension directory dir, relative to the +// dialect, to a catalog that has already been seeded. Unlike the dialect's +// own seed, an extension lands in a catalog full of types, so everything it +// names is resolved against what is there before being created. +func applyExtension(cat *core.Catalog, fsys fs.FS, dir string) error { + name := path.Base(dir) sub, err := fs.Sub(fsys, dir) if err != nil { return fmt.Errorf("seed: extension %s: %w", name, err) diff --git a/internal/core/seed/seed.go b/internal/core/seed/seed.go index cc0c5a7977..6440a88436 100644 --- a/internal/core/seed/seed.go +++ b/internal/core/seed/seed.go @@ -18,7 +18,8 @@ // // A dialect may also hold an extensions/ directory with one directory per // extension, each a smaller bundle of the same files, applied when a schema -// says CREATE EXTENSION. +// says CREATE EXTENSION — or, for a dialect whose settings map virtual table +// modules to extensions, CREATE VIRTUAL TABLE ... USING. package seed import ( @@ -28,6 +29,7 @@ import ( "fmt" "io" "io/fs" + "path" "slices" "strings" @@ -78,6 +80,15 @@ type Settings struct { // same kind of value resolves. "*" makes every seeded type implicitly // castable to every other, for dialects that compare across categories. CastCategories string `json:"cast_categories,omitempty"` + + // Modules names the extension a virtual table module belongs to, for a + // dialect whose schemas say CREATE VIRTUAL TABLE ... USING rather than + // CREATE EXTENSION: SQLite's fts5 module comes with the functions its + // enable_fts5 compile option adds. + Modules map[string]string `json:"modules,omitempty"` + + // fsys is the dialect directory the settings were read from. + fsys fs.FS } // Type is a type the dialect defines. Aliases are spellings of the same type @@ -155,21 +166,60 @@ func Dialect(fsys fs.FS, dir string) core.Option { if err != nil { return fmt.Errorf("seed: %s: %w", dir, err) } - if err := apply(cat, sub); err != nil { + settings, err := loadSettings(sub) + if err != nil { + return err + } + if err := apply(cat, sub, settings); err != nil { return err } cat.SetExtensionLoader(func(name string) error { - return applyExtension(cat, sub, name) + dir, ok := settings.extensionDir(name) + if !ok { + // An extension sqlc has no data for adds nothing, the way + // the legacy catalog has always treated one. + return nil + } + return applyExtension(cat, sub, dir) }) return nil }) } -func apply(cat *core.Catalog, fsys fs.FS) error { - settings, err := loadSettings(fsys) +// ExtensionDir resolves what a schema named — an extension, or a virtual +// table module the dialect's settings map to one — to the extension's +// directory under dir, reporting whether the dialect has data for it. +func ExtensionDir(fsys fs.FS, dir, name string) (string, bool) { + sub, err := fs.Sub(fsys, dir) if err != nil { - return err + return "", false } + settings, err := loadSettings(sub) + if err != nil { + return "", false + } + rel, ok := settings.extensionDir(name) + if !ok { + return "", false + } + return path.Join(dir, rel), true +} + +// extensionDir is the directory of the extension a name refers to, +// relative to the dialect, if the dialect ships one. +func (s Settings) extensionDir(name string) (string, bool) { + if ext, ok := s.Modules[strings.ToLower(name)]; ok { + name = ext + } + dir := path.Join(ExtensionsDir, name) + if _, err := fs.Stat(s.fsys, dir); err != nil { + return "", false + } + return dir, true +} + +func apply(cat *core.Catalog, fsys fs.FS, settings Settings) error { + var err error b := &builder{ cat: cat, settings: settings, @@ -222,6 +272,7 @@ func loadSettings(fsys fs.FS) (Settings, error) { if settings.Dialect == "" { return Settings{}, fmt.Errorf("seed: %s: dialect has no name", SettingsFile) } + settings.fsys = fsys return settings, nil } diff --git a/internal/engine/sqlite/catalog.go b/internal/engine/sqlite/catalog.go index 1eee9def79..54d198ee9d 100644 --- a/internal/engine/sqlite/catalog.go +++ b/internal/engine/sqlite/catalog.go @@ -9,6 +9,7 @@ func NewCatalog() *catalog.Catalog { Schemas: []*catalog.Schema{ defaultSchema(def), }, - Extensions: map[string]struct{}{}, + LoadExtension: loadExtension, + Extensions: map[string]struct{}{}, } } diff --git a/internal/engine/sqlite/dialect/dialect.json b/internal/engine/sqlite/dialect/dialect.json index 06573b72b9..ba49e48300 100644 --- a/internal/engine/sqlite/dialect/dialect.json +++ b/internal/engine/sqlite/dialect/dialect.json @@ -11,5 +11,13 @@ "comparison_categories": "BNSDU", "arithmetic": ["+", "-", "*", "/", "%"], "arithmetic_categories": "N", - "cast_categories": "*" + "cast_categories": "*", + "modules": { + "fts3": "enable_fts3", + "fts4": "enable_fts3", + "fts5": "enable_fts5", + "geopoly": "enable_geopoly", + "rtree": "enable_rtree", + "rtree_i32": "enable_rtree" + } } diff --git a/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl new file mode 100644 index 0000000000..2553979309 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl @@ -0,0 +1,5 @@ +{"name":"matchinfo","args":[{"type":"any"}],"returns":"blob"} +{"name":"matchinfo","args":[{"type":"any"},{"type":"text"}],"returns":"blob"} +{"name":"offsets","args":[{"type":"any"}],"returns":"text"} +{"name":"optimize","args":[{"type":"any"}],"returns":"text"} +{"name":"snippet","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl new file mode 100644 index 0000000000..82339961b5 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl @@ -0,0 +1,7 @@ +{"name":"bm25","args":[{"type":"text","has_default":true},{"type":"real","mode":"v"}],"returns":"real"} +{"name":"fts5_get_locale","args":[{"type":"any","has_default":true},{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"fts5_insttoken","args":[{"type":"text"}],"returns":"text"} +{"name":"fts5_locale","args":[{"type":"text"},{"type":"text"}],"returns":"text"} +{"name":"fts5_source_id","returns":"text"} +{"name":"highlight","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"snippet","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl new file mode 100644 index 0000000000..3f7f51b9a0 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl @@ -0,0 +1,15 @@ +{"name":"geopoly_area","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"geopoly_bbox","args":[{"type":"any"}],"returns":"blob","nullable":true} +{"name":"geopoly_blob","args":[{"type":"any"}],"returns":"blob","nullable":true} +{"name":"geopoly_ccw","args":[{"type":"any"}],"returns":"blob","nullable":true} +{"name":"geopoly_contains_point","args":[{"type":"any"},{"type":"real"},{"type":"real"}],"returns":"integer","nullable":true} +{"name":"geopoly_group_bbox","kind":"a","args":[{"type":"any"}],"returns":"blob","nullable":true} +{"name":"geopoly_json","args":[{"type":"any"}],"returns":"text","nullable":true} +{"name":"geopoly_overlap","args":[{"type":"any"},{"type":"any"}],"returns":"integer","nullable":true} +{"name":"geopoly_regular","args":[{"type":"real"},{"type":"real"},{"type":"real"},{"type":"integer"}],"returns":"blob"} +{"name":"geopoly_svg","args":[{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"geopoly_within","args":[{"type":"any"},{"type":"any"}],"returns":"integer","nullable":true} +{"name":"geopoly_xform","args":[{"type":"any"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"}],"returns":"blob","nullable":true} +{"name":"rtreecheck","args":[{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"rtreedepth","args":[{"type":"blob"}],"returns":"integer"} +{"name":"rtreenode","args":[{"type":"integer"},{"type":"blob"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_offset_sql_func/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_offset_sql_func/functions.jsonl new file mode 100644 index 0000000000..7e8563d626 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_offset_sql_func/functions.jsonl @@ -0,0 +1 @@ +{"name":"sqlite_offset","args":[{"type":"any"}],"returns":"integer","nullable":true} diff --git a/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl new file mode 100644 index 0000000000..f8ce80d774 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl @@ -0,0 +1,4 @@ +{"name":"median","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"percentile","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile_cont","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile_disc","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"any","nullable":true} diff --git a/internal/engine/sqlite/dialect/extensions/enable_rtree/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_rtree/functions.jsonl new file mode 100644 index 0000000000..a154d3bfc0 --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/enable_rtree/functions.jsonl @@ -0,0 +1,3 @@ +{"name":"rtreecheck","args":[{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"rtreedepth","args":[{"type":"blob"}],"returns":"integer"} +{"name":"rtreenode","args":[{"type":"integer"},{"type":"blob"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/soundex/functions.jsonl b/internal/engine/sqlite/dialect/extensions/soundex/functions.jsonl new file mode 100644 index 0000000000..2644ef5d0b --- /dev/null +++ b/internal/engine/sqlite/dialect/extensions/soundex/functions.jsonl @@ -0,0 +1 @@ +{"name":"soundex","args":[{"type":"text"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/functions.jsonl b/internal/engine/sqlite/dialect/functions.jsonl index 3304b6a55f..f5c849cf80 100644 --- a/internal/engine/sqlite/dialect/functions.jsonl +++ b/internal/engine/sqlite/dialect/functions.jsonl @@ -9,7 +9,6 @@ {"name":"atan2","args":[{"type":"any"},{"type":"any"}],"returns":"real"} {"name":"atanh","args":[{"type":"any"}],"returns":"real"} {"name":"avg","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"bm25","args":[{"type":"text","has_default":true},{"type":"real","mode":"v"}],"returns":"real"} {"name":"ceil","args":[{"type":"any"}],"returns":"integer"} {"name":"ceiling","args":[{"type":"any"}],"returns":"integer"} {"name":"changes","returns":"integer"} @@ -37,7 +36,6 @@ {"name":"group_concat","kind":"a","args":[{"type":"any"}],"returns":"text","nullable":true} {"name":"group_concat","kind":"a","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} {"name":"hex","args":[{"type":"any"}],"returns":"text"} -{"name":"highlight","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} {"name":"if","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} {"name":"ifnull","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} {"name":"iif","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} @@ -100,7 +98,6 @@ {"name":"ltrim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} {"name":"max","kind":"a","args":[{"type":"any"}],"returns":"any","nullable":true} {"name":"max","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"median","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} {"name":"min","kind":"a","args":[{"type":"any"}],"returns":"any","nullable":true} {"name":"min","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} {"name":"mod","args":[{"type":"any"},{"type":"any"}],"returns":"real"} @@ -109,9 +106,6 @@ {"name":"nullif","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} {"name":"octet_length","args":[{"type":"any"}],"returns":"integer","nullable":true} {"name":"percent_rank","kind":"w","returns":"real"} -{"name":"percentile","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"real","nullable":true} -{"name":"percentile_cont","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"real","nullable":true} -{"name":"percentile_disc","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"any","nullable":true} {"name":"pi","returns":"real"} {"name":"pow","args":[{"type":"any"},{"type":"any"}],"returns":"real"} {"name":"power","args":[{"type":"any"},{"type":"any"}],"returns":"real"} @@ -130,10 +124,8 @@ {"name":"sign","args":[{"type":"any"}],"returns":"integer","nullable":true} {"name":"sin","args":[{"type":"any"}],"returns":"real"} {"name":"sinh","args":[{"type":"any"}],"returns":"real"} -{"name":"snippet","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} {"name":"sqlite_compileoption_get","args":[{"type":"integer"}],"returns":"text","nullable":true} {"name":"sqlite_compileoption_used","args":[{"type":"text"}],"returns":"integer"} -{"name":"sqlite_offset","args":[{"type":"any"}],"returns":"integer","nullable":true} {"name":"sqlite_source_id","returns":"text"} {"name":"sqlite_version","returns":"text"} {"name":"sqrt","args":[{"type":"any"}],"returns":"real"} diff --git a/internal/engine/sqlite/extension.go b/internal/engine/sqlite/extension.go new file mode 100644 index 0000000000..44265489a9 --- /dev/null +++ b/internal/engine/sqlite/extension.go @@ -0,0 +1,26 @@ +package sqlite + +import ( + "github.com/sqlc-dev/sqlc/internal/core/seed" + "github.com/sqlc-dev/sqlc/internal/sql/catalog" +) + +// loadExtension returns the functions a compile option adds, read from the +// option's directory under the dialect's extensions/. The name is either the +// directory's — enable_fts5 — or a virtual table module the dialect maps to +// one, since a schema that says CREATE VIRTUAL TABLE ... USING fts5 has said +// which build of SQLite it runs on. An option the dialect has no data for +// adds nothing. +func loadExtension(name string) *catalog.Schema { + dir, ok := seed.ExtensionDir(dialectFS, "dialect", name) + if !ok { + return nil + } + funcs, err := seed.Functions(dialectFS, dir) + if err != nil { + // The list is embedded in the binary: a failure here means sqlc was + // built from a broken tree, which no caller can do anything about. + panic(err) + } + return &catalog.Schema{Name: "main", Funcs: funcs} +} diff --git a/internal/engine/sqlite/stdlib.go b/internal/engine/sqlite/stdlib.go index 0a4c4960e4..210aa29961 100644 --- a/internal/engine/sqlite/stdlib.go +++ b/internal/engine/sqlite/stdlib.go @@ -5,8 +5,9 @@ import ( ) // defaultSchema is SQLite's standard library, read from the dialect -// directory's functions.jsonl, which internal/goldeneye generates from the -// sqlite3 shell's pragma_function_list. +// directory's functions.jsonl, which internal/goldeneye generates from a +// default build of SQLite. The functions further compile options add live +// under the dialect's extensions/, one directory per option. func defaultSchema(name string) *catalog.Schema { return &catalog.Schema{Name: name, Funcs: stdlib()} } diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index db6976a205..82e20c0b5b 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -15,7 +15,7 @@ reads the files: the files are the contract. Run it from this directory: ```bash go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary once -go run ./cmd/goldeneye install sqlite # download the pinned sqlite3 shell once +go run ./cmd/goldeneye install sqlite # build the pinned sqlite3 shells once; needs a C compiler go run ./cmd/goldeneye check # check every engine whose database is available go run ./cmd/goldeneye check postgresql # check one engine go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database @@ -56,19 +56,25 @@ the hand-written files alone, and the checks do not look at them. ClickHouse describes its functions no further than their names, so `functions.jsonl` is hand-written. - **`sqlite`** needs no server either: `functions.jsonl` comes from - `pragma_function_list` of the `sqlite3` shell sqlite.org publishes, run - against an in-memory database. SQLite describes its functions as far as - their names, their kinds and the number of arguments each overload takes, - and no further — it types values, not functions — so what each returns and - what its arguments hold comes from the table in `sqlite/signatures.go`, - and a built-in function the shell reports that the table does not know - fails the run rather than being guessed at. The shell is downloaded once - per pinned release by `install` into the user cache directory, or supplied - through the `SQLITE3` environment variable; the pinned release, which is - the one the main module's driver embeds, and the SHA3-256 of each - platform's download live in `sqlite/install.go`. SQLite has no catalog of - types or operators, so `types.jsonl` and `operators.jsonl` are - hand-written. + `pragma_function_list` of a `sqlite3` shell run against an in-memory + database. Which functions a SQLite has is decided when it is compiled, so + `install` downloads the pinned release's amalgamation, checked against the + SHA3-256 the download page lists, and compiles the shell from it with the + compiler `CC` names, or `cc` — once with the options sqlite.org's own + configure turns on by default, which gives `functions.jsonl`, and once + more per option in `sqlite/install.go`'s extension list, each of which + gets a directory under `extensions/` holding the functions its build adds + over the default one, the way each PostgreSQL contrib extension holds what + `CREATE EXTENSION` adds; a schema that says `CREATE VIRTUAL TABLE ... USING + fts5` loads the option's directory, through the `modules` map in the + hand-written `dialect.json`. SQLite describes its functions as far as their + names, their kinds and the number of arguments each overload takes, and no + further — it types values, not functions — so what each returns and what + its arguments hold comes from the table in `sqlite/signatures.go`, and a + function a shell reports that the table does not know fails the run rather + than being guessed at. The pinned release is the one the main module's + driver embeds. SQLite has no catalog of types or operators, so + `types.jsonl` and `operators.jsonl` are hand-written. ## Layout diff --git a/internal/goldeneye/cmd/goldeneye/main.go b/internal/goldeneye/cmd/goldeneye/main.go index 438f2c36bf..ba6ff1c229 100644 --- a/internal/goldeneye/cmd/goldeneye/main.go +++ b/internal/goldeneye/cmd/goldeneye/main.go @@ -5,7 +5,7 @@ // Usage, from internal/goldeneye: // // go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary -// go run ./cmd/goldeneye install sqlite # download the pinned sqlite3 shell +// go run ./cmd/goldeneye install sqlite # build the pinned sqlite3 shells from source // go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database // go run ./cmd/goldeneye check [engine] # compare the committed files with the database // @@ -39,7 +39,8 @@ func main() { const usage = `usage: goldeneye install clickhouse|sqlite [-version V] - download the pinned binary for an engine into the user cache directory + put the pinned release of an engine into the user cache directory: clickhouse is + downloaded, sqlite is built from the downloaded amalgamation with cc or $CC goldeneye generate [engine] rewrite the generated dialect files from the database, for every available engine or one goldeneye check [engine] @@ -66,8 +67,8 @@ var engines = []engine{ {sqlite.Engine, sqlite.Locate, sqlite.Version, sqlite.Generate}, } -// installer downloads the binary an engine is read through, for the engines -// that need no server and whose release is pinned in their package. +// installer puts the binary an engine is read through in place, for the +// engines that need no server and whose release is pinned in their package. type installer struct { defaultVersion string install func(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) diff --git a/internal/goldeneye/sqlite/install.go b/internal/goldeneye/sqlite/install.go index 3bc14c000f..0a0ed1d674 100644 --- a/internal/goldeneye/sqlite/install.go +++ b/internal/goldeneye/sqlite/install.go @@ -5,12 +5,16 @@ import ( "context" "crypto/sha3" "encoding/hex" + "errors" "fmt" "io" "net/http" "os" + "os/exec" "path/filepath" "runtime" + "strings" + "sync" ) // DefaultVersion is the SQLite release the dialect is generated from. It is @@ -18,16 +22,38 @@ import ( // the functions the dialect knows are the ones the tests run against. // Bumping it is a deliberate change: releases add functions and overloads, // so regenerate and review the dialect after changing it, and add the new -// release's downloads to the table below. +// release's amalgamation to the table below. const DefaultVersion = "3.53.4" -// asset is one downloadable build of the SQLite command-line tools: a zip -// published on sqlite.org holding the sqlite3 shell (sqlite3.exe on -// Windows) next to sqldiff, sqlite3_analyzer and sqlite3_rsync. +// defaultOptions are the compile options the base dialect is built with: +// the ones sqlite.org's own configure turns on by default, and so the ones +// a stock sqlite3 has. JSON is part of the library unless omitted, so only +// the math functions need naming. +var defaultOptions = []string{"SQLITE_ENABLE_MATH_FUNCTIONS"} + +// extensions are the compile options that each become a directory under +// the dialect's extensions/, holding the functions a build with the option +// adds over the default build — the way each PostgreSQL contrib extension +// holds what CREATE EXTENSION adds. Each is named after its option as +// pragma compile_options spells it, in lower case, and lists the options +// its build needs: GEOPOLY lives inside the RTREE module, so enabling it +// alone adds nothing. Options that add virtual tables but no functions, +// such as SESSION and DBSTAT, are not listed, since the dialect has nothing +// to say about them, and ENABLE_FTS4 is the same module as ENABLE_FTS3. +var extensions = []build{ + {"soundex", []string{"SQLITE_SOUNDEX"}}, + {"enable_fts3", []string{"SQLITE_ENABLE_FTS3"}}, + {"enable_fts5", []string{"SQLITE_ENABLE_FTS5"}}, + {"enable_geopoly", []string{"SQLITE_ENABLE_RTREE", "SQLITE_ENABLE_GEOPOLY"}}, + {"enable_offset_sql_func", []string{"SQLITE_ENABLE_OFFSET_SQL_FUNC"}}, + {"enable_percentile", []string{"SQLITE_ENABLE_PERCENTILE"}}, + {"enable_rtree", []string{"SQLITE_ENABLE_RTREE"}}, +} + +// asset is one downloadable amalgamation of SQLite: the zip published on +// sqlite.org holding sqlite3.c, sqlite3.h and the shell's shell.c. type asset struct { Version string - OS string - Arch string // Path is the download's address under https://sqlite.org/, the // release year included, as the download page's index lists it. Path string @@ -35,163 +61,189 @@ type asset struct { SHA3 string } -// assets lists every build Install knows how to fetch, with the SHA3-256 of -// the download. A version that is not in this table cannot be installed: -// verifying the download is the point of the table. -// -// The checksums are the ones in the index at the foot of -// https://sqlite.org/download.html. sqlite.org builds the tools for x64 -// Linux, both macOS architectures and both Windows architectures; there is -// no Linux arm64 build to list. +// assets lists every amalgamation Install knows how to fetch, with the +// SHA3-256 of the download. A version that is not in this table cannot be +// installed: verifying the download is the point of the table. The +// checksums are the ones in the index at the foot of +// https://sqlite.org/download.html. var assets = []asset{ - {"3.53.4", "linux", "amd64", "2026/sqlite-tools-linux-x64-3530400.zip", "6eeb57e8f2aef7687f9f016a980992cf2799c8c07a87c5e21495530f91915047"}, - {"3.53.4", "darwin", "amd64", "2026/sqlite-tools-osx-x64-3530400.zip", "3353bb4e5ac54f85c5b82012d30476d20539a9495abdd11a1707df578cff2d7e"}, - {"3.53.4", "darwin", "arm64", "2026/sqlite-tools-osx-arm64-3530400.zip", "58d53e0eb69c17cabebed2754bf399e4d44939be42dcf194769c00078bfd776d"}, - {"3.53.4", "windows", "amd64", "2026/sqlite-tools-win-x64-3530400.zip", "88b4659fe747896b853af10157316b4ade143553efb89c1c8ca7423a278dcc8b"}, - {"3.53.4", "windows", "arm64", "2026/sqlite-tools-win-arm64-3530400.zip", "0c99da3702b2517c1d738207db7e945e5c55be7748141a192a1c8f3b4455c44b"}, + {"3.53.4", "2026/sqlite-amalgamation-3530400.zip", "628a44cfe82c66aed1ccbbe85a562d2e33ebe64b3288981ed76285612227934e"}, } -// releaseAsset finds the build for a platform in the table. -func releaseAsset(version, goos, goarch string) (asset, error) { - for _, a := range assets { - if a.Version == version && a.OS == goos && a.Arch == goarch { - return a, nil - } - } +// sources are the files taken out of the amalgamation. +var sources = []string{"sqlite3.c", "sqlite3.h", "shell.c"} + +func releaseAsset(version string) (asset, error) { for _, a := range assets { if a.Version == version { - return asset{}, fmt.Errorf("no SQLite %s build is listed for %s/%s", version, goos, goarch) + return a, nil } } - return asset{}, fmt.Errorf("SQLite %s is not in the asset table; add its downloads and checksums to install.go", version) + return asset{}, fmt.Errorf("SQLite %s is not in the asset table; add its amalgamation and checksum to install.go", version) } -// url is the asset's download address. func (a asset) url() string { return "https://sqlite.org/" + a.Path } -// binaryName is what the shell is called inside the zip and in the cache. -func binaryName(goos string) string { - if goos == "windows" { - return "sqlite3.exe" - } - return "sqlite3" -} - -// cachedBinary is where Install puts the shell for a version. -func cachedBinary(version, goos string) (string, error) { +// cacheDir is where Install puts a version: the sources under src/, and one +// shell per build under default/ and under each option's extension name. +func cacheDir(version string) (string, error) { dir, err := os.UserCacheDir() if err != nil { return "", err } - return filepath.Join(dir, "sqlc-sqlite", version, binaryName(goos)), nil + return filepath.Join(dir, "sqlc-sqlite", version), nil +} + +// build is one compiled shell: its name, which is the directory it is +// under in the version's cache directory and, for an extension, under the +// dialect's extensions/, and the options it is built with beyond the +// default ones. +type build struct { + name string + options []string +} + +// builds lists the default build first, then one per extension. +func builds() []build { + return append([]build{{"default", nil}}, extensions...) +} + +// flags are every option a build is compiled with. +func (b build) flags() []string { + return append(append([]string{}, defaultOptions...), b.options...) +} + +func (b build) binary(dir string) string { + return filepath.Join(dir, b.name, "sqlite3") } -// Locate finds a sqlite3 shell: the SQLITE3 environment variable wins, then -// the cached copy of DefaultVersion. +// Locate finds the directory of shells Install made for DefaultVersion. func Locate() (string, error) { - if path := os.Getenv("SQLITE3"); path != "" { - return path, nil - } - path, err := cachedBinary(DefaultVersion, runtime.GOOS) + dir, err := cacheDir(DefaultVersion) if err != nil { return "", err } - if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("sqlite %s is not installed: run `go run ./cmd/goldeneye install sqlite` in internal/goldeneye, or set SQLITE3 to a sqlite3 shell", DefaultVersion) + for _, b := range builds() { + if _, err := os.Stat(b.binary(dir)); err != nil { + return "", fmt.Errorf("sqlite %s is not built with %s: run `go run ./cmd/goldeneye install sqlite` in internal/goldeneye", DefaultVersion, strings.Join(b.flags(), " ")) + } } - return path, nil + return dir, nil } -// Install downloads the sqlite3 shell for a version into the cache and -// returns its path. It is a no-op when the version is already cached. The -// zip is checked against the table's SHA3-256 before the shell is taken -// out of it. +// Install downloads the amalgamation for a version into the cache and +// compiles the shell from it once per build, returning the directory the +// shells are under. Builds already there are kept, so adding an option +// compiles only its shell. The download is checked against the table's +// SHA3-256 before it is unpacked. Compiling takes the compiler CC names, or +// cc, and a few seconds per build without optimisation, which a shell that +// only reads catalogs does not need. +// +// goos and goarch are what every installer is handed; the sources build the +// same everywhere a C compiler is, but no default compiler or link line is +// known for Windows. func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) { - dest, err := cachedBinary(version, goos) + if goos == "windows" { + return "", errors.New("building SQLite from source is not supported on Windows") + } + dir, err := cacheDir(version) if err != nil { return "", err } - if _, err := os.Stat(dest); err == nil { - return dest, nil + var missing []build + for _, b := range builds() { + if _, err := os.Stat(b.binary(dir)); err != nil { + missing = append(missing, b) + } } - a, err := releaseAsset(version, goos, goarch) - if err != nil { + if len(missing) == 0 { + return dir, nil + } + if err := download(ctx, version, dir, progress); err != nil { return "", err } - if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + if err := compile(ctx, dir, missing, progress); err != nil { return "", err } + return dir, nil +} + +// download fetches the amalgamation and unpacks the sources into src/, +// unless they are already there. +func download(ctx context.Context, version, dir string, progress io.Writer) error { + src := filepath.Join(dir, "src") + have := true + for _, name := range sources { + if _, err := os.Stat(filepath.Join(src, name)); err != nil { + have = false + } + } + if have { + return nil + } + a, err := releaseAsset(version) + if err != nil { + return err + } + if err := os.MkdirAll(src, 0o755); err != nil { + return err + } url := a.url() fmt.Fprintf(progress, "downloading %s\n", url) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return "", err + return err } resp, err := http.DefaultClient.Do(req) if err != nil { - return "", err + return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("downloading %s: %s", url, resp.Status) + return fmt.Errorf("downloading %s: %s", url, resp.Status) } // The zip has to land on disk before it can be read — its directory is - // at the end — so download it whole, next to the destination, and hash - // every byte on the way. - archive, err := os.CreateTemp(filepath.Dir(dest), "sqlite-tools-*.partial") + // at the end — so download it whole and hash every byte on the way. + archive, err := os.CreateTemp(dir, "sqlite-amalgamation-*.partial") if err != nil { - return "", err + return err } defer os.Remove(archive.Name()) sum := sha3.New256() if _, err := io.Copy(io.MultiWriter(archive, sum), resp.Body); err != nil { archive.Close() - return "", err + return err } if err := archive.Close(); err != nil { - return "", err + return err } if got := hex.EncodeToString(sum.Sum(nil)); got != a.SHA3 { - return "", fmt.Errorf("downloading %s: SHA3-256 mismatch: got %s, want %s", url, got, a.SHA3) + return fmt.Errorf("downloading %s: SHA3-256 mismatch: got %s, want %s", url, got, a.SHA3) } - // Write the shell next to the destination and rename so a partial - // extraction never masquerades as an installed binary. - tmp, err := os.CreateTemp(filepath.Dir(dest), "sqlite3-*.partial") + zr, err := zip.OpenReader(archive.Name()) if err != nil { - return "", err - } - defer os.Remove(tmp.Name()) - if err := extractBinary(archive.Name(), binaryName(goos), tmp); err != nil { - tmp.Close() - return "", fmt.Errorf("downloading %s: %w", url, err) - } - if err := tmp.Close(); err != nil { - return "", err - } - if err := os.Chmod(tmp.Name(), 0o755); err != nil { - return "", err + return err } - if err := os.Rename(tmp.Name(), dest); err != nil { - return "", err + defer zr.Close() + for _, name := range sources { + if err := extract(zr, name, filepath.Join(src, name)); err != nil { + return fmt.Errorf("downloading %s: %w", url, err) + } } - return dest, nil + return nil } -// extractBinary copies the named file out of a zip. The tools zips are -// flat, so the name is the whole path. -func extractBinary(archive, name string, dst io.Writer) error { - zr, err := zip.OpenReader(archive) - if err != nil { - return err - } - defer zr.Close() +// extract copies the named file out of the zip. The amalgamation's files +// sit in one directory named after the release, so the name is matched on +// its base. +func extract(zr *zip.ReadCloser, name, dest string) error { for _, f := range zr.File { - if f.Name != name { + if filepath.Base(f.Name) != name || f.FileInfo().IsDir() { continue } rc, err := f.Open() @@ -199,8 +251,68 @@ func extractBinary(archive, name string, dst io.Writer) error { return err } defer rc.Close() - _, err = io.Copy(dst, rc) - return err + tmp, err := os.CreateTemp(filepath.Dir(dest), name+"-*.partial") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := io.Copy(tmp, rc); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), dest) } return fmt.Errorf("zip does not contain %s", name) } + +// compile builds the shells, as many at a time as there are CPUs. +func compile(ctx context.Context, dir string, todo []build, progress io.Writer) error { + cc := os.Getenv("CC") + if cc == "" { + cc = "cc" + } + if _, err := exec.LookPath(cc); err != nil { + return fmt.Errorf("no C compiler found: put cc on PATH or set CC to one") + } + var wg sync.WaitGroup + slots := make(chan struct{}, max(1, runtime.NumCPU())) + errs := make([]error, len(todo)) + for i, b := range todo { + wg.Add(1) + go func() { + defer wg.Done() + slots <- struct{}{} + defer func() { <-slots }() + errs[i] = b.compile(ctx, cc, dir, progress) + }() + } + wg.Wait() + return errors.Join(errs...) +} + +// compile builds one shell from the sources in src/, writing it beside its +// destination and renaming so a failed build never masquerades as one. +func (b build) compile(ctx context.Context, cc, dir string, progress io.Writer) error { + dest := b.binary(dir) + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + tmp := dest + ".partial" + defer os.Remove(tmp) + args := []string{"-O0"} + for _, opt := range b.flags() { + args = append(args, "-D"+opt) + } + src := filepath.Join(dir, "src") + args = append(args, filepath.Join(src, "shell.c"), filepath.Join(src, "sqlite3.c"), "-o", tmp, "-lm", "-ldl", "-lpthread") + fmt.Fprintf(progress, "building %s: %s %s\n", b.name, cc, strings.Join(args, " ")) + cmd := exec.CommandContext(ctx, cc, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("building %s: %w\n%s", b.name, err, strings.TrimSpace(string(out))) + } + return os.Rename(tmp, dest) +} diff --git a/internal/goldeneye/sqlite/signatures.go b/internal/goldeneye/sqlite/signatures.go index d3da84e7c0..d4f6cddee9 100644 --- a/internal/goldeneye/sqlite/signatures.go +++ b/internal/goldeneye/sqlite/signatures.go @@ -60,14 +60,20 @@ var omitted = map[string]bool{ // SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION's stand-in for any function the // shell does not know, so that EXPLAIN works on queries that use one. "unknown": true, + // FTS3's and FTS5's ways of passing pointers to their virtual tables, + // not functions a query calls. + "fts3_tokenizer": true, + "fts5": true, + // A debugging aid of GEOPOLY's. + "geopoly_debug": true, } -// signatures covers every function the shell builds in, by the SQLite -// documentation of each — lang_corefunc, lang_mathfunc, lang_datefunc, -// lang_aggfunc, windowfunctions and json1 — and the auxiliary functions of -// FTS5, which the shell bundles as an extension. A function marked -// Nullable can return NULL for arguments that are not: an aggregate over -// no rows, a lookup that finds nothing, an input that does not parse. +// signatures covers every function of the default build and of each +// extension's, by the SQLite documentation of each — lang_corefunc, +// lang_mathfunc, lang_datefunc, lang_aggfunc, windowfunctions, json1, +// fts3, fts5, rtree and geopoly. A function marked Nullable can return NULL +// for arguments that are not: an aggregate over no rows, a lookup that +// finds nothing, an input that does not parse. // // SQLite's values carry their own types, so a function's result type is // what it typically produces. abs of an integer is an integer, but abs @@ -85,7 +91,7 @@ var signatures = map[string]signature{ "sum": {Returns: "real", Nullable: true}, "total": {Returns: "real"}, - // Percentiles, built in by SQLITE_ENABLE_PERCENTILE. + // Percentiles, added by SQLITE_ENABLE_PERCENTILE. "median": {Returns: "real", Nullable: true}, "percentile": {Args: []string{"any", "real"}, Returns: "real", Nullable: true}, "percentile_cont": {Args: []string{"any", "real"}, Returns: "real", Nullable: true}, @@ -136,7 +142,6 @@ var signatures = map[string]signature{ "sign": {Returns: "integer", Nullable: true}, "sqlite_compileoption_get": {Args: []string{"integer"}, Returns: "text", Nullable: true}, "sqlite_compileoption_used": {Args: []string{"text"}, Returns: "integer"}, - "sqlite_offset": {Returns: "integer", Nullable: true}, "sqlite_source_id": {Returns: "text"}, "sqlite_version": {Returns: "text"}, "substr": {Args: []string{"any", "integer", "integer"}, Returns: "text"}, @@ -235,9 +240,45 @@ var signatures = map[string]signature{ "jsonb_replace": {Args: []string{"any"}, Returns: "blob"}, "jsonb_set": {Args: []string{"any"}, Returns: "blob"}, - // FTS5's auxiliary functions, registered by the extension rather than - // built in, and the only ones of its functions a query calls. - "bm25": {Args: []string{"text"}, Variadic: "real", Returns: "real"}, - "highlight": {Args: []string{"text", "integer", "text", "text"}, Returns: "text"}, - "snippet": {Args: []string{"text", "integer", "text", "text", "text", "integer"}, Returns: "text"}, + // Added by SQLITE_SOUNDEX. + "soundex": {Args: []string{"text"}, Returns: "text"}, + + // Added by SQLITE_ENABLE_OFFSET_SQL_FUNC. + "sqlite_offset": {Returns: "integer", Nullable: true}, + + // FTS3's auxiliary functions, whose first argument names the table. + "matchinfo": {Args: []string{"any", "text"}, Returns: "blob"}, + "offsets": {Args: []string{"any"}, Returns: "text"}, + "optimize": {Args: []string{"any"}, Returns: "text"}, + + // FTS5's auxiliary and locale functions. snippet is FTS3's too, with the + // same result. + "bm25": {Args: []string{"text"}, Variadic: "real", Returns: "real"}, + "fts5_get_locale": {Args: []string{"any", "any"}, Returns: "text", Nullable: true}, + "fts5_insttoken": {Args: []string{"text"}, Returns: "text"}, + "fts5_locale": {Args: []string{"text", "text"}, Returns: "text"}, + "fts5_source_id": {Returns: "text"}, + "highlight": {Args: []string{"text", "integer", "text", "text"}, Returns: "text"}, + "snippet": {Args: []string{"text", "integer", "text", "text", "text", "integer"}, Returns: "text"}, + + // R*Tree's functions for inspecting an index's nodes. + "rtreecheck": {Args: []string{"text", "text"}, Returns: "text"}, + "rtreedepth": {Args: []string{"blob"}, Returns: "integer"}, + "rtreenode": {Args: []string{"integer", "blob"}, Returns: "text"}, + + // GEOPOLY's functions. A polygon argument is GeoJSON text or the binary + // form, so it is "any"; the functions that return a polygon return the + // binary form. + "geopoly_area": {Returns: "real", Nullable: true}, + "geopoly_bbox": {Returns: "blob", Nullable: true}, + "geopoly_blob": {Returns: "blob", Nullable: true}, + "geopoly_ccw": {Returns: "blob", Nullable: true}, + "geopoly_contains_point": {Args: []string{"any", "real", "real"}, Returns: "integer", Nullable: true}, + "geopoly_group_bbox": {Returns: "blob", Nullable: true}, + "geopoly_json": {Returns: "text", Nullable: true}, + "geopoly_overlap": {Returns: "integer", Nullable: true}, + "geopoly_regular": {Args: []string{"real", "real", "real", "integer"}, Returns: "blob"}, + "geopoly_svg": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, + "geopoly_within": {Returns: "integer", Nullable: true}, + "geopoly_xform": {Args: []string{"any", "real", "real", "real", "real", "real", "real"}, Returns: "blob", Nullable: true}, } diff --git a/internal/goldeneye/sqlite/sqlite.go b/internal/goldeneye/sqlite/sqlite.go index 56a1953bed..02c5362a95 100644 --- a/internal/goldeneye/sqlite/sqlite.go +++ b/internal/goldeneye/sqlite/sqlite.go @@ -1,6 +1,7 @@ // Package sqlite generates the SQLite dialect seed under -// internal/engine/sqlite/dialect from the sqlite3 shell sqlite.org -// publishes, run against an in-memory database that needs no server. +// internal/engine/sqlite/dialect from sqlite3 shells built from the +// amalgamation sqlite.org publishes, run against in-memory databases that +// need no server. // // SQLite describes its functions as far as their names, their kinds and the // number of arguments each takes — pragma_function_list — and no further: @@ -9,14 +10,20 @@ // is therefore built from both sides. The shell says which functions exist, // how many arguments each overload takes and whether it aggregates, and the // signatures table in this package says what each returns and what its -// arguments are meant to hold. A built-in function the shell reports that -// the table does not know fails generation rather than being guessed at, -// and so does a table entry the shell does not report. SQLite has no -// catalog of types or operators, so types.jsonl and operators.jsonl are -// hand-written. +// arguments are meant to hold. A function the shell reports that the table +// does not know fails generation rather than being guessed at, and so does +// a table entry no shell reports. // -// The shell is downloaded once per pinned version by Install, or supplied -// through the SQLITE3 environment variable. +// Which functions a SQLite has is decided when it is compiled, so the +// dialect treats compile options the way the PostgreSQL dialect treats +// contrib extensions. functions.jsonl is what a build with the default +// options has, and each further option gets a directory under extensions/ +// holding the functions a build with that option adds — found the way +// CREATE EXTENSION's additions are, by comparing the catalog with and +// without. SQLite has no catalog of types or operators, so types.jsonl and +// operators.jsonl are hand-written. +// +// The shells are built once per pinned version by Install. package sqlite import ( @@ -26,7 +33,10 @@ import ( "errors" "fmt" "io" + "maps" "os/exec" + "path" + "slices" "sort" "strconv" "strings" @@ -55,9 +65,14 @@ type functionRow struct { NArg int `json:"narg"` } -// Version reports the release a shell is. -func Version(ctx context.Context, binary string) (string, error) { - out, err := exec.CommandContext(ctx, binary, "--version").Output() +// key tells one overload from another. +func (r functionRow) key() string { + return r.Name + "\x00" + strconv.Itoa(r.NArg) +} + +// Version reports the release the default shell is. +func Version(ctx context.Context, dir string) (string, error) { + out, err := exec.CommandContext(ctx, builds()[0].binary(dir), "--version").Output() if err != nil { return "", fmt.Errorf("sqlite3 --version: %w", err) } @@ -147,35 +162,86 @@ func isWindowFunction(ctx context.Context, binary string, row functionRow) (bool return false, fmt.Errorf("sqlite3: %w", err) } -// readFunctions turns the shell's list into the dialect's, one record per -// overload of every function the signatures table knows. A built-in -// function without a signature is an error; a function without one that -// the shell or a bundled extension registered is not the dialect's -// business. -func readFunctions(ctx context.Context, binary string, rows []functionRow) ([]dialect.Function, error) { +// shell is one built sqlite3 and the functions it reports. +type shell struct { + build build + binary string + rows []functionRow +} + +// readShell lists a build's functions, after checking that the shell was +// built with the options the build says — a cached shell from before the +// option lists changed would otherwise describe the wrong dialect. +func readShell(ctx context.Context, dir string, b build) (*shell, error) { + s := &shell{build: b, binary: b.binary(dir)} + var used []struct { + Option string `json:"option"` + Used int `json:"used"` + } + known := map[string]bool{} + for _, b := range builds() { + for _, opt := range b.flags() { + known[opt] = true + } + } + var clauses []string + for _, opt := range slices.Sorted(maps.Keys(known)) { + clauses = append(clauses, fmt.Sprintf("SELECT '%s' AS option, sqlite_compileoption_used('%s') AS used", opt, opt)) + } + if err := query(ctx, s.binary, strings.Join(clauses, " UNION ALL "), &used); err != nil { + return nil, err + } + for _, u := range used { + want := 0 + if slices.Contains(b.flags(), u.Option) { + want = 1 + } + if u.Used != want { + return nil, fmt.Errorf("sqlite: the %s shell was not built with the options it should have been (%s is %d): remove %s and run `go run ./cmd/goldeneye install sqlite` again", b.name, u.Option, u.Used, dir) + } + } + if err := query(ctx, s.binary, functionList, &s.rows); err != nil { + return nil, err + } + return s, nil +} + +// generator accumulates the functions of every build, and remembers which +// names were reported so that the signatures table can be checked against +// the shells at the end. +type generator struct { + ctx context.Context + reported map[string]bool +} + +// functions turns rows into records, one per overload. Every row has to +// have a signature unless omitted; lenient says a row without one may be +// skipped instead when it is not built in, which is how the shell's own +// functions — edit, sha3, the bundled extensions — are kept out of the +// default build's list. A comparison with the default build has already +// removed them from an option's rows, so there nothing is skipped. +func (g *generator) functions(s *shell, lenient bool) ([]dialect.Function, error) { var funcs []dialect.Function seen := map[string]bool{} - reported := map[string]bool{} var missing []string - for _, row := range rows { - reported[row.Name] = true - if omitted[row.Name] { + for _, row := range s.rows { + g.reported[row.Name] = true + if omitted[row.Name] || seen[row.key()] { continue } + seen[row.key()] = true sig, ok := signatures[row.Name] if !ok { - if row.Builtin != 0 && !seen[row.Name] { + if lenient && row.Builtin == 0 { + continue + } + if !seen[row.Name] { seen[row.Name] = true missing = append(missing, row.Name) } continue } - key := row.Name + "\x00" + strconv.Itoa(row.NArg) - if seen[key] { - continue - } - seen[key] = true - kind, err := functionKind(ctx, binary, row) + kind, err := functionKind(g.ctx, s.binary, row) if err != nil { return nil, err } @@ -188,34 +254,68 @@ func readFunctions(ctx context.Context, binary string, rows []functionRow) ([]di }) } if len(missing) > 0 { - return nil, fmt.Errorf("sqlite: no signature for built-in function(s) %s: add them to signatures.go", strings.Join(missing, ", ")) + return nil, fmt.Errorf("sqlite: no signature for function(s) %s of the %s build: add them to signatures.go", strings.Join(missing, ", "), s.build.name) } - var stale []string - for name := range signatures { - if !reported[name] { - stale = append(stale, name) - } + return funcs, nil +} + +// added returns the rows of an option's shell that the default shell does +// not have: what the option adds. +func added(opt, base *shell) *shell { + have := map[string]bool{} + for _, row := range base.rows { + have[row.key()] = true } - if len(stale) > 0 { - sort.Strings(stale) - return nil, fmt.Errorf("sqlite: signatures.go lists function(s) the shell does not report: %s", strings.Join(stale, ", ")) + diff := &shell{build: opt.build, binary: opt.binary} + for _, row := range opt.rows { + if !have[row.key()] { + diff.rows = append(diff.rows, row) + } } - return funcs, nil + return diff } -// Generate reads the dialect from the shell. -func Generate(ctx context.Context, binary string) (dialect.Files, error) { - var rows []functionRow - if err := query(ctx, binary, functionList, &rows); err != nil { +// Generate reads the dialect from the shells under dir. +func Generate(ctx context.Context, dir string) (dialect.Files, error) { + g := &generator{ctx: ctx, reported: map[string]bool{}} + all := builds() + base, err := readShell(ctx, dir, all[0]) + if err != nil { return nil, err } - funcs, err := readFunctions(ctx, binary, rows) + funcs, err := g.functions(base, true) if err != nil { return nil, err } - functions, err := dialect.JSONL(funcs) - if err != nil { + files := dialect.Files{} + if files[dialect.FunctionsFile], err = dialect.JSONL(funcs); err != nil { return nil, err } - return dialect.Files{dialect.FunctionsFile: functions}, nil + for _, b := range all[1:] { + s, err := readShell(ctx, dir, b) + if err != nil { + return nil, err + } + funcs, err := g.functions(added(s, base), false) + if err != nil { + return nil, err + } + if len(funcs) == 0 { + return nil, fmt.Errorf("sqlite: %s adds no functions over the default build; drop it from the extensions in install.go", strings.Join(b.options, " ")) + } + if files[path.Join(dialect.ExtensionsDir, b.name, dialect.FunctionsFile)], err = dialect.JSONL(funcs); err != nil { + return nil, err + } + } + var stale []string + for name := range signatures { + if !g.reported[name] { + stale = append(stale, name) + } + } + if len(stale) > 0 { + sort.Strings(stale) + return nil, fmt.Errorf("sqlite: signatures.go lists function(s) no shell reports: %s", strings.Join(stale, ", ")) + } + return files, nil } diff --git a/internal/sql/catalog/extension.go b/internal/sql/catalog/extension.go index fdb717f2d2..268bf01927 100644 --- a/internal/sql/catalog/extension.go +++ b/internal/sql/catalog/extension.go @@ -9,16 +9,28 @@ func (c *Catalog) createExtension(stmt *ast.CreateExtensionStmt) error { return nil } // TODO: Implement IF NOT EXISTS - if _, exists := c.Extensions[*stmt.Extname]; exists { + return c.loadExtension(*stmt.Extname) +} + +// loadExtension adds what the engine knows of the named extension — or of +// the virtual table module that stands for one — to the default schema, +// once. An engine without extension data, or without data for this one, +// adds nothing. +func (c *Catalog) loadExtension(name string) error { + if _, exists := c.Extensions[name]; exists { return nil } if c.LoadExtension == nil { return nil } - ext := c.LoadExtension(*stmt.Extname) + ext := c.LoadExtension(name) if ext == nil { return nil } + if c.Extensions == nil { + c.Extensions = map[string]struct{}{} + } + c.Extensions[name] = struct{}{} s, err := c.getSchema(c.DefaultSchema) if err != nil { return err diff --git a/internal/sql/catalog/table.go b/internal/sql/catalog/table.go index ec2a122a96..a91e399a7e 100644 --- a/internal/sql/catalog/table.go +++ b/internal/sql/catalog/table.go @@ -248,6 +248,13 @@ func (c *Catalog) alterTableSetSchema(stmt *ast.AlterTableSetSchemaStmt) error { } func (c *Catalog) createTable(stmt *ast.CreateTableStmt) error { + // A virtual table's module is the engine's word for the extension it + // needs, the way CREATE EXTENSION is PostgreSQL's. + if stmt.Using != "" { + if err := c.loadExtension(stmt.Using); err != nil { + return err + } + } ns := stmt.Name.Schema if ns == "" { ns = c.DefaultSchema From 1c22d510237b4fbea5431e89112b43ecac777f03 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 17:07:10 +0000 Subject: [PATCH 3/3] goldeneye: read SQLite function signatures from the amalgamation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQLite records nothing about what its functions return or take, but its source does, in its way: every function is registered with the C functions that implement it, and those set their result through sqlite3_result_* and read their arguments through sqlite3_value_*. So instead of a hand-written table of signatures, the generator now reads the amalgamation the shells are built from. It indexes every function definition, reads the FuncDef macro tables, the sqlite3_create_function calls and the struct tables extensions walk, follows each implementation and the helpers it calls for result calls, and reads each argument's position for value calls. One kind of result is that type; a function that returns one of its arguments is "any"; integer and real together widen to real, text and blob together to text; a JSON function's registration says whether it returns text or JSONB; a function the VDBE implements in bytecode passes an argument through. A function the shell reports that the source does not register, or whose implementation sets no result, fails the run. Whether an aggregate returns NULL over no rows is found by running each over none, in one shell process per build. Which scalar functions return NULL for arguments that are not is what neither the shell nor the source can say — a SQLite function returns NULL as often by setting no result as by saying so — so that remains a short list, with the functions the dialect omits and the one inline function that does not pass an argument through. The regenerated dialect differs where the source knows better: ceil, floor and trunc return real for real input; unixepoch can; format, printf, instr and length are not nullable for arguments that are not, and neither are coalesce, iif and their kin; round's second argument and the math functions' arguments are typed; hex takes a blob. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RYkxvtwH6GgysuWYP87vyj --- .../builtins/sqlite/go/mathfunc.sql.go | 16 +- .../builtins/sqlite/go/scalarfunc.sql.go | 16 +- .../extensions/enable_fts3/functions.jsonl | 2 +- .../extensions/enable_fts5/functions.jsonl | 10 +- .../extensions/enable_geopoly/functions.jsonl | 2 +- .../enable_percentile/functions.jsonl | 8 +- .../engine/sqlite/dialect/functions.jsonl | 146 ++--- internal/goldeneye/README.md | 12 +- internal/goldeneye/sqlite/signatures.go | 292 +++------- internal/goldeneye/sqlite/source.go | 517 ++++++++++++++++++ internal/goldeneye/sqlite/sqlite.go | 160 ++++-- 11 files changed, 812 insertions(+), 369 deletions(-) create mode 100644 internal/goldeneye/sqlite/source.go diff --git a/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go index 30ae799ce5..152d80ad46 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go @@ -90,9 +90,9 @@ const getCeil = `-- name: GetCeil :one SELECT ceil(1.0) ` -func (q *Queries) GetCeil(ctx context.Context) (int64, error) { +func (q *Queries) GetCeil(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getCeil) - var ceil int64 + var ceil float64 err := row.Scan(&ceil) return ceil, err } @@ -101,9 +101,9 @@ const getCeilin = `-- name: GetCeilin :one SELECT ceiling(1.0) ` -func (q *Queries) GetCeilin(ctx context.Context) (int64, error) { +func (q *Queries) GetCeilin(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getCeilin) - var ceiling int64 + var ceiling float64 err := row.Scan(&ceiling) return ceiling, err } @@ -156,9 +156,9 @@ const getFloor = `-- name: GetFloor :one SELECT floor(1.0) ` -func (q *Queries) GetFloor(ctx context.Context) (int64, error) { +func (q *Queries) GetFloor(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getFloor) - var floor int64 + var floor float64 err := row.Scan(&floor) return floor, err } @@ -321,9 +321,9 @@ const getTrunc = `-- name: GetTrunc :one SELECT trunc(1.0) ` -func (q *Queries) GetTrunc(ctx context.Context) (int64, error) { +func (q *Queries) GetTrunc(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getTrunc) - var trunc int64 + var trunc float64 err := row.Scan(&trunc) return trunc, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go index adaf3985f7..06ec0436ce 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go @@ -69,9 +69,9 @@ const getFormat = `-- name: GetFormat :one SELECT format('Hello %s', 'world') ` -func (q *Queries) GetFormat(ctx context.Context) (sql.NullString, error) { +func (q *Queries) GetFormat(ctx context.Context) (string, error) { row := q.db.QueryRowContext(ctx, getFormat) - var format sql.NullString + var format string err := row.Scan(&format) return format, err } @@ -124,9 +124,9 @@ const getInstr = `-- name: GetInstr :one SELECT instr('hello', 'l') ` -func (q *Queries) GetInstr(ctx context.Context) (sql.NullInt64, error) { +func (q *Queries) GetInstr(ctx context.Context) (int64, error) { row := q.db.QueryRowContext(ctx, getInstr) - var instr sql.NullInt64 + var instr int64 err := row.Scan(&instr) return instr, err } @@ -146,9 +146,9 @@ const getLength = `-- name: GetLength :one SELECT length('12345') ` -func (q *Queries) GetLength(ctx context.Context) (sql.NullInt64, error) { +func (q *Queries) GetLength(ctx context.Context) (int64, error) { row := q.db.QueryRowContext(ctx, getLength) - var length sql.NullInt64 + var length int64 err := row.Scan(&length) return length, err } @@ -267,9 +267,9 @@ const getPrintf = `-- name: GetPrintf :one SELECT printf('Hello %s', 'world') ` -func (q *Queries) GetPrintf(ctx context.Context) (sql.NullString, error) { +func (q *Queries) GetPrintf(ctx context.Context) (string, error) { row := q.db.QueryRowContext(ctx, getPrintf) - var printf sql.NullString + var printf string err := row.Scan(&printf) return printf, err } diff --git a/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl index 2553979309..5f04bc6133 100644 --- a/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl +++ b/internal/engine/sqlite/dialect/extensions/enable_fts3/functions.jsonl @@ -2,4 +2,4 @@ {"name":"matchinfo","args":[{"type":"any"},{"type":"text"}],"returns":"blob"} {"name":"offsets","args":[{"type":"any"}],"returns":"text"} {"name":"optimize","args":[{"type":"any"}],"returns":"text"} -{"name":"snippet","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"snippet","args":[{"type":"any","has_default":true},{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl index 82339961b5..aac7a8163e 100644 --- a/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl +++ b/internal/engine/sqlite/dialect/extensions/enable_fts5/functions.jsonl @@ -1,7 +1,7 @@ -{"name":"bm25","args":[{"type":"text","has_default":true},{"type":"real","mode":"v"}],"returns":"real"} -{"name":"fts5_get_locale","args":[{"type":"any","has_default":true},{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} -{"name":"fts5_insttoken","args":[{"type":"text"}],"returns":"text"} +{"name":"bm25","args":[{"type":"any","has_default":true},{"type":"real","mode":"v"}],"returns":"real"} +{"name":"fts5_get_locale","args":[{"type":"any","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"fts5_insttoken","args":[{"type":"any"}],"returns":"any"} {"name":"fts5_locale","args":[{"type":"text"},{"type":"text"}],"returns":"text"} {"name":"fts5_source_id","returns":"text"} -{"name":"highlight","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} -{"name":"snippet","args":[{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"highlight","args":[{"type":"any","has_default":true},{"type":"integer","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"snippet","args":[{"type":"any","has_default":true},{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"integer","has_default":true},{"type":"integer","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl index 3f7f51b9a0..a37aa982b1 100644 --- a/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl +++ b/internal/engine/sqlite/dialect/extensions/enable_geopoly/functions.jsonl @@ -7,7 +7,7 @@ {"name":"geopoly_json","args":[{"type":"any"}],"returns":"text","nullable":true} {"name":"geopoly_overlap","args":[{"type":"any"},{"type":"any"}],"returns":"integer","nullable":true} {"name":"geopoly_regular","args":[{"type":"real"},{"type":"real"},{"type":"real"},{"type":"integer"}],"returns":"blob"} -{"name":"geopoly_svg","args":[{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"geopoly_svg","args":[{"type":"text","mode":"v"}],"returns":"text","nullable":true} {"name":"geopoly_within","args":[{"type":"any"},{"type":"any"}],"returns":"integer","nullable":true} {"name":"geopoly_xform","args":[{"type":"any"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"},{"type":"real"}],"returns":"blob","nullable":true} {"name":"rtreecheck","args":[{"type":"text","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} diff --git a/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl b/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl index f8ce80d774..d872692b01 100644 --- a/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl +++ b/internal/engine/sqlite/dialect/extensions/enable_percentile/functions.jsonl @@ -1,4 +1,4 @@ -{"name":"median","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"percentile","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"real","nullable":true} -{"name":"percentile_cont","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"real","nullable":true} -{"name":"percentile_disc","kind":"a","args":[{"type":"any"},{"type":"real"}],"returns":"any","nullable":true} +{"name":"median","kind":"a","args":[{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile","kind":"a","args":[{"type":"real"},{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile_cont","kind":"a","args":[{"type":"real"},{"type":"real"}],"returns":"real","nullable":true} +{"name":"percentile_disc","kind":"a","args":[{"type":"real"},{"type":"real"}],"returns":"real","nullable":true} diff --git a/internal/engine/sqlite/dialect/functions.jsonl b/internal/engine/sqlite/dialect/functions.jsonl index f5c849cf80..da25d4d581 100644 --- a/internal/engine/sqlite/dialect/functions.jsonl +++ b/internal/engine/sqlite/dialect/functions.jsonl @@ -1,23 +1,23 @@ -{"name":"-\u003e","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} -{"name":"-\u003e\u003e","args":[{"type":"any"},{"type":"text"}],"returns":"any","nullable":true} +{"name":"-\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"text","nullable":true} +{"name":"-\u003e\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} {"name":"abs","args":[{"type":"any"}],"returns":"real"} -{"name":"acos","args":[{"type":"any"}],"returns":"real"} -{"name":"acosh","args":[{"type":"any"}],"returns":"real"} -{"name":"asin","args":[{"type":"any"}],"returns":"real"} -{"name":"asinh","args":[{"type":"any"}],"returns":"real"} -{"name":"atan","args":[{"type":"any"}],"returns":"real"} -{"name":"atan2","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"atanh","args":[{"type":"any"}],"returns":"real"} +{"name":"acos","args":[{"type":"real"}],"returns":"real"} +{"name":"acosh","args":[{"type":"real"}],"returns":"real"} +{"name":"asin","args":[{"type":"real"}],"returns":"real"} +{"name":"asinh","args":[{"type":"real"}],"returns":"real"} +{"name":"atan","args":[{"type":"real"}],"returns":"real"} +{"name":"atan2","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"atanh","args":[{"type":"real"}],"returns":"real"} {"name":"avg","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"ceil","args":[{"type":"any"}],"returns":"integer"} -{"name":"ceiling","args":[{"type":"any"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"any"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"any"}],"returns":"real"} {"name":"changes","returns":"integer"} {"name":"char","args":[{"type":"integer","mode":"v"}],"returns":"text"} -{"name":"coalesce","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"coalesce","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} {"name":"concat","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"text"} {"name":"concat_ws","args":[{"type":"text"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"text"} -{"name":"cos","args":[{"type":"any"}],"returns":"real"} -{"name":"cosh","args":[{"type":"any"}],"returns":"real"} +{"name":"cos","args":[{"type":"real"}],"returns":"real"} +{"name":"cosh","args":[{"type":"real"}],"returns":"real"} {"name":"count","kind":"a","returns":"integer"} {"name":"count","kind":"a","args":[{"type":"any"}],"returns":"integer"} {"name":"cume_dist","kind":"w","returns":"real"} @@ -26,133 +26,133 @@ {"name":"current_timestamp","returns":"text"} {"name":"date","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} {"name":"datetime","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} -{"name":"degrees","args":[{"type":"any"}],"returns":"real"} +{"name":"degrees","args":[{"type":"real"}],"returns":"real"} {"name":"dense_rank","kind":"w","returns":"integer"} -{"name":"exp","args":[{"type":"any"}],"returns":"real"} +{"name":"exp","args":[{"type":"real"}],"returns":"real"} {"name":"first_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"floor","args":[{"type":"any"}],"returns":"integer"} -{"name":"format","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"floor","args":[{"type":"any"}],"returns":"real"} +{"name":"format","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} {"name":"glob","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} -{"name":"group_concat","kind":"a","args":[{"type":"any"}],"returns":"text","nullable":true} -{"name":"group_concat","kind":"a","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} -{"name":"hex","args":[{"type":"any"}],"returns":"text"} -{"name":"if","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"ifnull","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"iif","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"instr","args":[{"type":"text"},{"type":"text"}],"returns":"integer","nullable":true} +{"name":"group_concat","kind":"a","args":[{"type":"text"}],"returns":"text","nullable":true} +{"name":"group_concat","kind":"a","args":[{"type":"text"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"hex","args":[{"type":"blob"}],"returns":"text"} +{"name":"if","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} +{"name":"ifnull","args":[{"type":"any"},{"type":"any"}],"returns":"any"} +{"name":"iif","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} +{"name":"instr","args":[{"type":"any"},{"type":"any"}],"returns":"integer"} {"name":"json","args":[{"type":"any"}],"returns":"text"} {"name":"json_array","args":[{"type":"any","mode":"v"}],"returns":"text"} -{"name":"json_array_insert","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_array_insert","args":[{"type":"any","mode":"v"}],"returns":"text"} {"name":"json_array_length","args":[{"type":"any"}],"returns":"integer","nullable":true} {"name":"json_array_length","args":[{"type":"any"},{"type":"text"}],"returns":"integer","nullable":true} -{"name":"json_error_position","args":[{"type":"any"}],"returns":"integer"} -{"name":"json_extract","args":[{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"json_error_position","args":[{"type":"text"}],"returns":"integer"} +{"name":"json_extract","args":[{"type":"text","mode":"v"}],"returns":"any","nullable":true} {"name":"json_group_array","kind":"a","args":[{"type":"any"}],"returns":"text"} {"name":"json_group_object","kind":"a","args":[{"type":"text"},{"type":"any"}],"returns":"text"} -{"name":"json_insert","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} -{"name":"json_object","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_insert","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_object","args":[{"type":"text","mode":"v"}],"returns":"text"} {"name":"json_patch","args":[{"type":"any"},{"type":"any"}],"returns":"text"} {"name":"json_pretty","args":[{"type":"any"}],"returns":"text"} {"name":"json_pretty","args":[{"type":"any"},{"type":"text"}],"returns":"text"} {"name":"json_quote","args":[{"type":"any"}],"returns":"text"} -{"name":"json_remove","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} -{"name":"json_replace","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} -{"name":"json_set","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_remove","args":[{"type":"text","mode":"v"}],"returns":"text"} +{"name":"json_replace","args":[{"type":"any","mode":"v"}],"returns":"text"} +{"name":"json_set","args":[{"type":"any","mode":"v"}],"returns":"text"} {"name":"json_type","args":[{"type":"any"}],"returns":"text","nullable":true} {"name":"json_type","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} {"name":"json_valid","args":[{"type":"any"}],"returns":"integer"} {"name":"json_valid","args":[{"type":"any"},{"type":"integer"}],"returns":"integer"} {"name":"jsonb","args":[{"type":"any"}],"returns":"blob"} {"name":"jsonb_array","args":[{"type":"any","mode":"v"}],"returns":"blob"} -{"name":"jsonb_array_insert","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} -{"name":"jsonb_extract","args":[{"type":"any","has_default":true},{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"jsonb_array_insert","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_extract","args":[{"type":"text","mode":"v"}],"returns":"any","nullable":true} {"name":"jsonb_group_array","kind":"a","args":[{"type":"any"}],"returns":"blob"} {"name":"jsonb_group_object","kind":"a","args":[{"type":"text"},{"type":"any"}],"returns":"blob"} -{"name":"jsonb_insert","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} -{"name":"jsonb_object","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_insert","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_object","args":[{"type":"text","mode":"v"}],"returns":"blob"} {"name":"jsonb_patch","args":[{"type":"any"},{"type":"any"}],"returns":"blob"} -{"name":"jsonb_remove","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} -{"name":"jsonb_replace","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} -{"name":"jsonb_set","args":[{"type":"any","has_default":true},{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_remove","args":[{"type":"text","mode":"v"}],"returns":"blob"} +{"name":"jsonb_replace","args":[{"type":"any","mode":"v"}],"returns":"blob"} +{"name":"jsonb_set","args":[{"type":"any","mode":"v"}],"returns":"blob"} {"name":"julianday","args":[{"type":"any","mode":"v"}],"returns":"real","nullable":true} {"name":"lag","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"lag","kind":"w","args":[{"type":"any"},{"type":"integer"}],"returns":"any","nullable":true} -{"name":"lag","kind":"w","args":[{"type":"any"},{"type":"integer"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"lag","kind":"w","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"lag","kind":"w","args":[{"type":"any"},{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} {"name":"last_insert_rowid","returns":"integer"} {"name":"last_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} {"name":"lead","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"lead","kind":"w","args":[{"type":"any"},{"type":"integer"}],"returns":"any","nullable":true} -{"name":"lead","kind":"w","args":[{"type":"any"},{"type":"integer"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"length","args":[{"type":"any"}],"returns":"integer","nullable":true} +{"name":"lead","kind":"w","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"lead","kind":"w","args":[{"type":"any"},{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} +{"name":"length","args":[{"type":"text"}],"returns":"integer"} {"name":"like","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} {"name":"like","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"integer"} -{"name":"likelihood","args":[{"type":"any"},{"type":"real"}],"returns":"any","nullable":true} -{"name":"likely","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"ln","args":[{"type":"any"}],"returns":"real"} -{"name":"log","args":[{"type":"any"}],"returns":"real"} -{"name":"log","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"log10","args":[{"type":"any"}],"returns":"real"} -{"name":"log2","args":[{"type":"any"}],"returns":"real"} +{"name":"likelihood","args":[{"type":"any"},{"type":"any"}],"returns":"any"} +{"name":"likely","args":[{"type":"any"}],"returns":"any"} +{"name":"ln","args":[{"type":"real"}],"returns":"real"} +{"name":"log","args":[{"type":"real"}],"returns":"real"} +{"name":"log","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"log10","args":[{"type":"real"}],"returns":"real"} +{"name":"log2","args":[{"type":"real"}],"returns":"real"} {"name":"lower","args":[{"type":"text"}],"returns":"text"} {"name":"ltrim","args":[{"type":"text"}],"returns":"text"} {"name":"ltrim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} {"name":"max","kind":"a","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"max","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} +{"name":"max","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} {"name":"min","kind":"a","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"min","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any","nullable":true} -{"name":"mod","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"nth_value","kind":"w","args":[{"type":"any"},{"type":"integer"}],"returns":"any","nullable":true} +{"name":"min","args":[{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} +{"name":"mod","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"nth_value","kind":"w","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} {"name":"ntile","kind":"w","args":[{"type":"integer"}],"returns":"integer"} {"name":"nullif","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"octet_length","args":[{"type":"any"}],"returns":"integer","nullable":true} +{"name":"octet_length","args":[{"type":"any"}],"returns":"integer"} {"name":"percent_rank","kind":"w","returns":"real"} {"name":"pi","returns":"real"} -{"name":"pow","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"power","args":[{"type":"any"},{"type":"any"}],"returns":"real"} -{"name":"printf","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} +{"name":"pow","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"power","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"printf","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} {"name":"quote","args":[{"type":"any"}],"returns":"text"} -{"name":"radians","args":[{"type":"any"}],"returns":"real"} +{"name":"radians","args":[{"type":"real"}],"returns":"real"} {"name":"random","returns":"integer"} {"name":"randomblob","args":[{"type":"integer"}],"returns":"blob"} {"name":"rank","kind":"w","returns":"integer"} {"name":"replace","args":[{"type":"text"},{"type":"text"},{"type":"text"}],"returns":"text"} {"name":"round","args":[{"type":"real"}],"returns":"real"} -{"name":"round","args":[{"type":"real"},{"type":"real"}],"returns":"real"} +{"name":"round","args":[{"type":"real"},{"type":"integer"}],"returns":"real"} {"name":"row_number","kind":"w","returns":"integer"} {"name":"rtrim","args":[{"type":"text"}],"returns":"text"} {"name":"rtrim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"sign","args":[{"type":"any"}],"returns":"integer","nullable":true} -{"name":"sin","args":[{"type":"any"}],"returns":"real"} -{"name":"sinh","args":[{"type":"any"}],"returns":"real"} +{"name":"sign","args":[{"type":"real"}],"returns":"integer","nullable":true} +{"name":"sin","args":[{"type":"real"}],"returns":"real"} +{"name":"sinh","args":[{"type":"real"}],"returns":"real"} {"name":"sqlite_compileoption_get","args":[{"type":"integer"}],"returns":"text","nullable":true} {"name":"sqlite_compileoption_used","args":[{"type":"text"}],"returns":"integer"} {"name":"sqlite_source_id","returns":"text"} {"name":"sqlite_version","returns":"text"} -{"name":"sqrt","args":[{"type":"any"}],"returns":"real"} +{"name":"sqrt","args":[{"type":"real"}],"returns":"real"} {"name":"strftime","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text","nullable":true} -{"name":"string_agg","kind":"a","args":[{"type":"any"},{"type":"text"}],"returns":"text","nullable":true} +{"name":"string_agg","kind":"a","args":[{"type":"text"},{"type":"text"}],"returns":"text","nullable":true} {"name":"substr","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} {"name":"substr","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} {"name":"substring","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} {"name":"substring","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} {"name":"subtype","args":[{"type":"any"}],"returns":"integer"} {"name":"sum","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"tan","args":[{"type":"any"}],"returns":"real"} -{"name":"tanh","args":[{"type":"any"}],"returns":"real"} +{"name":"tan","args":[{"type":"real"}],"returns":"real"} +{"name":"tanh","args":[{"type":"real"}],"returns":"real"} {"name":"time","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} {"name":"timediff","args":[{"type":"any"},{"type":"any"}],"returns":"text","nullable":true} {"name":"total","kind":"a","args":[{"type":"any"}],"returns":"real"} {"name":"total_changes","returns":"integer"} {"name":"trim","args":[{"type":"text"}],"returns":"text"} {"name":"trim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"trunc","args":[{"type":"any"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"any"}],"returns":"real"} {"name":"typeof","args":[{"type":"any"}],"returns":"text"} {"name":"unhex","args":[{"type":"text"}],"returns":"blob","nullable":true} {"name":"unhex","args":[{"type":"text"},{"type":"text"}],"returns":"blob","nullable":true} {"name":"unicode","args":[{"type":"text"}],"returns":"integer"} {"name":"unistr","args":[{"type":"text"}],"returns":"text"} -{"name":"unistr_quote","args":[{"type":"text"}],"returns":"text"} -{"name":"unixepoch","args":[{"type":"any","mode":"v"}],"returns":"integer","nullable":true} -{"name":"unlikely","args":[{"type":"any"}],"returns":"any","nullable":true} +{"name":"unistr_quote","args":[{"type":"any"}],"returns":"text"} +{"name":"unixepoch","args":[{"type":"any","mode":"v"}],"returns":"real","nullable":true} +{"name":"unlikely","args":[{"type":"any"}],"returns":"any"} {"name":"upper","args":[{"type":"text"}],"returns":"text"} {"name":"zeroblob","args":[{"type":"integer"}],"returns":"blob"} diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index 82e20c0b5b..f06dfba04e 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -70,9 +70,15 @@ the hand-written files alone, and the checks do not look at them. hand-written `dialect.json`. SQLite describes its functions as far as their names, their kinds and the number of arguments each overload takes, and no further — it types values, not functions — so what each returns and what - its arguments hold comes from the table in `sqlite/signatures.go`, and a - function a shell reports that the table does not know fails the run rather - than being guessed at. The pinned release is the one the main module's + its arguments hold is read from the amalgamation: every function is + registered with the C functions that implement it, and those set their + result through `sqlite3_result_*` and read their arguments through + `sqlite3_value_*`. A function a shell reports that the source does not + register fails the run rather than being guessed at. Whether an aggregate + returns NULL over no rows is found by running it over none; the scalar + functions that return NULL for arguments that are not are a short list in + `sqlite/signatures.go`, since a SQLite function returns NULL as often by + setting no result as by saying so. The pinned release is the one the main module's driver embeds. SQLite has no catalog of types or operators, so `types.jsonl` and `operators.jsonl` are hand-written. diff --git a/internal/goldeneye/sqlite/signatures.go b/internal/goldeneye/sqlite/signatures.go index d4f6cddee9..121bf36507 100644 --- a/internal/goldeneye/sqlite/signatures.go +++ b/internal/goldeneye/sqlite/signatures.go @@ -2,14 +2,14 @@ package sqlite import "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" -// signature is what SQLite does not record about a function: the type it -// returns, whether that can be NULL when its arguments are not, and the -// types its arguments are meant to hold. Args types the leading arguments -// in order; a position it does not cover is "any", which the analyzer -// resolves to the argument's own type. Variadic types the arguments an -// overload of variable arity repeats, "any" when empty. How many arguments -// an overload takes, and how many of them it requires, comes from the -// shell, not from here. +// signature is what a function returns and what its arguments hold, read +// from the amalgamation by source.signature. Args types the leading +// arguments in order; a position it does not cover is "any", which the +// analyzer resolves to the argument's own type. Variadic types the +// arguments an overload of variable arity repeats, "any" when empty. How +// many arguments an overload takes, and how many of them it requires, +// comes from the shell. Nullable is decided afterwards: for an aggregate by +// running it over no rows, for a scalar by the nullable list below. type signature struct { Args []string Variadic string @@ -49,17 +49,22 @@ func (s signature) argType(i int) string { return "any" } -// omitted are built-in functions the dialect leaves out: ones that exist -// for their side effect and return nothing a query can use, and one the -// shell's build adds that the library does not have. +// inlineReturns is what the functions the VDBE implements in bytecode +// return, by the INLINEFUNC_* constant their registration carries. All but +// one hand back one of their arguments, which is what the default of "any" +// says; sqlite_offset is a byte offset. +var inlineReturns = map[string]string{ + "INLINEFUNC_sqlite_offset": "integer", +} + +// omitted are functions the dialect leaves out: ones that exist for their +// side effect and return nothing a query can use, and ones an extension +// uses to pass pointers to itself. var omitted = map[string]bool{ // Loads a shared library and returns NULL. "load_extension": true, // Writes to the error log and returns NULL. "sqlite_log": true, - // SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION's stand-in for any function the - // shell does not know, so that EXPLAIN works on queries that use one. - "unknown": true, // FTS3's and FTS5's ways of passing pointers to their virtual tables, // not functions a query calls. "fts3_tokenizer": true, @@ -68,217 +73,52 @@ var omitted = map[string]bool{ "geopoly_debug": true, } -// signatures covers every function of the default build and of each -// extension's, by the SQLite documentation of each — lang_corefunc, -// lang_mathfunc, lang_datefunc, lang_aggfunc, windowfunctions, json1, -// fts3, fts5, rtree and geopoly. A function marked Nullable can return NULL -// for arguments that are not: an aggregate over no rows, a lookup that -// finds nothing, an input that does not parse. -// -// SQLite's values carry their own types, so a function's result type is -// what it typically produces. abs of an integer is an integer, but abs -// returns real here, as sum does, because the analyzer wants one answer; -// the polymorphic "any" is for functions that hand back one of their -// arguments. -var signatures = map[string]signature{ - // Aggregates. - "avg": {Returns: "real", Nullable: true}, - "count": {Returns: "integer"}, - "group_concat": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, - "max": {Returns: "any", Nullable: true}, - "min": {Returns: "any", Nullable: true}, - "string_agg": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, - "sum": {Returns: "real", Nullable: true}, - "total": {Returns: "real"}, - - // Percentiles, added by SQLITE_ENABLE_PERCENTILE. - "median": {Returns: "real", Nullable: true}, - "percentile": {Args: []string{"any", "real"}, Returns: "real", Nullable: true}, - "percentile_cont": {Args: []string{"any", "real"}, Returns: "real", Nullable: true}, - "percentile_disc": {Args: []string{"any", "real"}, Returns: "any", Nullable: true}, - - // Window functions. - "cume_dist": {Returns: "real"}, - "dense_rank": {Returns: "integer"}, - "first_value": {Returns: "any", Nullable: true}, - "lag": {Args: []string{"any", "integer", "any"}, Returns: "any", Nullable: true}, - "last_value": {Returns: "any", Nullable: true}, - "lead": {Args: []string{"any", "integer", "any"}, Returns: "any", Nullable: true}, - "nth_value": {Args: []string{"any", "integer"}, Returns: "any", Nullable: true}, - "ntile": {Args: []string{"integer"}, Returns: "integer"}, - "percent_rank": {Returns: "real"}, - "rank": {Returns: "integer"}, - "row_number": {Returns: "integer"}, - +// nullable are the scalar functions that return NULL for arguments that +// are not: a lookup that finds nothing, an input that does not parse, a +// value with no sign. The source cannot say this — a SQLite function +// returns NULL as often by setting no result as by calling +// sqlite3_result_null — so it is listed by the documentation of each. +// Aggregates are not listed: whether one returns NULL over no rows is +// found by running it over none. +var nullable = map[string]bool{ // Core functions. - "changes": {Returns: "integer"}, - "char": {Variadic: "integer", Returns: "text"}, - "coalesce": {Returns: "any", Nullable: true}, - "concat": {Returns: "text"}, - "concat_ws": {Args: []string{"text"}, Returns: "text"}, - "format": {Args: []string{"text"}, Returns: "text", Nullable: true}, - "glob": {Args: []string{"text", "text"}, Returns: "integer"}, - "hex": {Returns: "text"}, - "if": {Returns: "any", Nullable: true}, - "ifnull": {Returns: "any", Nullable: true}, - "iif": {Returns: "any", Nullable: true}, - "instr": {Args: []string{"text", "text"}, Returns: "integer", Nullable: true}, - "last_insert_rowid": {Returns: "integer"}, - "length": {Returns: "integer", Nullable: true}, - "like": {Args: []string{"text", "text", "text"}, Returns: "integer"}, - "likelihood": {Args: []string{"any", "real"}, Returns: "any", Nullable: true}, - "likely": {Returns: "any", Nullable: true}, - "lower": {Args: []string{"text"}, Returns: "text"}, - "ltrim": {Args: []string{"text", "text"}, Returns: "text"}, - "nullif": {Returns: "any", Nullable: true}, - "octet_length": {Returns: "integer", Nullable: true}, - "printf": {Args: []string{"text"}, Returns: "text", Nullable: true}, - "quote": {Returns: "text"}, - "random": {Returns: "integer"}, - "randomblob": {Args: []string{"integer"}, Returns: "blob"}, - "replace": {Args: []string{"text", "text", "text"}, Returns: "text"}, - "round": {Args: []string{"real", "real"}, Returns: "real"}, - "rtrim": {Args: []string{"text", "text"}, Returns: "text"}, - "sign": {Returns: "integer", Nullable: true}, - "sqlite_compileoption_get": {Args: []string{"integer"}, Returns: "text", Nullable: true}, - "sqlite_compileoption_used": {Args: []string{"text"}, Returns: "integer"}, - "sqlite_source_id": {Returns: "text"}, - "sqlite_version": {Returns: "text"}, - "substr": {Args: []string{"any", "integer", "integer"}, Returns: "text"}, - "substring": {Args: []string{"any", "integer", "integer"}, Returns: "text"}, - "subtype": {Returns: "integer"}, - "total_changes": {Returns: "integer"}, - "trim": {Args: []string{"text", "text"}, Returns: "text"}, - "typeof": {Returns: "text"}, - "unhex": {Args: []string{"text", "text"}, Returns: "blob", Nullable: true}, - "unicode": {Args: []string{"text"}, Returns: "integer"}, - "unistr": {Args: []string{"text"}, Returns: "text"}, - "unistr_quote": {Args: []string{"text"}, Returns: "text"}, - "unlikely": {Returns: "any", Nullable: true}, - "upper": {Args: []string{"text"}, Returns: "text"}, - "zeroblob": {Args: []string{"integer"}, Returns: "blob"}, - - // Math functions. - "abs": {Returns: "real"}, - "acos": {Returns: "real"}, - "acosh": {Returns: "real"}, - "asin": {Returns: "real"}, - "asinh": {Returns: "real"}, - "atan": {Returns: "real"}, - "atan2": {Returns: "real"}, - "atanh": {Returns: "real"}, - "ceil": {Returns: "integer"}, - "ceiling": {Returns: "integer"}, - "cos": {Returns: "real"}, - "cosh": {Returns: "real"}, - "degrees": {Returns: "real"}, - "exp": {Returns: "real"}, - "floor": {Returns: "integer"}, - "ln": {Returns: "real"}, - "log": {Returns: "real"}, - "log10": {Returns: "real"}, - "log2": {Returns: "real"}, - "mod": {Returns: "real"}, - "pi": {Returns: "real"}, - "pow": {Returns: "real"}, - "power": {Returns: "real"}, - "radians": {Returns: "real"}, - "sin": {Returns: "real"}, - "sinh": {Returns: "real"}, - "sqrt": {Returns: "real"}, - "tan": {Returns: "real"}, - "tanh": {Returns: "real"}, - "trunc": {Returns: "integer"}, - - // Date and time functions, which return text in ISO-8601 form and NULL - // for a time value they cannot parse. - "current_date": {Returns: "text"}, - "current_time": {Returns: "text"}, - "current_timestamp": {Returns: "text"}, - "date": {Returns: "text", Nullable: true}, - "datetime": {Returns: "text", Nullable: true}, - "julianday": {Returns: "real", Nullable: true}, - "strftime": {Args: []string{"text"}, Returns: "text", Nullable: true}, - "time": {Returns: "text", Nullable: true}, - "timediff": {Returns: "text", Nullable: true}, - "unixepoch": {Returns: "integer", Nullable: true}, - - // JSON functions. A JSON argument is text or a JSONB blob, so it is - // "any"; a path is text. The json_ forms return JSON text and the - // jsonb_ forms JSONB blobs, and the extracting forms return whatever - // the path leads to, or NULL when it leads nowhere. - "->": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, - "->>": {Args: []string{"any", "text"}, Returns: "any", Nullable: true}, - "json": {Returns: "text"}, - "json_array": {Returns: "text"}, - "json_array_insert": {Args: []string{"any"}, Returns: "text"}, - "json_array_length": {Args: []string{"any", "text"}, Returns: "integer", Nullable: true}, - "json_error_position": {Returns: "integer"}, - "json_extract": {Args: []string{"any", "text"}, Returns: "any", Nullable: true}, - "json_group_array": {Returns: "text"}, - "json_group_object": {Args: []string{"text", "any"}, Returns: "text"}, - "json_insert": {Args: []string{"any"}, Returns: "text"}, - "json_object": {Returns: "text"}, - "json_patch": {Returns: "text"}, - "json_pretty": {Args: []string{"any", "text"}, Returns: "text"}, - "json_quote": {Returns: "text"}, - "json_remove": {Args: []string{"any"}, Returns: "text"}, - "json_replace": {Args: []string{"any"}, Returns: "text"}, - "json_set": {Args: []string{"any"}, Returns: "text"}, - "json_type": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, - "json_valid": {Args: []string{"any", "integer"}, Returns: "integer"}, - "jsonb": {Returns: "blob"}, - "jsonb_array": {Returns: "blob"}, - "jsonb_array_insert": {Args: []string{"any"}, Returns: "blob"}, - "jsonb_extract": {Args: []string{"any", "text"}, Returns: "any", Nullable: true}, - "jsonb_group_array": {Returns: "blob"}, - "jsonb_group_object": {Args: []string{"text", "any"}, Returns: "blob"}, - "jsonb_insert": {Args: []string{"any"}, Returns: "blob"}, - "jsonb_object": {Returns: "blob"}, - "jsonb_patch": {Returns: "blob"}, - "jsonb_remove": {Args: []string{"any"}, Returns: "blob"}, - "jsonb_replace": {Args: []string{"any"}, Returns: "blob"}, - "jsonb_set": {Args: []string{"any"}, Returns: "blob"}, - - // Added by SQLITE_SOUNDEX. - "soundex": {Args: []string{"text"}, Returns: "text"}, - - // Added by SQLITE_ENABLE_OFFSET_SQL_FUNC. - "sqlite_offset": {Returns: "integer", Nullable: true}, - - // FTS3's auxiliary functions, whose first argument names the table. - "matchinfo": {Args: []string{"any", "text"}, Returns: "blob"}, - "offsets": {Args: []string{"any"}, Returns: "text"}, - "optimize": {Args: []string{"any"}, Returns: "text"}, - - // FTS5's auxiliary and locale functions. snippet is FTS3's too, with the - // same result. - "bm25": {Args: []string{"text"}, Variadic: "real", Returns: "real"}, - "fts5_get_locale": {Args: []string{"any", "any"}, Returns: "text", Nullable: true}, - "fts5_insttoken": {Args: []string{"text"}, Returns: "text"}, - "fts5_locale": {Args: []string{"text", "text"}, Returns: "text"}, - "fts5_source_id": {Returns: "text"}, - "highlight": {Args: []string{"text", "integer", "text", "text"}, Returns: "text"}, - "snippet": {Args: []string{"text", "integer", "text", "text", "text", "integer"}, Returns: "text"}, - - // R*Tree's functions for inspecting an index's nodes. - "rtreecheck": {Args: []string{"text", "text"}, Returns: "text"}, - "rtreedepth": {Args: []string{"blob"}, Returns: "integer"}, - "rtreenode": {Args: []string{"integer", "blob"}, Returns: "text"}, - - // GEOPOLY's functions. A polygon argument is GeoJSON text or the binary - // form, so it is "any"; the functions that return a polygon return the - // binary form. - "geopoly_area": {Returns: "real", Nullable: true}, - "geopoly_bbox": {Returns: "blob", Nullable: true}, - "geopoly_blob": {Returns: "blob", Nullable: true}, - "geopoly_ccw": {Returns: "blob", Nullable: true}, - "geopoly_contains_point": {Args: []string{"any", "real", "real"}, Returns: "integer", Nullable: true}, - "geopoly_group_bbox": {Returns: "blob", Nullable: true}, - "geopoly_json": {Returns: "text", Nullable: true}, - "geopoly_overlap": {Returns: "integer", Nullable: true}, - "geopoly_regular": {Args: []string{"real", "real", "real", "integer"}, Returns: "blob"}, - "geopoly_svg": {Args: []string{"any", "text"}, Returns: "text", Nullable: true}, - "geopoly_within": {Returns: "integer", Nullable: true}, - "geopoly_xform": {Args: []string{"any", "real", "real", "real", "real", "real", "real"}, Returns: "blob", Nullable: true}, + "nullif": true, + "sign": true, + "sqlite_compileoption_get": true, + "sqlite_offset": true, + "unhex": true, + // Date and time functions, for a time value they cannot parse. + "date": true, + "datetime": true, + "julianday": true, + "strftime": true, + "time": true, + "timediff": true, + "unixepoch": true, + // JSON functions, for a path that leads nowhere. + "->": true, + "->>": true, + "json_array_length": true, + "json_extract": true, + "json_type": true, + "jsonb_extract": true, + // Window functions, for a row outside the frame. + "first_value": true, + "lag": true, + "last_value": true, + "lead": true, + "nth_value": true, + // FTS5, for a table with no locale. + "fts5_get_locale": true, + // GEOPOLY functions, for an argument that is not a polygon. + "geopoly_area": true, + "geopoly_bbox": true, + "geopoly_blob": true, + "geopoly_ccw": true, + "geopoly_contains_point": true, + "geopoly_json": true, + "geopoly_overlap": true, + "geopoly_svg": true, + "geopoly_within": true, + "geopoly_xform": true, } diff --git a/internal/goldeneye/sqlite/source.go b/internal/goldeneye/sqlite/source.go new file mode 100644 index 0000000000..1599e839f9 --- /dev/null +++ b/internal/goldeneye/sqlite/source.go @@ -0,0 +1,517 @@ +package sqlite + +import ( + "fmt" + "os" + "regexp" + "sort" + "strings" +) + +// source is what the amalgamation says about how SQLite's functions answer. +// A function is registered in one of a few shapes — the FuncDef macros of +// the built-in tables, a sqlite3_create_function call, or a struct table an +// extension walks — each naming the C functions that implement it. Those +// implementations set their result through sqlite3_result_* and read their +// arguments through sqlite3_value_*, which is as close as SQLite comes to +// declaring a signature. +type source struct { + funcs map[string]cfunc + aliases map[string]string + regs map[string]*registration +} + +// cfunc is one C function definition: its parameter list and its body. +type cfunc struct { + params string + body string +} + +// registration is the union of everything the source registers under one +// SQL function name. +type registration struct { + // scalar implements a scalar overload: its results and its arguments. + scalar []string + // step and final implement an aggregate or window overload: step reads + // the arguments, final and value set the result. + step []string + final []string + // inline names the INLINEFUNC_* constant of a function the VDBE + // implements in bytecode, which has no C body to read. + inline string + // json records what a JSON function's registration says it returns, + // "text" or "blob", since the json_ and jsonb_ forms share an + // implementation; jsonAlways says the registration promises JSON text + // whatever the implementation might otherwise return, as -> does. + json string + jsonAlways bool + // table marks a registration found only in a struct table, which is + // consulted when nothing more direct registered the name. + table bool +} + +// The FuncDef macros, with the positions of the C functions in each. +var macroEntry = regexp.MustCompile(`\b(FUNCTION2|FUNCTION|VFUNCTION|SFUNCTION|MFUNCTION|JFUNCTION|INLINE_FUNC|DFUNCTION|PURE_DATE|STR_FUNCTION|LIKEFUNC|WAGGREGATE|WINDOWFUNCX|WINDOWFUNCALL|WINDOWFUNCNOOP)\(`) + +var ( + createFunction = regexp.MustCompile(`\bsqlite3_create_(window_)?function\(`) + defineAlias = regexp.MustCompile(`(?m)^#define\s+(\w+)\s+(\w+)\s*$`) + definition = regexp.MustCompile(`(?m)^(?:static\s+|SQLITE_PRIVATE\s+)?(?:const\s+)?(?:unsigned\s+)?[A-Za-z_]\w*(?:\s*\*+\s*|\s+)([A-Za-z_]\w*)\(`) + initializer = regexp.MustCompile(`\{[^{}]*"([a-z_0-9>-]+)"[^{}]*\}`) + identifier = regexp.MustCompile(`\b([A-Za-z_]\w*)\b`) + resultCall = regexp.MustCompile(`\bsqlite3_result_(\w+)\(`) + valueCall = regexp.MustCompile(`\bsqlite3_value_(\w+)\(\s*(?:argv|apVal|apArg)\[(\w+)\]`) + callee = regexp.MustCompile(`\b([A-Za-z_]\w*)\s*\(`) +) + +// readSource reads the amalgamation. +func readSource(path string) (*source, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + text := string(data) + s := &source{ + funcs: map[string]cfunc{}, + aliases: map[string]string{}, + regs: map[string]*registration{}, + } + for _, m := range defineAlias.FindAllStringSubmatch(text, -1) { + s.aliases[m[1]] = m[2] + } + s.readDefinitions(text) + s.readMacros(text) + s.readCreateFunctions(text) + s.readTables(text) + return s, nil +} + +// readDefinitions indexes every function defined at the left margin. +func (s *source) readDefinitions(text string) { + for _, m := range definition.FindAllStringSubmatchIndex(text, -1) { + name := text[m[2]:m[3]] + params, end, ok := balanced(text, m[1]-1, '(', ')') + if !ok { + continue + } + i := end + for i < len(text) && (text[i] == ' ' || text[i] == '\n' || text[i] == '\r' || text[i] == '\t') { + i++ + } + if i >= len(text) || text[i] != '{' { + continue + } + body, _, ok := balanced(text, i, '{', '}') + if !ok { + continue + } + if _, dup := s.funcs[name]; !dup { + s.funcs[name] = cfunc{params: params, body: body} + } + } +} + +// balanced returns the text between the bracket at open and its match, +// skipping string and character literals and comments, and the index after +// the closing bracket. +func balanced(text string, open int, lb, rb byte) (string, int, bool) { + depth := 0 + for i := open; i < len(text); i++ { + switch c := text[i]; { + case c == '"' || c == '\'': + for i++; i < len(text) && text[i] != c; i++ { + if text[i] == '\\' { + i++ + } + } + case c == '/' && i+1 < len(text) && text[i+1] == '*': + end := strings.Index(text[i+2:], "*/") + if end < 0 { + return "", 0, false + } + i += end + 3 + case c == '/' && i+1 < len(text) && text[i+1] == '/': + end := strings.IndexByte(text[i:], '\n') + if end < 0 { + return "", 0, false + } + i += end + case c == lb: + depth++ + case c == rb: + depth-- + if depth == 0 { + return text[open+1 : i], i + 1, true + } + } + } + return "", 0, false +} + +// arguments splits a bracketed argument list at the commas of its own +// level, and reports the index after the closing bracket. +func arguments(text string, open int) ([]string, int, bool) { + inner, end, ok := balanced(text, open, '(', ')') + if !ok { + return nil, 0, false + } + var args []string + depth, start := 0, 0 + for i := 0; i < len(inner); i++ { + switch inner[i] { + case '(', '[', '{': + depth++ + case ')', ']', '}': + depth-- + case '"': + for i++; i < len(inner) && inner[i] != '"'; i++ { + if inner[i] == '\\' { + i++ + } + } + case ',': + if depth == 0 { + args = append(args, strings.TrimSpace(inner[start:i])) + start = i + 1 + } + } + } + args = append(args, strings.TrimSpace(inner[start:])) + return args, end, true +} + +func (s *source) reg(name string) *registration { + name = strings.ToLower(name) + r, ok := s.regs[name] + if !ok { + r = ®istration{} + s.regs[name] = r + } + return r +} + +// readMacros reads the FuncDef tables. The macro definitions themselves +// match too, and are told apart by their formal parameter names. +func (s *source) readMacros(text string) { + for _, m := range macroEntry.FindAllStringSubmatchIndex(text, -1) { + macro := text[m[2]:m[3]] + args, _, ok := arguments(text, m[1]-1) + if !ok || len(args) < 2 || args[0] == "zName" || args[0] == "name" { + continue + } + name := args[0] + switch macro { + case "FUNCTION", "FUNCTION2", "VFUNCTION", "SFUNCTION", "DFUNCTION", "PURE_DATE", "STR_FUNCTION": + if len(args) > 4 { + r := s.reg(name) + r.scalar = append(r.scalar, args[4]) + } + case "MFUNCTION": + if len(args) > 3 { + r := s.reg(name) + r.scalar = append(r.scalar, args[3]) + } + case "JFUNCTION": + // JFUNCTION(zName, nArg, bUseCache, bWS, bRS, bJsonB, iArg, xFunc) + if len(args) > 7 { + r := s.reg(name) + r.scalar = append(r.scalar, args[7]) + r.json = "text" + if args[5] == "1" { + r.json = "blob" + } + r.jsonAlways = strings.Contains(args[6], "JSON_JSON") + } + case "INLINE_FUNC": + if len(args) > 2 { + s.reg(name).inline = args[2] + } + case "LIKEFUNC": + r := s.reg(name) + r.scalar = append(r.scalar, "likeFunc") + case "WAGGREGATE": + // WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f) + // The JSON aggregates use it too, told by their implementation, + // with JSON_BLOB as the user data of the jsonb_ forms. + if len(args) > 6 { + r := s.reg(name) + r.step = append(r.step, args[4]) + r.final = append(r.final, args[5], args[6]) + if strings.HasPrefix(args[4], "json") { + r.json = "text" + if strings.Contains(args[2], "JSON_BLOB") { + r.json = "blob" + } + } + } + case "WINDOWFUNCX", "WINDOWFUNCALL": + r := s.reg(name) + r.step = append(r.step, name+"StepFunc") + r.final = append(r.final, name+"ValueFunc") + case "WINDOWFUNCNOOP": + s.reg(name).inline = "bytecode" + } + } +} + +// readCreateFunctions reads the functions extensions register by calling +// sqlite3_create_function or sqlite3_create_window_function with a literal +// name. +func (s *source) readCreateFunctions(text string) { + for _, m := range createFunction.FindAllStringSubmatchIndex(text, -1) { + window := m[2] != -1 + args, _, ok := arguments(text, m[1]-1) + if !ok || len(args) < 8 || !strings.HasPrefix(args[1], `"`) { + continue + } + r := s.reg(strings.Trim(args[1], `"`)) + if window { + // (db, zName, nArg, eTextRep, pApp, xStep, xFinal, xValue, xInverse, xDestroy) + r.step = given(r.step, args[5]) + r.final = given(r.final, args[6], args[7]) + } else { + // (db, zName, nArg, eTextRep, pApp, xFunc, xStep, xFinal) + r.scalar = given(r.scalar, args[5]) + r.step = given(r.step, args[6]) + r.final = given(r.final, args[7]) + } + } +} + +// given appends the implementations a registration names, leaving out the +// methods it passes as 0 or NULL. +func given(impls []string, names ...string) []string { + for _, name := range names { + if name != "0" && name != "NULL" { + impls = append(impls, name) + } + } + return impls +} + +// readTables reads the struct tables extensions walk to register their +// functions — geopoly's aFunc, FTS5's aBuiltin, FTS3's aOverload — each row +// an initializer holding the function's name and the C functions that +// implement it. Any C function in the row counts as an implementation; +// which reads arguments and which sets the result comes out in the +// reading. A row is consulted only for a name nothing else registered. +func (s *source) readTables(text string) { + for _, m := range initializer.FindAllStringSubmatch(text, -1) { + var impls []string + for _, id := range identifier.FindAllStringSubmatch(m[0], -1) { + if _, ok := s.funcs[s.resolve(id[1])]; ok { + impls = append(impls, id[1]) + } + } + if len(impls) == 0 { + continue + } + name := strings.ToLower(m[1]) + r, ok := s.regs[name] + if ok && !r.table { + continue + } + r = s.reg(name) + r.table = true + r.scalar = append(r.scalar, impls...) + } +} + +// resolve follows #define aliases from a C function name to the one that +// is defined. +func (s *source) resolve(name string) string { + for i := 0; i < 8; i++ { + alias, ok := s.aliases[name] + if !ok { + return name + } + name = alias + } + return name +} + +// noop reports a C function that does nothing: the stand-in for a function +// the VDBE implements in bytecode, whose result is one of its arguments. +func noop(name string) bool { + return strings.HasPrefix(name, "noop") +} + +// The kinds a result or argument call names, by the sqlite3_result_* or +// sqlite3_value_* suffix with any 64 dropped. Suffixes not listed — error, +// subtype, type, bytes, dup — say nothing about a type. +var ( + resultKinds = map[string]string{ + "int": "integer", "double": "real", + "text": "text", "text16": "text", "text16le": "text", "text16be": "text", + "blob": "blob", "zeroblob": "blob", + "value": "any", "pointer": "any", + } + valueKinds = map[string]string{ + "int": "integer", "double": "real", + "text": "text", "text16": "text", "text16le": "text", "text16be": "text", + "blob": "blob", + } +) + +func kindOf(kinds map[string]string, suffix string) (string, bool) { + k, ok := kinds[strings.TrimSuffix(suffix, "64")] + return k, ok +} + +// results collects the kinds a C function sets its result to, following +// the helpers it calls a few levels down, since many functions hand their +// result to one. +func (s *source) results(name string, depth int, seen map[string]bool) map[string]bool { + kinds := map[string]bool{} + name = s.resolve(name) + if noop(name) { + kinds["any"] = true + return kinds + } + fn, ok := s.funcs[name] + if !ok || seen[name] || depth > 3 { + return kinds + } + seen[name] = true + for _, m := range resultCall.FindAllStringSubmatch(fn.body, -1) { + if k, ok := kindOf(resultKinds, m[1]); ok { + kinds[k] = true + } + } + for _, m := range callee.FindAllStringSubmatch(fn.body, -1) { + c := m[1] + if c == name || strings.HasPrefix(c, "sqlite3_") || strings.HasPrefix(c, "sqlite3Vdbe") { + continue + } + if _, ok := s.funcs[c]; ok { + for k := range s.results(c, depth+1, seen) { + kinds[k] = true + } + } + } + return kinds +} + +// args collects the kinds a C function reads each of its arguments as, by +// position, and the kind it reads a run of arguments as under a loop +// index. An implementation called through the FTS5 extension API is handed +// the table as an implicit first argument, so its positions shift by one. +func (s *source) args(name string, positions map[int]map[string]bool, variadic map[string]bool) { + fn, ok := s.funcs[s.resolve(name)] + if !ok { + return + } + shift := 0 + if strings.Contains(fn.params, "Fts5ExtensionApi") { + shift = 1 + if positions[0] == nil { + positions[0] = map[string]bool{} + } + positions[0]["any"] = true + } + for _, m := range valueCall.FindAllStringSubmatch(fn.body, -1) { + k, ok := kindOf(valueKinds, m[1]) + if !ok { + continue + } + var pos int + if _, err := fmt.Sscanf(m[2], "%d", &pos); err != nil { + variadic[k] = true + continue + } + pos += shift + if positions[pos] == nil { + positions[pos] = map[string]bool{} + } + positions[pos][k] = true + } +} + +// single reduces the kinds seen at one position to a type: the one kind +// seen, or "any" for a mixture or nothing. +func single(kinds map[string]bool) string { + if len(kinds) == 1 { + for k := range kinds { + return k + } + } + return "any" +} + +// signature derives what the source says a SQL function returns and takes. +// A result of one kind is that type. A function that returns one of its +// arguments, or a mixture of kinds, takes the type of its first argument, +// which the seed spells "any" — except that integer and real together widen +// to real, as SQLite's own arithmetic does, and text and blob together to +// text, since a function that returns either is handing back the bytes it +// was given, and the legacy compiler cannot follow "any" to an argument. +func (s *source) signature(name string) (signature, error) { + r, ok := s.regs[strings.ToLower(name)] + if !ok { + return signature{}, fmt.Errorf("the amalgamation registers no function named %s", name) + } + var sig signature + switch { + case r.inline != "": + sig.Returns = inlineReturns[r.inline] + if sig.Returns == "" { + sig.Returns = "any" + } + default: + kinds := map[string]bool{} + for _, fn := range append(append([]string{}, r.scalar...), r.final...) { + for k := range s.results(fn, 0, map[string]bool{}) { + kinds[k] = true + } + } + jsonOnly := r.json != "" && len(kinds) <= 2 && !kinds["integer"] && !kinds["real"] && !kinds["any"] + switch { + case len(kinds) == 0: + return signature{}, fmt.Errorf("cannot tell what %s returns: no sqlite3_result call in %s", name, strings.Join(append(append([]string{}, r.scalar...), r.final...), ", ")) + case r.jsonAlways: + sig.Returns = "text" + case kinds["any"]: + sig.Returns = "any" + case len(kinds) == 1: + for k := range kinds { + sig.Returns = k + } + case jsonOnly: + // The shared implementation writes JSON text or JSONB; the + // registration says which this form gets. + sig.Returns = r.json + case len(kinds) == 2 && kinds["integer"] && kinds["real"]: + sig.Returns = "real" + case len(kinds) == 2 && kinds["text"] && kinds["blob"]: + sig.Returns = "text" + default: + sig.Returns = "any" + } + } + + positions := map[int]map[string]bool{} + variadic := map[string]bool{} + for _, fn := range append(append([]string{}, r.scalar...), r.step...) { + s.args(fn, positions, variadic) + } + if len(positions) > 0 { + var order []int + for pos := range positions { + order = append(order, pos) + } + sort.Ints(order) + sig.Args = make([]string, order[len(order)-1]+1) + for i := range sig.Args { + sig.Args[i] = single(positions[i]) + } + // Trailing positions nothing typed say nothing, unless a variadic + // tail follows them, when they keep it from starting early. + for len(variadic) == 0 && len(sig.Args) > 0 && sig.Args[len(sig.Args)-1] == "any" { + sig.Args = sig.Args[:len(sig.Args)-1] + } + } + if len(variadic) == 1 { + sig.Variadic = single(variadic) + } + return sig, nil +} diff --git a/internal/goldeneye/sqlite/sqlite.go b/internal/goldeneye/sqlite/sqlite.go index 02c5362a95..d8aa1b62ba 100644 --- a/internal/goldeneye/sqlite/sqlite.go +++ b/internal/goldeneye/sqlite/sqlite.go @@ -1,18 +1,24 @@ // Package sqlite generates the SQLite dialect seed under // internal/engine/sqlite/dialect from sqlite3 shells built from the // amalgamation sqlite.org publishes, run against in-memory databases that -// need no server. +// need no server, and from the amalgamation itself. // // SQLite describes its functions as far as their names, their kinds and the // number of arguments each takes — pragma_function_list — and no further: // it types values rather than columns or functions, so nothing in the -// database says what a function returns or what it expects. functions.jsonl -// is therefore built from both sides. The shell says which functions exist, -// how many arguments each overload takes and whether it aggregates, and the -// signatures table in this package says what each returns and what its -// arguments are meant to hold. A function the shell reports that the table -// does not know fails generation rather than being guessed at, and so does -// a table entry no shell reports. +// database says what a function returns or what it expects. The source +// does, in its way. Every function is registered with the C functions that +// implement it, and those set their result through sqlite3_result_* and +// read their arguments through sqlite3_value_*, which is as close as SQLite +// comes to declaring a signature. So functions.jsonl is built from both: +// the shell says which functions exist, how many arguments each overload +// takes and whether it aggregates, and the amalgamation says what each +// returns and what its arguments hold. A function the shell reports that +// the source does not register, or whose implementation sets no result, +// fails generation rather than being guessed at. What neither can say — +// which scalar functions return NULL for arguments that are not — is a +// short list in signatures.go; for aggregates it is found by running each +// over no rows. // // Which functions a SQLite has is decided when it is compiled, so the // dialect treats compile options the way the PostgreSQL dialect treats @@ -36,6 +42,7 @@ import ( "maps" "os/exec" "path" + "path/filepath" "slices" "sort" "strconv" @@ -67,7 +74,7 @@ type functionRow struct { // key tells one overload from another. func (r functionRow) key() string { - return r.Name + "\x00" + strconv.Itoa(r.NArg) + return r.Name + "/" + strconv.Itoa(r.NArg) } // Version reports the release the default shell is. @@ -206,38 +213,44 @@ func readShell(ctx context.Context, dir string, b build) (*shell, error) { return s, nil } -// generator accumulates the functions of every build, and remembers which -// names were reported so that the signatures table can be checked against -// the shells at the end. +// generator accumulates the functions of every build, reading their +// signatures from the amalgamation, and remembers which names were +// reported so that the lists in signatures.go can be checked against the +// shells at the end. type generator struct { ctx context.Context + src *source reported map[string]bool } -// functions turns rows into records, one per overload. Every row has to -// have a signature unless omitted; lenient says a row without one may be -// skipped instead when it is not built in, which is how the shell's own -// functions — edit, sha3, the bundled extensions — are kept out of the -// default build's list. A comparison with the default build has already -// removed them from an option's rows, so there nothing is skipped. -func (g *generator) functions(s *shell, lenient bool) ([]dialect.Function, error) { - var funcs []dialect.Function +// overload is one row of a shell's list with what was found out about it. +type overload struct { + row functionRow + kind string + sig signature +} + +// functions turns rows into records, one per overload. Every row has to be +// a function the amalgamation registers unless omitted. In the default +// build only what the library builds in counts: the rest is what the shell +// registers on top — edit, sha3, the extensions it bundles — which is not +// the dialect's business, and which a comparison with the default build has +// already removed from an option's rows. +func (g *generator) functions(s *shell, base bool) ([]dialect.Function, error) { + var overloads []overload seen := map[string]bool{} var missing []string for _, row := range s.rows { g.reported[row.Name] = true - if omitted[row.Name] || seen[row.key()] { + if omitted[row.Name] || seen[row.key()] || base && row.Builtin == 0 { continue } seen[row.key()] = true - sig, ok := signatures[row.Name] - if !ok { - if lenient && row.Builtin == 0 { - continue - } + sig, err := g.src.signature(row.Name) + if err != nil { if !seen[row.Name] { seen[row.Name] = true - missing = append(missing, row.Name) + missing = append(missing, err.Error()) } continue } @@ -245,20 +258,80 @@ func (g *generator) functions(s *shell, lenient bool) ([]dialect.Function, error if err != nil { return nil, err } - funcs = append(funcs, dialect.Function{ - Name: row.Name, - Kind: kind, - Args: sig.args(row.NArg), - Returns: sig.Returns, - Nullable: sig.Nullable, - }) + overloads = append(overloads, overload{row: row, kind: kind, sig: sig}) } if len(missing) > 0 { - return nil, fmt.Errorf("sqlite: no signature for function(s) %s of the %s build: add them to signatures.go", strings.Join(missing, ", "), s.build.name) + return nil, fmt.Errorf("sqlite: %s build: %s", s.build.name, strings.Join(missing, "; ")) + } + empty, err := overNoRows(g.ctx, s.binary, overloads) + if err != nil { + return nil, err + } + funcs := make([]dialect.Function, 0, len(overloads)) + for _, o := range overloads { + isNullable := nullable[o.row.Name] + if o.kind == "a" { + isNullable = empty[o.row.key()] == "null" + } + funcs = append(funcs, dialect.Function{ + Name: o.row.Name, + Kind: o.kind, + Args: o.sig.args(o.row.NArg), + Returns: o.sig.Returns, + Nullable: isNullable, + }) } return funcs, nil } +// overNoRows runs every aggregate over no rows and reports the type of +// what each returns, "null" for the ones — avg, max, group_concat — that +// return NULL when there is nothing to aggregate, and not count or total. +// The probes go through one shell process, each statement labelled with +// its overload so that the answers can be told apart. +func overNoRows(ctx context.Context, binary string, overloads []overload) (map[string]string, error) { + var script strings.Builder + for _, o := range overloads { + if o.kind != "a" { + continue + } + n := o.row.NArg + if n < 0 { + n = minArgs(n) + } + args := strings.TrimSuffix(strings.Repeat("x, ", n), ", ") + fmt.Fprintf(&script, "SELECT '%s', typeof(%s(%s)) FROM (SELECT NULL AS x) WHERE 0;\n", o.row.key(), o.row.Name, args) + } + results := map[string]string{} + if script.Len() == 0 { + return results, nil + } + cmd := exec.CommandContext(ctx, binary, "-list", ":memory:", script.String()) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) { + return nil, fmt.Errorf("sqlite3: %w", err) + } + } + for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") { + key, typ, ok := strings.Cut(line, "|") + if ok { + results[key] = typ + } + } + for _, o := range overloads { + if o.kind == "a" { + if _, ok := results[o.row.key()]; !ok { + return nil, fmt.Errorf("sqlite: cannot run %s over no rows: %s", o.row.Name, strings.TrimSpace(stderr.String())) + } + } + } + return results, nil +} + // added returns the rows of an option's shell that the default shell does // not have: what the option adds. func added(opt, base *shell) *shell { @@ -275,9 +348,14 @@ func added(opt, base *shell) *shell { return diff } -// Generate reads the dialect from the shells under dir. +// Generate reads the dialect from the shells under dir and the +// amalgamation they were built from. func Generate(ctx context.Context, dir string) (dialect.Files, error) { - g := &generator{ctx: ctx, reported: map[string]bool{}} + src, err := readSource(filepath.Join(dir, "src", "sqlite3.c")) + if err != nil { + return nil, err + } + g := &generator{ctx: ctx, src: src, reported: map[string]bool{}} all := builds() base, err := readShell(ctx, dir, all[0]) if err != nil { @@ -308,9 +386,11 @@ func Generate(ctx context.Context, dir string) (dialect.Files, error) { } } var stale []string - for name := range signatures { - if !g.reported[name] { - stale = append(stale, name) + for _, list := range []map[string]bool{omitted, nullable} { + for name := range list { + if !g.reported[name] { + stale = append(stale, name) + } } } if len(stale) > 0 {