From 0315e571019a645ad1ccbad092eee8aa45c58bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Oct 2026 17:58:50 +0200 Subject: [PATCH 1/5] fix(sql): date arithmetic follows the SQL engines; type checks wait for the column types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DATE - DATE is calendar days (BIGINT). TIMESTAMP - TIMESTAMP, or a DATE and a TIMESTAMP, is fractional days (DOUBLE). DATE Β± n moves the date by n days (a DATE, or a TIMESTAMP for a fractional n), and TIMESTAMP Β± n moves it by n days. date + date, a date under * / %, and n - date are refused by name, as PostgreSQL, DuckDB, Oracle, Snowflake, Trino and SQL Server refuse them. The rules hold in every venue: row SELECT, WHERE, per group, HAVING, SCRIPT AS computed columns, CREATE TABLE ... AS SELECT and the view calculation channel. Type rules move from parse time to validateResolved(), after the mapping is read. A statement is never refused on a guessed column type, and with no type to read nothing is refused. One comparison rule covers =, <, >, BETWEEN, IN, simple CASE, NULLIF and CASE conditions: a quoted literal that spells a number or a date is read as one, and DATE / TIMESTAMP compare on their declared types. It was checked against PostgreSQL on the statements the move first refused. A date field elasticsql did not declare is a TIMESTAMP everywhere, DESCRIBE included. A column declared DATE stays DATE. Also: - casts over arithmetic apply, and keep NULL; - GREATEST / LEAST compare dates; - index patterns and comma lists merge their mappings field by field, subqueries included, and DDL refreshes the cached types; - the clock and COUNT(d) run per group; - DATE_TRUNC(ts, DAY) compared with a date runs; - ES 6.8 compares a date column with a computed date. The normalisation is rendered for 6.8 only, through a per-major PainlessTarget. Breaking changes (release note): - date differences are days, not milliseconds; - DESCRIBE reports undeclared date fields as TIMESTAMP; - statements that answered wrong are now refused: MAX(d) / 1000, MAX(d) + MIN(d), n + s, WHERE id = n, and CASE conditions over incompatible types; - PainlessContext's arity, the bridge implicits and the ArithmeticExpression companion change: downstream projects must rebuild. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../sql/bridge/BucketScriptParams.scala | 92 + .../sql/bridge/ElasticAggregation.scala | 17 +- .../elastic/sql/bridge/ElasticBridge.scala | 5 +- .../elastic/sql/bridge/ElasticCriteria.scala | 5 +- .../sql/bridge/ElasticSearchRequest.scala | 7 +- .../elastic/sql/bridge/package.scala | 62 +- .../elastic/sql/BucketScriptParamsSpec.scala | 82 + .../elastic/sql/PainlessTargetSpec.scala | 99 + .../client/ElasticClientDelegator.scala | 3 + .../elastic/client/GatewayApi.scala | 15 + .../elastic/client/IndicesApi.scala | 135 ++ .../elastic/client/SearchApi.scala | 99 +- .../client/metrics/MetricsElasticClient.scala | 5 + .../client/RelationalClosureGuardSpec.scala | 28 +- .../elastic/client/SubqueryResolverSpec.scala | 65 + documentation/sql/ddl_statements.md | 7 +- documentation/sql/dql_statements.md | 15 +- documentation/sql/functions_conditional.md | 64 +- documentation/sql/functions_date_time.md | 20 + .../sql/functions_type_conversion.md | 5 + documentation/sql/operators.md | 156 +- .../sql/bridge/BucketScriptParams.scala | 92 + .../sql/bridge/ElasticAggregation.scala | 17 +- .../elastic/sql/bridge/ElasticBridge.scala | 5 +- .../elastic/sql/bridge/ElasticCriteria.scala | 5 +- .../sql/bridge/ElasticSearchRequest.scala | 7 +- .../elastic/sql/bridge/package.scala | 62 +- .../elastic/sql/BucketScriptParamsSpec.scala | 82 + .../elastic/sql/PainlessTargetSpec.scala | 99 + .../elastic/client/jest/JestSearchApi.scala | 12 +- .../client/JestClientDateArithmeticSpec.scala | 19 + .../client/rest/RestHighLevelClientApi.scala | 12 +- ...estHighLevelClientDateArithmeticSpec.scala | 19 + .../client/rest/RestHighLevelClientApi.scala | 11 +- ...estHighLevelClientDateArithmeticSpec.scala | 19 + .../elastic/client/java/JavaClientApi.scala | 9 +- .../client/JavaClientDateArithmeticSpec.scala | 19 + .../elastic/client/java/JavaClientApi.scala | 9 +- .../client/JavaClientDateArithmeticSpec.scala | 19 + .../softnetwork/elastic/schema/package.scala | 28 +- .../sql/function/aggregate/package.scala | 49 + .../elastic/sql/function/cond/package.scala | 553 +++++- .../sql/function/convert/package.scala | 43 +- .../elastic/sql/function/time/package.scala | 26 +- .../operator/math/ArithmeticExpression.scala | 700 ++++++- .../app/softnetwork/elastic/sql/package.scala | 267 ++- .../elastic/sql/parser/Validator.scala | 29 +- .../softnetwork/elastic/sql/query/Where.scala | 500 ++++- .../elastic/sql/query/package.scala | 103 +- .../elastic/sql/schema/package.scala | 102 ++ .../elastic/sql/type/SQLTypeUtils.scala | 53 +- .../elastic/sql/parser/ParserSpec.scala | 20 +- .../elastic/sql/parser/SetOperatorSpec.scala | 23 +- .../sql/perf/ResolutionCostProbe.scala | 529 ++++++ .../sql/query/DateArithmeticTypeSpec.scala | 432 +++++ .../query/HavingAggregateResolutionSpec.scala | 28 +- .../query/MixedTemporalComparisonSpec.scala | 78 +- .../elastic/sql/query/NullIfOperandSpec.scala | 43 +- .../sql/query/PainlessResidualsSpec.scala | 5 +- .../sql/query/PainlessTargetSpec.scala | 126 ++ .../sql/query/TemporalLiteralsSpec.scala | 19 +- .../sql/query/TypeCheckResolutionSpec.scala | 206 +++ .../sql/schema/DdlScriptSchemaSpec.scala | 8 +- .../sql/schema/IngestTemporalSpec.scala | 9 +- .../elastic/sql/schema/MergedSchemaSpec.scala | 179 ++ .../elastic/client/DateArithmeticSpec.scala | 1615 +++++++++++++++++ .../client/GatewayApiIntegrationSpec.scala | 45 +- 67 files changed, 6862 insertions(+), 459 deletions(-) create mode 100644 bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala create mode 100644 bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala create mode 100644 bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala create mode 100644 es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala create mode 100644 es6/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala create mode 100644 es6/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala create mode 100644 es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientDateArithmeticSpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala create mode 100644 es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala create mode 100644 es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/perf/ResolutionCostProbe.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessTargetSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/schema/MergedSchemaSpec.scala create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala new file mode 100644 index 000000000..a56d5cb5c --- /dev/null +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.bridge + +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import com.sksamuel.elastic4s.requests.searches.SearchRequest +import com.sksamuel.elastic4s.requests.searches.aggs.{AbstractAggregation, Aggregation} +import com.sksamuel.elastic4s.requests.searches.aggs.pipeline.BucketScriptPipelineAgg + +import scala.jdk.CollectionConverters._ + +/** The PARAMS of a `bucket_script`'s script, which the elastic4s body builder drops. + * + * πŸ”΄ elastic4s writes a `bucket_script` as `"script": ""` -- the bare source string + * (`BucketScriptPipelineAggBuilder`, the same in elastic4s 6.7, 7.17, 8.18 and 9.0) -- while a + * `bucket_selector` gets the whole script object. So a per-group calculation reading the request + * clock, `CURRENT_DATE - MAX(d)` or `NOW() - MAX(ts)`, was sent WITHOUT the `__now__` its script + * reads, and Elasticsearch failed it; the same clock in a HAVING worked. MEASURED on 8.18.3. + * + * The script is sent as the object Elasticsearch documents -- `source`, `lang`, `params` -- for + * exactly the `bucket_script`s that carry params, so the source stays the SAME text from one + * request to the next and Elasticsearch compiles it once: inlining the clock instead would compile + * a new script on every request, against the cluster's compilation rate limit. Every other body is + * returned untouched, unparsed. + */ +object BucketScriptParams { + + private val mapper = new ObjectMapper() + + /** The params of every `bucket_script` of `aggs` that carries some, by aggregation name, at any + * depth. + */ + private def withParams(aggs: Seq[AbstractAggregation]): Map[String, Map[String, Any]] = + aggs.flatMap { + case script: BucketScriptPipelineAgg if script.script.params.nonEmpty => + Seq(script.name -> script.script.params) + case parent: Aggregation => withParams(parent.subaggs).toSeq + case _ => Nil + }.toMap + + /** `body`, the JSON of `search`, with the params of its `bucket_script`s bound. */ + def bind(body: String, search: SearchRequest): String = { + val params = withParams(search.aggs) + if (params.isEmpty) body + else { + val root = mapper.readTree(body) + bind(root, params) + mapper.writeValueAsString(root) + } + } + + private def bind(node: JsonNode, params: Map[String, Map[String, Any]]): Unit = + node match { + case obj: ObjectNode => + obj.fields().asScala.toList.foreach { entry => + (params.get(entry.getKey), entry.getValue) match { + case (Some(values), agg: ObjectNode) if agg.has("bucket_script") => + agg.get("bucket_script") match { + case bucketScript: ObjectNode if bucketScript.path("script").isTextual => + val script = mapper.createObjectNode() + script.put("source", bucketScript.get("script").asText()) + script.put("lang", "painless") + val bound = script.putObject("params") + values.foreach { case (name, value) => + bound.set[JsonNode](name, mapper.valueToTree[JsonNode](value)) + } + bucketScript.set[JsonNode]("script", script) + case _ => + } + case _ => + } + bind(entry.getValue, params) + } + case array: ArrayNode => array.elements().asScala.foreach(bind(_, params)) + case _ => + } +} diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index d441615cc..0dc21aad3 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, @@ -162,7 +162,8 @@ object ElasticAggregation { allAggregations: Map[String, SQLAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -218,14 +219,14 @@ object ElasticAggregation { buildScript: (String, Script) => Aggregation ): Aggregation = { if (transformFuncs.nonEmpty) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val scriptSrc = identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val scriptSrc = th.identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) @@ -475,7 +476,8 @@ object ElasticAggregation { allElasticAggregations: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = @@ -492,7 +494,7 @@ object ElasticAggregation { val aggScript = if (!bucket.isBucketScript && bucket.shouldBeScripted) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val painless = bucket.painless(Some(context)) Some(now(Script(s"$context$painless").lang("painless"))) } else { @@ -720,7 +722,8 @@ object ElasticAggregation { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): Option[Aggregation] = request.having.flatMap(_.criteria) match { case Some(criteria) if request.wholeTableHaving && aggs.nonEmpty => diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 37f64eefd..667992081 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.operator.AND import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -48,7 +48,8 @@ case class ElasticBridge(filter: ElasticFilter) { currentQuery: Option[ElasticBoolQuery] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { filter match { case boolQuery: ElasticBoolQuery => diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index 89e86e969..b196a3923 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.requests.searches.queries.Query @@ -24,7 +24,8 @@ case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { val query = criteria.boolQuery.copy(group = group) query diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index 3337d604e..4bad08ea7 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -49,5 +49,10 @@ case class ElasticSearchRequest( def hasTransformExtendedStats: Boolean = SearchBodySerializer.hasTransformExtendedStats(search) def query: String = - serializer.serialize(search).replace("\"version\":true,", "") /*FIXME*/ + // the params of a per-group calculation's script, which the body builder drops: the request + // clock `CURRENT_DATE - MAX(d)` reads (`BucketScriptParams`) + BucketScriptParams.bind( + serializer.serialize(search).replace("\"version\":true,", ""), /*FIXME*/ + search + ) } diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index c680ad446..d3a5a2851 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -77,7 +77,8 @@ package object bridge { innerHitsName: String )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { @@ -156,7 +157,8 @@ package object bridge { request: SingleSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -184,7 +186,8 @@ package object bridge { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) @@ -259,7 +262,8 @@ package object bridge { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations @@ -472,10 +476,16 @@ package object bridge { // SearchBodySerializer in implicit scope of this conversion; with none in scope the default // argument applies (the one-argument elastic4s builder, refusing a transform-bearing // extended_stats). Same defaulted-implicit pattern as `contextType`. + // + // `target` is the same seam for SCRIPTS: the Elasticsearch major the client module talks to + // (`PainlessTarget`), which every conversion below hands to the `PainlessContext`s it creates. + // With none in scope a script renders as for the majors from 7 on; the ES 6 client modules + // give 6, whose query scripts compare a `date` doc value differently. implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query, - serializer: SearchBodySerializer = SearchBodySerializer.Default + serializer: SearchBodySerializer = SearchBodySerializer.Default, + target: PainlessTarget = PainlessTarget.Default ): ElasticSearchRequest = ElasticSearchRequest( request.sql, @@ -537,7 +547,8 @@ package object bridge { request: SingleSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): SearchRequest = { import request._ @@ -600,7 +611,7 @@ package object bridge { case Nil => _search case _ => _search scriptfields scriptFields.map { field => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = field.painless(Some(context)) scriptField( field.scriptName, @@ -621,7 +632,7 @@ package object bridge { case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map { sort => if (sort.isScriptSort) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val painless = sort.field.painless(Some(context)) val painlessScript = s"$context$painless" val script = @@ -687,7 +698,8 @@ package object bridge { request: MultiSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): MultiSearchRequest = { // Story 22.6 β€” a NAMED backstop, never a silent `UNION ALL`. Only `UNION ALL` is an // Elasticsearch `_msearch`; every other set operator needs the relational engine and is @@ -741,9 +753,10 @@ package object bridge { private[bridge] def scriptQueryOf(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): Query = { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = criteria.painless(Some(context)) scriptQuery( now(Script(script = s"$context$script").lang("painless").scriptType("source")) @@ -757,7 +770,8 @@ package object bridge { implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { import expression._ if (isAggregation) @@ -982,7 +996,7 @@ package object bridge { case _ => scriptQueryOf(expression) } case _ => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = painless(Some(context)) scriptQuery( now( @@ -993,7 +1007,7 @@ package object bridge { ) } case _ => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = painless(Some(context)) scriptQuery( now( @@ -1037,7 +1051,8 @@ package object bridge { implicit def inToQuery[R, T <: Value[R]](in: InExpr[R, T])(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { if (requiresScript(in.identifier)) return scriptQueryOf(in) import in._ @@ -1062,7 +1077,8 @@ package object bridge { between: BetweenExpr )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { import between._ // ⚠️ NOT the two-operand form: `BetweenExpr.maybeValue` is always a `FromTo`, never an @@ -1215,14 +1231,16 @@ package object bridge { implicit def criteriaToQuery(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { ElasticCriteria(criteria).asQuery() } implicit def criteriaToNode(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): JsonNode = { queryToJson(criteriaToQuery(criteria)) } @@ -1240,7 +1258,8 @@ package object bridge { )(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query, - serializer: SearchBodySerializer = SearchBodySerializer.Default + serializer: SearchBodySerializer = SearchBodySerializer.Default, + target: PainlessTarget = PainlessTarget.Default ): Seq[ElasticAggregation] = { import query._ statement @@ -1280,7 +1299,10 @@ package object bridge { aggregation.copy( sources = l.sources, query = Some( - serializer.serialize(body).replace("\"version\":true,", "") /*FIXME*/ + BucketScriptParams.bind( + serializer.serialize(body).replace("\"version\":true,", ""), /*FIXME*/ + body + ) ) ) }) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala new file mode 100644 index 000000000..f6927d6e9 --- /dev/null +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime +import scala.jdk.CollectionConverters._ + +/** A per-group calculation that reads the request clock (`CURRENT_DATE - MAX(d)`, `NOW() - + * MAX(ts)`) is sent WITH the clock: its `bucket_script` carries the script object -- `source`, + * `lang`, `params.__now__` -- where the elastic4s builder writes the bare source string and drops + * the params (`BucketScriptParams`). Every other `bucket_script` is sent as it was. + * + * The clock is a PARAM, not text spliced into the source: the source is the same from one request + * to the next, so Elasticsearch compiles it once. + */ +class BucketScriptParamsSpec extends AnyFlatSpec with Matchers { + + import scala.language.implicitConversions + + private val now: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + implicit def timestamp: Long = now + + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) + case other => throw new IllegalArgumentException(s"Not a single search: $other") + } + + private val mapper = new ObjectMapper() + + private def bucketScripts(sql: String): Seq[JsonNode] = { + val request: ElasticSearchRequest = SelectStatement(sql) + def all(node: JsonNode): Seq[JsonNode] = + Option(node.get("bucket_script")).toSeq ++ node.elements().asScala.toSeq.flatMap(all) + all(mapper.readTree(request.query)) + } + + "a per-group calculation reading the clock" should "send the clock as a script param" in { + Seq( + "SELECT g, CURRENT_DATE - MAX(d) AS x FROM t GROUP BY g", + "SELECT g, NOW() - MAX(ts) AS x FROM t GROUP BY g", + "SELECT g, DATEDIFF(CURRENT_DATE, MAX(d)) AS x FROM t GROUP BY g" + ).foreach { sql => + withClue(s"[$sql] ") { + val scripts = bucketScripts(sql) + scripts should have size 1 + val script = scripts.head.get("script") + script.isObject shouldBe true + script.get("source").asText() should include("params.__now__") + script.get("lang").asText() shouldBe "painless" + script.get("params").get("__now__").asLong() shouldBe now + } + } + } + + "a per-group calculation without the clock" should "be sent as it always was" in { + val scripts = bucketScripts("SELECT g, MAX(d) - MIN(d) AS x FROM t GROUP BY g") + scripts should have size 1 + scripts.head.get("script").isTextual shouldBe true + } +} diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala new file mode 100644 index 000000000..e385013b6 --- /dev/null +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime + +/** The Elasticsearch major the client module puts in implicit scope (`PainlessTarget`) reaches + * EVERY query script the bridge's conversions create: a WHERE, a script field, an aggregation over + * an expression. + * + * The probe is a comparison between a `date` doc value and another date, which Elasticsearch 6.8 + * alone needs normalised to UTC: with 6 in scope the body carries the normalisation, with any + * other major -- or none -- it is the very body no target produces. + */ +class PainlessTargetSpec extends AnyFlatSpec with Matchers { + + private val now: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + private val normalisation = ".withZoneSameInstant(ZoneId.of('Z'))" + + private val schema = Parser( + "CREATE TABLE t (id KEYWORD, g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, n INT, PRIMARY KEY (id))" + ) match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"$other") + } + + private def resolved(sql: String): SingleSearch = + Parser(sql) match { + case Right(ss: SingleSearch) => + val r = ss.update(Some(schema)) + r.validateResolved() shouldBe Right(()) + r + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + /** The body `requestToElasticSearchRequest` sends. (`sqlQueryToAggregations`, the other door, + * parses its statement from text: no schema, so no type to normalise a comparison by.) + */ + private def body(sql: String, target: PainlessTarget): String = + requestToElasticSearchRequest(resolved(sql))( + now, + PainlessContextType.Query, + SearchBodySerializer.Default, + target + ).query + + private val venues: Seq[String] = Seq( + // a WHERE: a script query, alone and inside a bool + "SELECT id FROM t WHERE CAST(ts AS DATE) = d", + "SELECT id FROM t WHERE CURRENT_DATE > d AND n > 1", + // a script field + "SELECT id, CASE WHEN CURRENT_DATE > d THEN 1 ELSE 0 END AS c FROM t", + // an aggregation over an expression + "SELECT SUM(CASE WHEN CURRENT_DATE > d THEN 1 ELSE 0 END) AS s FROM t", + // the WHERE of an aggregate query + "SELECT g, COUNT(*) AS c FROM t WHERE CAST(ts AS DATE) = d GROUP BY g" + ) + + "a script comparing a date doc value" should + "carry the UTC normalisation when the client module gives Elasticsearch 6" in { + venues.foreach { sql => + withClue(s"[$sql] ") { + body(sql, PainlessTarget(6)) should include(normalisation) + } + } + } + + it should "be the body no target produces for Elasticsearch 7, 8 and 9" in { + venues.foreach { sql => + withClue(s"[$sql] ") { + val default = body(sql, PainlessTarget.Default) + default should not include normalisation + Seq(7, 8, 9).foreach(major => body(sql, PainlessTarget(major)) shouldBe default) + body(sql, PainlessTarget(6)).replace(normalisation, "") shouldBe default + } + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 63cf62945..4d4cb1185 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -327,6 +327,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def loadSchema(index: String): ElasticResult[Schema] = delegate.loadSchema(index) + override private[client] def loadPatternSchema(pattern: String): ElasticResult[Schema] = + delegate.loadPatternSchema(pattern) + override def invalidateSchema(index: String): Unit = delegate.invalidateSchema(index) override def invalidateAllSchemas(): Unit = delegate.invalidateAllSchemas() diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index cf027019d..1640f9e02 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -1065,6 +1065,21 @@ class TableExecutor( case Some(from) => api.loadSchema(from) match { case ElasticSuccess(fromSchema) => + // πŸ”΄ A TYPE refusal (`MAX(d) / 1000`, `d + d2`, `n + 'x'`) is asked HERE, once + // the source's column types are known, before the target is created or -- `OR + // REPLACE` -- the existing one deleted. Its SELECT is refused anyway when it runs + // (`validateResolved`), but that runs after the destructive step, and since no + // type is refused when a statement is parsed (the lead's ruling of 2026-10-05), + // a type error would otherwise lose the user's index and put nothing back. + single.update(Some(fromSchema)).validateResolved().left.foreach { reason => + val error = ElasticError( + message = reason, + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Left(error) + } // we update the schema based on the DQL select clause fromSchema .mergeWithSearch(single) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 8389fd858..c629bd280 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -34,6 +34,7 @@ import app.softnetwork.elastic.sql.query.{ Update } import app.softnetwork.elastic.sql.schema.{ + mergedSchema, GenericProcessor, IngestPipeline, Schema, @@ -250,6 +251,8 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { // #238 β€” a shard count cached for an expression this index is the stem of (a name probed // before it existed, `orders*` before the load) must not survive the creation invalidateShardCounts(Some(index)) + // ...nor a pattern's merged schema that did not read this index yet + invalidatePatternsOf(index) logger.info(s"βœ… Index '$index' created successfully") success case success @ ElasticSuccess(_) => @@ -305,6 +308,91 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { result } + /** The schema an index PATTERN or a comma list answers with (`FROM logs-*`, `FROM a,b`): the + * columns of every index it matches, merged field by field (`schema.mergedSchema`), so a + * statement over a pattern is typed as one over a single index is. A field the indices map + * differently is left untyped: nothing is refused on it, and Elasticsearch answers for it. + * + * ONE mapping read per pattern -- `GET ` answers every matched index at once -- through + * the SAME cache as [[loadSchema]], keyed by the pattern, so a hit costs no round trip. A + * pattern that matches no index answers a `404` failure: no schema is attached, and `SearchApi` + * remembers the miss for its TTL as it does an unknown index's. + * + * ⚠️ A merged schema carries the columns and nothing else -- no settings, no pipeline, no + * primary key: it types a QUERY, never a DDL statement. An elasticsql DDL statement that + * creates, alters or drops an index the pattern matches drops the entry, as it refreshes the + * single index's ([[invalidatePatternsOf]]): the next statement reads the members again. + */ + private[client] def loadPatternSchema(pattern: String): ElasticResult[Schema] = { + val now = System.currentTimeMillis() + var result: ElasticResult[Schema] = null + var fetched = false + schemaCache.compute( + pattern, + (_, existing) => + existing match { + case entry: CachedSchema if !entry.isExpired(now) => + logger.debug(s"πŸ“¦ Schema cache hit for '$pattern'") + result = ElasticSuccess(entry.schema) + entry + case _ => + fetched = true + fetchPatternSchema(pattern) match { + case success @ ElasticSuccess(schema) => + result = success + CachedSchema(schema, now, schemaCacheTtlMs) + case failure => + result = failure + existing + } + } + ) + if (fetched && schemaCache.size() > schemaCachePurgeThreshold) purgeExpiredSchemas(now) + result + } + + private def fetchPatternSchema(pattern: String): ElasticResult[Schema] = { + def notFound(message: String): ElasticResult[Schema] = + ElasticFailure( + ElasticError( + message = message, + statusCode = Some(404), + index = Some(pattern), + operation = Some("loadSchema") + ) + ) + pattern + .split(",") + .map(_.trim) + .find(part => validateIndexName(part, pattern = true).isDefined) match { + case Some(part) => return notFound(s"Invalid index pattern '$part' in '$pattern'") + case None => + } + executeGetIndex(pattern) match { + case ElasticSuccess(Some(json)) => + val root = mapper.readTree(json) + val members: Seq[(String, JsonNode)] = + if (root.has(pattern)) Seq(pattern -> root.get(pattern)) else Index.indexDocuments(root) + if (members.isEmpty) notFound(s"No index matches '$pattern'") + else { + val (schema, disagreements) = mergedSchema( + pattern, + members.map { case (name, document) => name -> Index(name, document).schema } + ) + if (disagreements.nonEmpty) + logger.info( + s"⚠️ The indices of '$pattern' disagree on ${disagreements.mkString(", ")}: " + + "statements are typed by Elasticsearch on those fields" + ) + ElasticSuccess(schema) + } + case ElasticSuccess(None) => notFound(s"No index matches '$pattern'") + case ElasticFailure(error) if error.statusCode.contains(404) => + notFound(s"No index matches '$pattern'") + case ElasticFailure(error) => ElasticFailure(error) + } + } + /** Drop every expired entry β€” each on ITS own clock, through the single `isExpired` rule β€” plus * the alias->target rows for exactly those entries. * @@ -362,10 +450,31 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { logger.debug(s"πŸ“¦ Schema cache invalidated for alias '$alias' (target '$index')") } + /** Drop every cached PATTERN schema ([[loadPatternSchema]]) whose pattern matches `index`: its + * merged columns read `index`'s mapping, so an elasticsql DDL statement that creates, alters or + * drops `index` changes the pattern's answer exactly as it changes the single index's -- and, + * like the single index's, it is read again on the next statement rather than served stale for + * its TTL. A pattern this one does not match keeps its entry. + */ + private def invalidatePatternsOf(index: String): Unit = + // (a pattern is never the subject of a DDL statement: its own entry, written through + // `updateSchema(, …)`, is not one of the entries an index's change invalidates) + if (!IndicesApi.isPattern(index)) + schemaCache + .keySet() + .asScala + .filter(key => IndicesApi.isPattern(key) && IndicesApi.patternMatches(key, index)) + .toList + .foreach { pattern => + schemaCache.remove(pattern) + logger.debug(s"πŸ“¦ Schema cache invalidated for pattern '$pattern' (index '$index')") + } + def updateSchema(index: String, schema: Schema): Unit = { val now = System.currentTimeMillis() schemaCache.put(index, CachedSchema(schema, now, resolveTtlMs(index, schema))) invalidateAliasesOf(index) + invalidatePatternsOf(index) // #238 β€” ALTER TABLE may have reindexed into a different shard count invalidateShardCounts(Some(index)) logger.debug(s"πŸ“¦ Schema cache updated for '$index'") @@ -375,6 +484,7 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { schemaCache.remove(index) val _ = schemaAliasTargets.remove(index) invalidateAliasesOf(index) + invalidatePatternsOf(index) invalidateShardCounts(Some(index)) // #238 β€” the sliced-paging shard counts follow the schema logger.info(s"πŸ—‘οΈ Schema cache invalidated for '$index'") } @@ -2158,3 +2268,28 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { refresh: Boolean ): ElasticResult[Long] } + +object IndicesApi { + + /** A FROM expression that names several indices: a `*` wildcard or a comma list. */ + private[client] def isPattern(source: String): Boolean = + source.contains("*") || source.contains(",") + + /** Does the index `pattern` -- a comma list of names and `*` wildcards, as Elasticsearch reads + * one -- match `index`? An EXCLUSION (`-logs-old`) is not read: a pattern that matches `index` + * through another part is matched, and one matched only through a part an exclusion would remove + * is matched too -- reading a mapping again costs one request, serving it stale a wrong type. + */ + private[client] def patternMatches(pattern: String, index: String): Boolean = + pattern.split(",").iterator.map(_.trim).filter(p => p.nonEmpty && !p.startsWith("-")).exists { + part => + part + .split("\\*", -1) + .map(java.util.regex.Pattern.quote) + .mkString(".*") + .r + .pattern + .matcher(index) + .matches() + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 8b0cad2a4..c264deb36 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -95,15 +95,16 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC * without a temporal candidate β€” `baseType` is needed for every statement, and skipping would * have left the attach dead for nearly every query. Cost: one lookup per index per cache TTL. * - * Schema-absent path = verbatim (#276 AD-S4-1.2): the lookup is skipped when the FROM does not - * name exactly ONE concrete index (several sources, a `*` wildcard or a `,` list), when this - * client is not an [[IndicesApi]], and when the schema cannot be loaded (alias without a - * template, unknown index, cluster error or a thrown lookup) - * -- the literal is then forwarded exactly as before. The schema comes from - * [[IndicesApi.loadSchema]] 's 5-minute cache, so a miss costs one `GET ` per index per - * TTL. A literal that cannot be a date under a fully-understood mapping format is a `400` that - * names the literal and the field, instead of Elasticsearch's - * `search_phase_execution_exception`. + * Schema-absent path = verbatim (#276 AD-S4-1.2): the lookup is skipped when this client is not + * an [[IndicesApi]]; no schema is attached when it cannot be loaded (alias without a template, + * unknown index, cluster error or a thrown lookup) -- the literal is then forwarded exactly as + * before, and no TYPE is refused: Elasticsearch answers. A field the indices of a `*` pattern or + * a list of sources disagree on is attached untyped, with the same effect for that field alone. + * The schema comes from [[IndicesApi.loadSchema]] 's 5-minute cache (a pattern's, merged, from + * [[IndicesApi.loadPatternSchema]] 's, under the same key rules), so a miss costs one `GET + * ` per index -- or per pattern -- per TTL. A literal that cannot be a date under a + * fully-understood mapping format is a `400` that names the literal and the field, instead of + * Elasticsearch's `search_phase_execution_exception`. * * `loadSchema` caches successes only, so a source it cannot resolve -- an ALIAS on the es8/es9 * clients (`executeGetIndex` looks the alias up as a key and finds nothing), an unknown index @@ -177,11 +178,17 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC // That was correct while the only job was rewriting those literals, and is WRONG now that the // schema is also attached to the AST: almost no statement carries a temporal WHERE literal, // so the lookup would be skipped for almost every query and `baseType` would stay `Any`. + // An index PATTERN or a comma list -- `FROM logs-*`, `FROM a, b` (several sources searched as + // one) -- answers like the single index: the schema of the indices it matches, merged field by + // field (`IndicesApi.loadPatternSchema`), one mapping read per list, cached. Before, it attached + // NO schema, so `FROM zwild_*` typed every column `Any` and answered `MAX(d) - MIN(d)` in + // milliseconds where the single index answered days. phaseOne.sources.distinct match { - case Seq(source) if !source.contains("*") && !source.contains(",") => - this match { - case indices: IndicesApi if !schemaMissed(source) => - Try(indices.loadSchema(source)) match { + case sources if sources.nonEmpty => + val source = sources.mkString(",") + lookupSchema(sources) match { + case Some(lookup) => + lookup match { case Success(ElasticSuccess(schema)) => schemaMisses.remove(source) TemporalLiterals(phaseOne, schema) match { @@ -254,20 +261,41 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC ) ElasticResult.success(phaseOne) } - case _ => ElasticResult.success(phaseOne) + case None => ElasticResult.success(phaseOne) } case _ => ElasticResult.success(phaseOne) } } + /** THE schema lookup for a statement's sources, for every consumer -- the attach + * ([[resolveWithSchema]]) and the scope of a subquery ([[resolvedSchema]]) alike (#292): ONE + * concrete index reads its own mapping ([[IndicesApi.loadSchema]]); an index PATTERN, a comma + * list or several sources read the mapping of every index they match, merged field by field + * ([[IndicesApi.loadPatternSchema]]), under the key the miss is remembered by -- the sources + * joined with commas. + * + * `None` when no lookup is made: a client that is not an [[IndicesApi]], or a remembered miss. + */ + private def lookupSchema(sources: Seq[String]): Option[Try[ElasticResult[Schema]]] = { + val source = sources.mkString(",") + this match { + case indices: IndicesApi if !schemaMissed(source) => + val pattern = sources.size > 1 || source.contains("*") || source.contains(",") + Some(Try(if (pattern) indices.loadPatternSchema(source) else indices.loadSchema(source))) + case _ => None + } + } + /** The mapped type of a column PROJECTED BY AN INNER STATEMENT, when it can be known for free * (story 22.2). * - * `None` under every #306 skip condition β€” several sources, a wildcard, a client that is not an - * `IndicesApi`, a remembered miss, a failed or absent schema. The caller (`SubqueryResolver`) - * reads it for ONE decision: a `text` column cannot carry a terms aggregation, so mode P is - * declined for it. An unknown mapping therefore keeps the cheap default, and an Elasticsearch - * refusal propagates loudly with the inner statement's own message. + * `None` for a client that is not an `IndicesApi`, a remembered miss, a failed or absent schema; + * an index pattern or a list of sources answers from its merged schema, the one + * [[resolveWithSchema]] attaches (a field its indices disagree on is `Any`, which declines + * nothing). The caller (`SubqueryResolver`) reads it for ONE decision: a `text` column cannot + * carry a terms aggregation, so mode P is declined for it. An unknown mapping therefore keeps + * the cheap default, and an Elasticsearch refusal propagates loudly with the inner statement's + * own message. */ private[client] def innerColumnType(inner: SingleSearch, name: String): Option[SQLType] = resolvedSchema(inner).flatMap(_.find(name).map(_.dataType)) @@ -366,16 +394,16 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC outerMapping.name ) + /** The schema of a statement's sources, when it can be read: [[lookupSchema]], the one the attach + * reads -- an index pattern or a list of sources included (their merged schema), so a subquery + * over `logs-*` is scoped as one over a single index is. + */ private def resolvedSchema(s: SingleSearch): Option[Schema] = s.sources.distinct match { - case Seq(source) if !source.contains("*") && !source.contains(",") => - this match { - case indices: IndicesApi if !schemaMissed(source) => - Try(indices.loadSchema(source)) match { - case Success(ElasticSuccess(schema)) => Some(schema) - case _ => None - } - case _ => None + case sources if sources.nonEmpty => + lookupSchema(sources).flatMap { + case Success(ElasticSuccess(schema)) => Some(schema) + case _ => None } case _ => None } @@ -452,11 +480,18 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC case (failure, _) => failure } match { case ElasticSuccess(resolved) => - // Story 22.6 β€” with each branch's schema attached, bare columns are TYPED: re-run the - // pairwise compatibility check that passed vacuously at parse time (where a bare column is - // `Any`). This is the only type guard the family has β€” DuckDB casts implicitly across - // branches, so the relational engine is never a backstop either. - MultiSearch.branchTypes(resolved) match { + // Story 22.6 β€” with each branch's schema attached, bare columns are TYPED: this is where + // the pairwise compatibility check is asked. This is the only type guard the family has β€” + // DuckDB casts implicitly across branches, so the relational engine is never a backstop + // either. + // + // πŸ”΄ The lead's ruling of 2026-10-05: no TYPE refusal before the column types are known. + // Parsing no longer asks it at all, and with NO schema attached to any branch (a client + // that is not an `IndicesApi`, mappings that cannot be read or that disagree) nothing is + // refused on types: Elasticsearch answers, exactly as for a single statement. A branch + // without a schema keeps its bare columns `Any`, which pass. + (if (resolved.exists(_.schema.isDefined)) MultiSearch.branchTypes(resolved) + else Right(())) match { case Left(reason) => ElasticResult.failure( ElasticError( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 420e2d76f..3f43458ba 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -317,6 +317,11 @@ class MetricsElasticClient( delegate.loadSchema(index) } + override private[client] def loadPatternSchema(pattern: String): ElasticResult[Schema] = + measureResult("loadSchema", Some(pattern.indices.mkString(","))) { + delegate.loadPatternSchema(pattern) + } + override def invalidateSchema(index: String): Unit = delegate.invalidateSchema(index) override def invalidateAllSchemas(): Unit = delegate.invalidateAllSchemas() diff --git a/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala index b31b43ce3..03602ae19 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.sql.query.{ ctesPresent, relationalClosureRequired, setOperationsPresent, + MultiSearch, SearchStatement, SingleSearch } @@ -364,15 +365,26 @@ class RelationalClosureGuardSpec extends AnyFlatSpec with Matchers { } } - /** Story 22.6 AD-4's second half: at parse time a bare column is `SQLTypes.Any` and passes, so - * the check must RE-RUN once schemas are attached. `NopeClientApi` attaches none, so this row - * pins the LITERAL case the parse-time half already rejects plus the seam's own wiring; the - * schema-attached case is covered by `TemporalLiteralSearchSpec`'s stub-schema idiom. + /** Story 22.6 AD-4: the branch TYPE check. The lead's ruling of 2026-10-05 -- no TYPE refusal + * before the column types are known -- took it off parsing entirely: the seam asks it once a + * branch's schema is attached, which `TemporalLiteralSearchSpec`'s stub-schema idiom pins, with + * nothing sent. `NopeClientApi` attaches no schema, so this row pins the rest: the statement + * PARSES, the rule the seam asks refuses it by name, and with no schema at all nothing is + * refused on types -- Elasticsearch answers. */ - it should "reject a typeable branch mismatch before any request is built" in { - Parser("SELECT 1 AS n FROM t UNION ALL SELECT 'a' AS n FROM u").swap.toOption - .map(_.msg) - .getOrElse("") should include("compatible types at column 1") + it should "ask the branch type check after resolution, and refuse nothing on types without a schema" in { + val sql = "SELECT 1 AS n FROM t UNION ALL SELECT 'a' AS n FROM u" + Parser(sql) match { + case Right(multi: MultiSearch) => + MultiSearch.branchTypes(multi.requests).swap.getOrElse("") should include( + "compatible types at column 1" + ) + case other => fail(s"[$sql] parsed as $other: no TYPE is refused at parse") + } + client().search(searchStatement(sql)) match { + case ElasticFailure(e) => fail(s"refused without a schema: ${e.message}") + case ElasticSuccess(_) => succeed + } } it should "refuse an INSERT ... SELECT carrying a set operation" in { diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala index b04bf19ed..ea78b53e2 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala @@ -633,6 +633,71 @@ class SubqueryResolverSpec extends AnyFlatSpec with Matchers { msg should include("must be QUALIFIED") } + /** A subquery over an index PATTERN is scoped by the schema the pattern is attached with -- the + * merged mapping of the indices it matches, through the ONE lookup the attach reads (#292) -- + * where it used to be read for a single concrete index only: over a pattern the bare-name rule + * passed vacuously, and a correlated reference ran as an uncorrelated one. A field the indices + * disagree on stays a column of the pattern (untyped), so it is never mistaken for an outer one. + */ + it should "scope a subquery over an index pattern by the pattern's merged mapping" in { + val client = seededClient() + client.updateSchema( + "members", + schema.Table( + "members", + columns = List( + schema.Column("id", SQLTypes.Keyword), + schema.Column("tier", SQLTypes.Keyword), + schema.Column("level", SQLTypes.Keyword) + ) + ) + ) + val (merged, disagreements) = schema.mergedSchema( + "orders_*", + Seq( + "orders_a" -> schema.Table( + "orders_a", + columns = List( + schema.Column("id", SQLTypes.Keyword), + schema.Column("cid", SQLTypes.Keyword), + schema.Column("tier", SQLTypes.Keyword) + ) + ), + "orders_b" -> schema.Table( + "orders_b", + columns = List( + schema.Column("id", SQLTypes.Keyword), + schema.Column("cid", SQLTypes.Keyword), + schema.Column("tier", SQLTypes.Int) + ) + ) + ) + ) + disagreements shouldBe Seq("'tier' (INT, KEYWORD)") + // the cache key a pattern's schema is read under is the pattern itself + client.updateSchema("orders_*", merged) + + // a name the pattern does not map while the outer index does: a correlated reference + val correlated = single( + "SELECT id FROM members m WHERE id IN (SELECT cid FROM orders_* o WHERE level = 'gold')" + ) + val correlatedInner = correlated.whereSubqueries.head.inner.getOrElse(fail("no body")) + client.innerColumnType(correlatedInner, "cid") shouldBe Some(SQLTypes.Keyword) + client + .scopeCorrelation(correlated, correlatedInner) + .getOrElse(fail("expected a rejection")) should include( + "'level' is not a column of 'orders_*'" + ) + + // the field the indices disagree on is still the pattern's own column, untyped + val own = single( + "SELECT id FROM members m WHERE id IN (SELECT cid FROM orders_* o WHERE tier = 'gold')" + ) + val ownInner = own.whereSubqueries.head.inner.getOrElse(fail("no body")) + client.innerColumnType(ownInner, "tier") shouldBe Some(SQLTypes.Any) + client.scopeCorrelation(own, ownInner) shouldBe None + } + /** πŸ”΄ Story 22.3b's DEFENSIVE arm. `GatewayApi.run(statement: Statement)` never calls * `validate()` and the seam's closure guard reads the STATEMENT, so a node handed straight to * this object must never have its correlated body executed as if it were self-contained. The diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index 3c032b3eb..5f6cdc415 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -545,9 +545,10 @@ value of the incoming document, not the temporal object a query sees, so `YEAR(c `DATE_TRUNC(...)`, `DATE_ADD(...)`, `DATE_FORMAT(...)` and `DATE_DIFF(...)` parse it first. Both shapes Elasticsearch accepts into a `date` field work β€” an ISO string (`"2025-01-10"`, `"2025-01-10 14:30:00"`, `"2025-01-10T14:30:00Z"`) and epoch milliseconds (`1736467200000`) β€” and -the column's DECLARED type chooses how a string is read, so a `DATE` column expects a date-only -spelling. `CURRENT_DATE`, `CURRENT_TIMESTAMP`, `NOW` and `TODAY` are the ingest time of the -document. +the column's DECLARED type chooses how a string is read: a `DATE` column reads a date-only +spelling as that day, and a date-time spelling (which Elasticsearch accepts into it) as the UTC day +of the instant it spells. `CURRENT_DATE`, `CURRENT_TIMESTAMP`, `NOW` and `TODAY` are the ingest time +of the document. > A document that does not carry the source field leaves the computed column absent, as before. diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index 1e5db4dc3..caa6fa35b 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -645,7 +645,9 @@ The SQL Gateway executes `UNION ALL` using **Elasticsearch Multi‑Search (`_mse - `UNION ALL` does **not** sort or deduplicate results. - Column names in the final output are taken from the **first SELECT**. - All subsequent SELECTs must produce columns with the **same names**. -- Type mismatches should result in a validation error before execution. (⚠️ not implemented yet) +- A type mismatch between branches is refused before execution, once each branch's mapping is read + (`SELECT id FROM a UNION ALL SELECT amount FROM b`, a `keyword` against a number); with no + mapping to read, nothing is refused on types and Elasticsearch answers. --- @@ -1400,11 +1402,14 @@ GREATEST(e1, e2, ...) LEAST(e1, e2, ...) ``` -`GREATEST` returns the largest non-null numeric value among the given expressions; `LEAST` -returns the smallest. NULL arguments are ignored (ANSI semantics); the result is NULL only +`GREATEST` returns the largest non-null value among the given expressions; `LEAST` +returns the smallest. The arguments are all numeric, or all dates and timestamps (a `DATE` +compares as the start of its day, UTC; the result is a `DATE` over dates and a `TIMESTAMP` when +they mix). NULL arguments are ignored (ANSI semantics); the result is NULL only when every argument is NULL. Both are emitted as Painless ternary chains over -`Math.max` / `Math.min`. They are row-level conditional functions, not aggregates β€” -`GREATEST(...) OVER (...)` is not supported. +`Math.max` / `Math.min` β€” over dates, over the epoch milliseconds each one denotes. They are +conditional functions, not aggregates β€” `GREATEST(...) OVER (...)` is not supported β€” but over +aggregates they are computed per group (`GREATEST(MAX(a), MAX(b))` beside a `GROUP BY`). ```sql SELECT GREATEST(price_us, price_eu, price_uk) AS max_price FROM products; diff --git a/documentation/sql/functions_conditional.md b/documentation/sql/functions_conditional.md index c15bf783b..f570c9b84 100644 --- a/documentation/sql/functions_conditional.md +++ b/documentation/sql/functions_conditional.md @@ -323,14 +323,15 @@ FROM products failing β€” so the `NULLIF` returned its first argument for every row and the query answered `200` with the wrong values. If you relied on one of these, write the comparison in the type you mean. -- A **string literal against a `date` column** is refused, and the message depends on the literal. - A malformed one is named as such β€” `NULLIF(created_at, 'yesterday')` reports that `'yesterday'` - is not a date for that field's format, checked against the column's mapping exactly as a `WHERE` - comparison is. A **well-formed** one β€” `NULLIF(created_at, '2024-01-15')` β€” is refused too, with - a message saying the comparison is not supported yet: the engine cannot turn a string into a - temporal value inside a script, so accepting it would mean comparing a date with a string, which - silently never matches. Cast the literal instead: - `NULLIF(created_at, CAST('2024-01-15' AS DATE))`. +- A **string literal against a `date` column** is read by what it is compared with, as in every + comparison: `NULLIF(created_at, '2024-01-15')` compares `created_at` with the date 2024-01-15, + and `NULLIF(qty, '3')` compares `qty` with the number 3 β€” in a query and in a computed column + alike (before **0.24.0** the first was refused, and a computed column compared a date with a + string, which never matched). A literal that spells no date is refused, and the message depends + on the literal: a malformed one is named as such β€” `NULLIF(created_at, 'yesterday')` reports + that `'yesterday'` is not a date for that field's format, checked against the column's mapping + exactly as a `WHERE` comparison is β€” and Elasticsearch date math (`'now-1d'`), which no script + can evaluate, is refused as not supported. - `expr2` being NULL is not a match: SQL says the comparison is then UNKNOWN, so the answer is `expr1`, not NULL. @@ -451,7 +452,7 @@ FROM products ### GREATEST -Returns the largest value among the given numeric expressions. NULL arguments are ignored (PostgreSQL-style NULL handling); the result is NULL only if every argument is NULL. +Returns the largest value among the given expressions β€” numbers, or dates and timestamps. NULL arguments are ignored (PostgreSQL-style NULL handling); the result is NULL only if every argument is NULL. **Syntax:** ```sql @@ -459,10 +460,11 @@ GREATEST(expression1, expression2, ...) ``` **Inputs:** -- One or more numeric expressions to compare +- One or more expressions of one kind: all numeric, or all `DATE` / `TIMESTAMP` / `DATETIME` **Output:** -- Numeric (widest input type), or NULL if every argument is NULL +- Numeric (widest input type) over numbers; over dates, a `DATE` when every argument is a `DATE` + and a `TIMESTAMP` when they mix; NULL if every argument is NULL **Examples:** @@ -479,16 +481,37 @@ FROM orders -- Never lets the net go below zero ``` +**3. The latest of two dates, and the days since the first:** +```sql +SELECT GREATEST(ordered_on, shipped_on) AS last_step, + GREATEST(ordered_on, shipped_on) - ordered_on AS days_to_last_step +FROM orders +``` + +**4. Per group, over aggregates:** +```sql +SELECT region, GREATEST(MAX(ordered_on), MAX(shipped_on)) AS last_activity +FROM orders +GROUP BY region +``` + **Notes:** - Ignores NULL arguments (PostgreSQL-style NULL handling); returns NULL only if every argument is NULL. -- All arguments should be numeric and have comparable types. +- The arguments are all numeric, or all dates and timestamps, as in every SQL engine; any other + argument, or numbers mixed with dates, is refused by name once the column types are known. +- Dates compare as instants, a `DATE` as the start of its day (UTC); the result is a date like any + other: `GREATEST(d, d2) - d` is a number of days (see [Date arithmetic](operators.md)). +- Over aggregates it is computed per group. In a SELECT, a group missing one of the aggregates' + values answers NULL: Elasticsearch does not run a per-group calculation there (its + `bucket_script` skips a group with a missing metric), as for any per-group calculation. Written + inline in `HAVING`, the group filter reads every aggregate and skips a NULL one. - See also: `LEAST`, `COALESCE`. --- ### LEAST -Returns the smallest value among the given numeric expressions. NULL arguments are ignored (PostgreSQL-style NULL handling); the result is NULL only if every argument is NULL. +Returns the smallest value among the given expressions β€” numbers, or dates and timestamps. NULL arguments are ignored (PostgreSQL-style NULL handling); the result is NULL only if every argument is NULL. **Syntax:** ```sql @@ -496,10 +519,11 @@ LEAST(expression1, expression2, ...) ``` **Inputs:** -- One or more numeric expressions to compare +- One or more expressions of one kind: all numeric, or all `DATE` / `TIMESTAMP` / `DATETIME` **Output:** -- Numeric (widest input type), or NULL if every argument is NULL +- Numeric (widest input type) over numbers; over dates, a `DATE` when every argument is a `DATE` + and a `TIMESTAMP` when they mix; NULL if every argument is NULL **Examples:** @@ -516,9 +540,17 @@ FROM orders -- Never lets the rebate exceed the base price ``` +**3. The earlier of a date and a timestamp:** +```sql +SELECT LEAST(due_on, delivered_at) AS first_event +FROM orders +-- a TIMESTAMP: the arguments mix a DATE and a TIMESTAMP +``` + **Notes:** - Ignores NULL arguments (PostgreSQL-style NULL handling); returns NULL only if every argument is NULL. -- All arguments should be numeric and have comparable types. +- The arguments are all numeric, or all dates and timestamps; see `GREATEST` for dates, and for + `LEAST` over aggregates per group. - See also: `GREATEST`, `COALESCE`. [Back to index](README.md) diff --git a/documentation/sql/functions_date_time.md b/documentation/sql/functions_date_time.md index baecc694f..fb4e2a234 100644 --- a/documentation/sql/functions_date_time.md +++ b/documentation/sql/functions_date_time.md @@ -103,6 +103,24 @@ SELECT CURTIME() AS current_time; ### Date/Time Arithmetic Functions +> **`+` and `-` on dates.** Since `0.24.0` a date or a timestamp can also be moved and subtracted +> with the plain operators, by the rules of the SQL engines: `DATE - DATE` is the calendar days +> between the two dates (`BIGINT`, as in PostgreSQL, DuckDB, Oracle and Snowflake); a difference with +> a `TIMESTAMP` on either side is the time between them in fractional days (`DOUBLE`, as Oracle +> answers for its date with a time of day); `DATE + n` is the date `n` days later (`DATE`, as in +> PostgreSQL, DuckDB, Oracle and BigQuery) β€” a fractional `n` gives a `TIMESTAMP` whose fraction is +> a time of day, as in Oracle; `TIMESTAMP + n` is `n` days later; a number written as a string +> (`'30'`) counts as that number, as PostgreSQL reads an untyped literal. `*`, `/` and `%` on a +> date, two dates added and a number minus a date are refused, as PostgreSQL, DuckDB, Oracle, Trino, +> Snowflake and SQL Server refuse them. See +> [Date arithmetic](operators.md#date-arithmetic--and---with-a-date-timestamp-or-datetime). +> +> ```sql +> SELECT ship_date - order_date AS days_to_ship FROM orders; -- BIGINT +> SELECT due_date + 30 AS reminder FROM invoices; -- DATE +> SELECT * FROM orders WHERE order_date > CURRENT_DATE - 7; +> ``` + #### INTERVAL Literal syntax for time intervals. @@ -346,6 +364,8 @@ TIMESTAMPDIFF(unit, date1, date2) -- MySQL **Direction, by layout:** every one of these databases computes *end βˆ’ start*; the layout decides where the end sits. - The dates first: `date1 - date2`. `DATEDIFF('2025-01-10', '2025-01-01')` is `9`, as in MySQL; `DATE_DIFF('2010-07-07', '2008-12-25', DAY)` is `559`, as in BigQuery. - The unit first: `date2 - date1`. `DATEDIFF(DAY, '2025-01-01', '2025-01-10')` is `9`. +- Between two `DATE`s, `DATEDIFF(date1, date2)` is the same number as the operator `date1 - date2` + (see [Date arithmetic](operators.md#date-arithmetic--and---with-a-date-timestamp-or-datetime)). **Counting, by name:** - `DATEDIFF` and `DATE_DIFF` count the calendar **boundaries crossed**, as SQL Server, Snowflake, Redshift, BigQuery and DuckDB do: one second across a year end is 1 `YEAR`, `1992-09-15` to `1992-11-14` is 2 `MONTH`s, `10:59` to `11:00` is 1 `HOUR`. `DAY` counts calendar days: `2025-01-10T23:30:00Z` and `2025-01-11T00:30:00Z` are 1 day apart. diff --git a/documentation/sql/functions_type_conversion.md b/documentation/sql/functions_type_conversion.md index 93028120d..851989579 100644 --- a/documentation/sql/functions_type_conversion.md +++ b/documentation/sql/functions_type_conversion.md @@ -147,6 +147,10 @@ SELECT CAST(CURRENT_TIMESTAMP AS DATE) AS d; -- a seconds-precision epoch lands in 1970. Multiply by 1000, or use a millisecond epoch. SELECT CAST(1704902400000 AS TIMESTAMP) AS ts; -- Result: 2024-01-10 12:00:00 + +-- The same over an integer or decimal column (created_ms), in a SELECT, in WHERE and in a +-- computed column; a fraction of a millisecond is dropped, toward zero +SELECT CAST(created_ms AS TIMESTAMP) AS created_at FROM events; ``` **Boolean Conversions:** @@ -656,6 +660,7 @@ WHERE denominator != 0; | INT | BOOLEAN | `CAST(1 AS BOOLEAN)` | 0=false, non-zero=true | | DOUBLE | INT | `CAST(123.99 AS INT)` | Truncates decimal | | DOUBLE | VARCHAR | `CAST(123.45 AS VARCHAR)` | Always succeeds | +| BIGINT | TIMESTAMP | `CAST(epoch_ms AS TIMESTAMP)` | Epoch milliseconds, UTC | | DATE | VARCHAR | `CAST(date_col AS VARCHAR)` | Format: 'YYYY-MM-DD' | | DATE | TIMESTAMP | `CAST(date_col AS TIMESTAMP)` | Time set to 00:00:00 | | TIMESTAMP | DATE | `CAST(ts_col AS DATE)` | Drops time component | diff --git a/documentation/sql/operators.md b/documentation/sql/operators.md index f286ae890..b8d07f288 100644 --- a/documentation/sql/operators.md +++ b/documentation/sql/operators.md @@ -30,10 +30,11 @@ expr1 + expr2 ``` **Inputs:** -- `expr1`, `expr2` - Numeric expressions (`INT`, `DOUBLE`, `DECIMAL`, etc.) +- `expr1`, `expr2` - Numeric expressions (`INT`, `DOUBLE`, `DECIMAL`, etc.), or a `DATE`, + `TIMESTAMP` or `DATETIME` and a number of days (see [Date arithmetic](#date-arithmetic--and---with-a-date-timestamp-or-datetime)) **Output:** -- Numeric type (result type depends on operand types) +- Numeric type (result type depends on operand types); a date or a timestamp for date arithmetic **Examples:** @@ -65,6 +66,15 @@ SELECT 10.5 + 20.3 AS total; -- Mixed types (INT + DOUBLE) SELECT 10 + 20.5 AS total; -- Result: 30.5 (promoted to DOUBLE) + +-- A number and a string column (n INT, s KEYWORD) are a type error, refused by name once the +-- column types are known +SELECT n + s AS x FROM t; +-- Error: Type mismatch: output 'INT' is not compatible with input 'KEYWORD' + +-- To join them as text, convert the number +SELECT CONCAT(CAST(n AS VARCHAR), s) AS x FROM t; +-- Result: '3a' for n = 3, s = 'a' ``` **NULL Handling:** @@ -129,10 +139,14 @@ SELECT 100 + (-50) AS result; -- Result: 50 ``` -**Date Arithmetic:** +**Date Arithmetic** (see [Date arithmetic](#date-arithmetic--and---with-a-date-timestamp-or-datetime)): ```sql --- Date subtraction (days between) -SELECT order_date - ship_date AS days_to_ship +-- Date subtraction: the days between the two dates (BIGINT) +SELECT ship_date - order_date AS days_to_ship +FROM orders; + +-- A date moved back by days (DATE) +SELECT order_date - 7 AS week_before FROM orders; -- With INTERVAL @@ -453,6 +467,130 @@ SELECT 10 % -3 AS result; --- +### Date arithmetic: `+` and `-` with a `DATE`, `TIMESTAMP` or `DATETIME` + +**Description:** +Since `0.24.0`, `+` and `-` follow the rules SQL engines apply to dates, in every place a script +runs: a row `SELECT`, a `WHERE`, a per-group calculation over aggregates (`MAX(d) - MIN(d) AS x`), +a `HAVING` over its alias, a computed column (`SCRIPT AS`) and a materialized view. + +| Expression | Result | Type | +|---|---|---| +| `DATE - DATE` | the calendar days between the two dates | `BIGINT` | +| `TIMESTAMP - TIMESTAMP`, or a `DATE` and a `TIMESTAMP` | the milliseconds between the two, in days (`1.5` is a day and a half) | `DOUBLE` | +| `DATE + n`, `DATE - n`, `n` a whole number (`INT`, `BIGINT`, ...) | the date `n` days later / earlier | `DATE` | +| `DATE + n`, `DATE - n`, `n` a fractional number (`DOUBLE`, `REAL`, `DECIMAL`) | `n` days later / earlier, the fraction as a time of day | `TIMESTAMP` | +| `TIMESTAMP + n`, `TIMESTAMP - n`, any number `n` | `n` days later / earlier | `TIMESTAMP` | +| `n + temporal` | the same as `temporal + n` | as above | +| a temporal and a string literal that is a number (`'30'`, `'1.5'`) | the same as with that number | as above | +| a temporal and `NULL` | `NULL` | the type above, for a typed `NULL` (`CAST(NULL AS DATE) - d` is a `BIGINT`) | +| two temporals added, a temporal under `*`, `/` or `%`, a number minus a temporal, a temporal combined with any other string or a boolean | a type error, refused before anything runs | β€” | + +`DATETIME` behaves as `TIMESTAMP`. Days are UTC calendar days: a `DATE` operand is the UTC day it +falls in, so `DATE - DATE` is the same number as `DATEDIFF(date1, date2)`. + +An operand's type is the type it produces: a column's declared type; an aggregate's own type +(`COUNT` is a `BIGINT`, `AVG` a `DOUBLE`, `SUM` the column's numeric type, `MIN`, `MAX`, +`FIRST_VALUE` and `LAST_VALUE` the column's type); `CASE`, `COALESCE`, `NULLIF`, `GREATEST` and +`LEAST` the common type of their branches or arguments. So `MAX(order_date) - AVG(delay)` is a +`TIMESTAMP`, `COUNT(order_date) - 1` a number, and +`CASE WHEN express THEN ship_date ELSE due_date END - order_date` and +`GREATEST(ship_date, due_date) - order_date` numbers of days (`GREATEST` and `LEAST` compare dates +and timestamps too: see [GREATEST](functions_conditional.md#greatest)). + +**The engines this follows:** +- PostgreSQL, DuckDB, Oracle and Snowflake subtract two dates into a number of days; Oracle, whose + `DATE` carries a time of day, answers fractional days. +- PostgreSQL, DuckDB, Oracle and BigQuery add a number of days to a date; Oracle turns the fraction + of `DATE + 1.5` into a time of day. +- PostgreSQL, DuckDB, Oracle, Trino, Snowflake and SQL Server refuse the other combinations. + +**Examples:** +```sql +-- Days between two dates +SELECT ship_date - order_date AS days_to_ship FROM orders; +-- 2024-01-31 -> 2024-02-01 is 1; 2024-02-28 -> 2024-03-01 is 2 (2024 is a leap year) + +-- A date moved by days +SELECT due_date + 30 AS reminder FROM invoices; -- DATE +SELECT due_date + 1.5 AS reminder_at FROM invoices; -- TIMESTAMP, 12:00 on the next day + +-- Fractional days between two instants +SELECT closed_at - opened_at AS days_open FROM tickets; -- 1.25 is a day and six hours + +-- A filter relative to today +SELECT * FROM orders WHERE order_date > CURRENT_DATE - 7; + +-- Per group, and in HAVING through the alias +SELECT customer, MAX(order_date) - MIN(order_date) AS span +FROM orders GROUP BY customer HAVING span > 30; + +-- A computed column +CREATE TABLE tickets ( + id KEYWORD, opened_at TIMESTAMP, closed_at TIMESTAMP, + days_open DOUBLE SCRIPT AS (closed_at - opened_at), + PRIMARY KEY (id) +); +``` + +**Refused combinations** are refused by name, with the operator, the two operand types and the +remedy: +```sql +SELECT order_date * 2 FROM orders; +-- Type mismatch: operator * cannot be applied to DATE and BIGINT in expression: order_date * 2; +-- subtract two dates for the days between them, or add or subtract a number of days +``` + +**Notes:** +- A temporal operand is recognised by its **declared** type: Elasticsearch stores `DATE`, + `TIMESTAMP` and `DATETIME` alike as a `date`, and the declaration is what tells a date from a + timestamp. A `TIME` operand is not covered by these rules. +- A `date` field elasticsql did not declare β€” an index created by a bulk load, Logstash or any other + client, or a field a bulk load added to a table elasticsql created β€” is a `TIMESTAMP`: `DESCRIBE + TABLE` reports it so, a `CREATE TABLE ... AS SELECT` declares it so, and it follows the + `TIMESTAMP` rules (`created_at + 1` keeps the time of day, `closed_at - opened_at` is + fractional). A column declared `DATE` through elasticsql stays a `DATE`. +- A string literal is read by what it is combined or compared with, as PostgreSQL reads one: + `order_date - '2024-01-01'` is the days between two dates, `qty + '1'` adds the number 1, and in + a comparison β€” `=`, `<>`, `<`, `<=`, `>`, `>=`, `BETWEEN`, `IN`, `CASE x WHEN v`, `NULLIF`, a + `CASE` condition β€” a literal beside a number is the number it spells, beside a `DATE` the date + it spells (the date of a date-time literal), beside a `TIMESTAMP` the instant it spells + (`'2024-01-31'` is its midnight), and beside a `BOOLEAN` the boolean it spells. A `DATE` and a + `TIMESTAMP` compare, the `DATE` being the instant its day starts at. Any other literal is text, + and comparing it with a number or a date is refused by name. +- An index pattern or a comma list (`FROM logs-*`, `FROM orders_2024, orders_2025`) answers like a + single index: every field its indices map alike has its type, read once per pattern from the + merged mapping. ⚠️ A field two of them map differently (a `date` in one index, a `keyword` in + another) has no type: nothing is refused on it and Elasticsearch answers for it β€” per group, + `MAX(order_date) - MIN(order_date)` over such a field answers milliseconds rather than days. Every + other field of the pattern keeps its type. +- Type errors are checked once the column types are known, when the statement runs and the mapping + is read, and before anything is sent to Elasticsearch: `WHERE due_date + 7 > CURRENT_DATE` runs, + `SELECT order_date * 2` is refused. With no type to read (a field the indices of a pattern map + differently, a mapping that cannot be read), nothing is refused on types. +- In a per-group calculation, a date result comes back as the same number a date aggregate does + (`MAX(order_date)`): milliseconds since the epoch. `CURRENT_DATE` and `NOW()` there are the time + the statement runs (`CURRENT_DATE - MAX(order_date)`); a materialized view refuses them. +- An explicit cast applies to the result, and a `NULL` stays `NULL`: + `(ship_date - order_date)::TIMESTAMP` converts the number of days as any number is converted to a + `TIMESTAMP` β€” as milliseconds since the epoch β€” and `(quantity + ratio)::BIGINT` truncates, as + `CAST(3.25 AS BIGINT)` does. +- `INTERVAL` arithmetic (`order_date - INTERVAL 7 DAY`) and `DATEDIFF` / `TIMESTAMPDIFF` are + unchanged. +- ⚠️ On Elasticsearch 6.8, a `date` column compared with another `date` column + (`WHERE ship_date > order_date`), or with a date function of one + (`WHERE ship_date > DATE_TRUNC(order_date, MONTH)`), fails with a 400 (`failed to parse date + field`); date arithmetic on one side runs: `WHERE ship_date > order_date + 0`. Elasticsearch 8 + answers all three. + +**Before `0.24.0`:** `MAX(d) - MIN(d)` answered milliseconds and was typed `NUMERIC`; `+` and `-` +with a date failed in Elasticsearch at row level β€” except a date plus a string, which concatenated +the two β€” and per group `MAX(d) + 1` failed while `MAX(d) + 1.5` added 1.5 milliseconds; +`CURRENT_DATE - 7` was refused when the statement was parsed; and a computed column over date +arithmetic stored epoch-millisecond arithmetic or rejected the document at ingest. + +--- + ## Comparison Operators ### Operator: `=` @@ -485,6 +623,14 @@ SELECT * FROM emp WHERE department = 'IT'; -- Compare columns SELECT * FROM orders WHERE customer_id = shipping_customer_id; + +-- Columns of incompatible types (id KEYWORD, n INT) are refused by name once the column types +-- are known, rather than answering no rows +SELECT * FROM t WHERE id = n; +-- Error: Type mismatch: 'KEYWORD' is not compatible with 'INT' in expression: id = n + +-- Cast one side to the other's type +SELECT * FROM t WHERE id = CAST(n AS VARCHAR); ``` **String Comparison:** diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala new file mode 100644 index 000000000..7e661b3aa --- /dev/null +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.bridge + +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import com.sksamuel.elastic4s.searches.SearchRequest +import com.sksamuel.elastic4s.searches.aggs.{AbstractAggregation, Aggregation} +import com.sksamuel.elastic4s.searches.aggs.pipeline.BucketScriptPipelineAgg + +import scala.jdk.CollectionConverters._ + +/** The PARAMS of a `bucket_script`'s script, which the elastic4s body builder drops. + * + * πŸ”΄ elastic4s writes a `bucket_script` as `"script": ""` -- the bare source string + * (`BucketScriptPipelineAggBuilder`, the same in elastic4s 6.7, 7.17, 8.18 and 9.0) -- while a + * `bucket_selector` gets the whole script object. So a per-group calculation reading the request + * clock, `CURRENT_DATE - MAX(d)` or `NOW() - MAX(ts)`, was sent WITHOUT the `__now__` its script + * reads, and Elasticsearch failed it; the same clock in a HAVING worked. MEASURED on 8.18.3. + * + * The script is sent as the object Elasticsearch documents -- `source`, `lang`, `params` -- for + * exactly the `bucket_script`s that carry params, so the source stays the SAME text from one + * request to the next and Elasticsearch compiles it once: inlining the clock instead would compile + * a new script on every request, against the cluster's compilation rate limit. Every other body is + * returned untouched, unparsed. + */ +object BucketScriptParams { + + private val mapper = new ObjectMapper() + + /** The params of every `bucket_script` of `aggs` that carries some, by aggregation name, at any + * depth. + */ + private def withParams(aggs: Seq[AbstractAggregation]): Map[String, Map[String, Any]] = + aggs.flatMap { + case script: BucketScriptPipelineAgg if script.script.params.nonEmpty => + Seq(script.name -> script.script.params) + case parent: Aggregation => withParams(parent.subaggs).toSeq + case _ => Nil + }.toMap + + /** `body`, the JSON of `search`, with the params of its `bucket_script`s bound. */ + def bind(body: String, search: SearchRequest): String = { + val params = withParams(search.aggs) + if (params.isEmpty) body + else { + val root = mapper.readTree(body) + bind(root, params) + mapper.writeValueAsString(root) + } + } + + private def bind(node: JsonNode, params: Map[String, Map[String, Any]]): Unit = + node match { + case obj: ObjectNode => + obj.fields().asScala.toList.foreach { entry => + (params.get(entry.getKey), entry.getValue) match { + case (Some(values), agg: ObjectNode) if agg.has("bucket_script") => + agg.get("bucket_script") match { + case bucketScript: ObjectNode if bucketScript.path("script").isTextual => + val script = mapper.createObjectNode() + script.put("source", bucketScript.get("script").asText()) + script.put("lang", "painless") + val bound = script.putObject("params") + values.foreach { case (name, value) => + bound.set[JsonNode](name, mapper.valueToTree[JsonNode](value)) + } + bucketScript.set[JsonNode]("script", script) + case _ => + } + case _ => + } + bind(entry.getValue, params) + } + case array: ArrayNode => array.elements().asScala.foreach(bind(_, params)) + case _ => + } +} diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 5eccb1ea1..7eacc6332 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, @@ -162,7 +162,8 @@ object ElasticAggregation { allAggregations: Map[String, SQLAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -219,14 +220,14 @@ object ElasticAggregation { buildScript: (String, Script) => Aggregation ): Aggregation = { if (transformFuncs.nonEmpty) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val scriptSrc = identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val scriptSrc = th.identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) @@ -471,7 +472,8 @@ object ElasticAggregation { allElasticAggregations: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = @@ -488,7 +490,7 @@ object ElasticAggregation { val aggScript = if (!bucket.isBucketScript && bucket.shouldBeScripted) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val painless = bucket.painless(Some(context)) Some(now(Script(s"$context$painless").lang("painless"))) } else { @@ -716,7 +718,8 @@ object ElasticAggregation { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): Option[Aggregation] = request.having.flatMap(_.criteria) match { case Some(criteria) if request.wholeTableHaving && aggs.nonEmpty => diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 6eca023d4..a6985f06a 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.operator.AND import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -51,7 +51,8 @@ case class ElasticBridge(filter: ElasticFilter) { currentQuery: Option[ElasticBoolQuery] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { filter match { case boolQuery: ElasticBoolQuery => diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index efb14be16..b8610990c 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.searches.queries.Query @@ -24,7 +24,8 @@ case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { val query = criteria.boolQuery.copy(group = group) query diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index 6782773f9..49dcc4e28 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -49,5 +49,10 @@ case class ElasticSearchRequest( def hasTransformExtendedStats: Boolean = SearchBodySerializer.hasTransformExtendedStats(search) def query: String = - serializer.serialize(search).replace("\"version\":true,", "") /*FIXME*/ + // the params of a per-group calculation's script, which the body builder drops: the request + // clock `CURRENT_DATE - MAX(d)` reads (`BucketScriptParams`) + BucketScriptParams.bind( + serializer.serialize(search).replace("\"version\":true,", ""), /*FIXME*/ + search + ) } diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index f8dc25247..daadf6e07 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -75,7 +75,8 @@ package object bridge { innerHitsName: String )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { @@ -154,7 +155,8 @@ package object bridge { request: SingleSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -182,7 +184,8 @@ package object bridge { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) @@ -255,7 +258,8 @@ package object bridge { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations @@ -468,10 +472,16 @@ package object bridge { // SearchBodySerializer in implicit scope of this conversion; with none in scope the default // argument applies (the one-argument elastic4s builder, refusing a transform-bearing // extended_stats). Same defaulted-implicit pattern as `contextType`. + // + // `target` is the same seam for SCRIPTS: the Elasticsearch major the client module talks to + // (`PainlessTarget`), which every conversion below hands to the `PainlessContext`s it creates. + // With none in scope a script renders as for the majors from 7 on; the ES 6 client modules + // give 6, whose query scripts compare a `date` doc value differently. implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query, - serializer: SearchBodySerializer = SearchBodySerializer.Default + serializer: SearchBodySerializer = SearchBodySerializer.Default, + target: PainlessTarget = PainlessTarget.Default ): ElasticSearchRequest = ElasticSearchRequest( request.sql, @@ -530,7 +540,8 @@ package object bridge { request: SingleSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): SearchRequest = { import request._ @@ -593,7 +604,7 @@ package object bridge { case Nil => _search case _ => _search scriptfields scriptFields.map { field => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = field.painless(Some(context)) scriptField( field.scriptName, @@ -614,7 +625,7 @@ package object bridge { case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map { sort => if (sort.isScriptSort) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val painless = sort.field.painless(Some(context)) val painlessScript = s"$context$painless" val script = @@ -680,7 +691,8 @@ package object bridge { request: MultiSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): MultiSearchRequest = { // Story 22.6 β€” a NAMED backstop, never a silent `UNION ALL`. Only `UNION ALL` is an // Elasticsearch `_msearch`; every other set operator needs the relational engine and is @@ -717,9 +729,10 @@ package object bridge { private[bridge] def scriptQueryOf(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): Query = { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = criteria.painless(Some(context)) scriptQuery( now(Script(script = s"$context$script").lang("painless").scriptType("source")) @@ -733,7 +746,8 @@ package object bridge { implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { import expression._ if (isAggregation) @@ -958,7 +972,7 @@ package object bridge { case _ => scriptQueryOf(expression) } case _ => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = painless(Some(context)) scriptQuery( now( @@ -969,7 +983,7 @@ package object bridge { ) } case _ => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = painless(Some(context)) scriptQuery( now( @@ -1013,7 +1027,8 @@ package object bridge { implicit def inToQuery[R, T <: Value[R]](in: InExpr[R, T])(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { if (requiresScript(in.identifier)) return scriptQueryOf(in) import in._ @@ -1038,7 +1053,8 @@ package object bridge { between: BetweenExpr )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { import between._ if (requiresScript(identifier)) return scriptQueryOf(between) @@ -1191,14 +1207,16 @@ package object bridge { implicit def criteriaToQuery(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { ElasticCriteria(criteria).asQuery() } implicit def criteriaToNode(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): JsonNode = { queryToJson(criteriaToQuery(criteria)) } @@ -1217,7 +1235,8 @@ package object bridge { )(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query, - serializer: SearchBodySerializer = SearchBodySerializer.Default + serializer: SearchBodySerializer = SearchBodySerializer.Default, + target: PainlessTarget = PainlessTarget.Default ): Seq[ElasticAggregation] = { import query._ statement @@ -1257,7 +1276,10 @@ package object bridge { aggregation.copy( sources = l.sources, query = Some( - serializer.serialize(body).replace("\"version\":true,", "") /*FIXME*/ + BucketScriptParams.bind( + serializer.serialize(body).replace("\"version\":true,", ""), /*FIXME*/ + body + ) ) ) }) diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala new file mode 100644 index 000000000..f6927d6e9 --- /dev/null +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime +import scala.jdk.CollectionConverters._ + +/** A per-group calculation that reads the request clock (`CURRENT_DATE - MAX(d)`, `NOW() - + * MAX(ts)`) is sent WITH the clock: its `bucket_script` carries the script object -- `source`, + * `lang`, `params.__now__` -- where the elastic4s builder writes the bare source string and drops + * the params (`BucketScriptParams`). Every other `bucket_script` is sent as it was. + * + * The clock is a PARAM, not text spliced into the source: the source is the same from one request + * to the next, so Elasticsearch compiles it once. + */ +class BucketScriptParamsSpec extends AnyFlatSpec with Matchers { + + import scala.language.implicitConversions + + private val now: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + implicit def timestamp: Long = now + + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) + case other => throw new IllegalArgumentException(s"Not a single search: $other") + } + + private val mapper = new ObjectMapper() + + private def bucketScripts(sql: String): Seq[JsonNode] = { + val request: ElasticSearchRequest = SelectStatement(sql) + def all(node: JsonNode): Seq[JsonNode] = + Option(node.get("bucket_script")).toSeq ++ node.elements().asScala.toSeq.flatMap(all) + all(mapper.readTree(request.query)) + } + + "a per-group calculation reading the clock" should "send the clock as a script param" in { + Seq( + "SELECT g, CURRENT_DATE - MAX(d) AS x FROM t GROUP BY g", + "SELECT g, NOW() - MAX(ts) AS x FROM t GROUP BY g", + "SELECT g, DATEDIFF(CURRENT_DATE, MAX(d)) AS x FROM t GROUP BY g" + ).foreach { sql => + withClue(s"[$sql] ") { + val scripts = bucketScripts(sql) + scripts should have size 1 + val script = scripts.head.get("script") + script.isObject shouldBe true + script.get("source").asText() should include("params.__now__") + script.get("lang").asText() shouldBe "painless" + script.get("params").get("__now__").asLong() shouldBe now + } + } + } + + "a per-group calculation without the clock" should "be sent as it always was" in { + val scripts = bucketScripts("SELECT g, MAX(d) - MIN(d) AS x FROM t GROUP BY g") + scripts should have size 1 + scripts.head.get("script").isTextual shouldBe true + } +} diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala new file mode 100644 index 000000000..e385013b6 --- /dev/null +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime + +/** The Elasticsearch major the client module puts in implicit scope (`PainlessTarget`) reaches + * EVERY query script the bridge's conversions create: a WHERE, a script field, an aggregation over + * an expression. + * + * The probe is a comparison between a `date` doc value and another date, which Elasticsearch 6.8 + * alone needs normalised to UTC: with 6 in scope the body carries the normalisation, with any + * other major -- or none -- it is the very body no target produces. + */ +class PainlessTargetSpec extends AnyFlatSpec with Matchers { + + private val now: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + private val normalisation = ".withZoneSameInstant(ZoneId.of('Z'))" + + private val schema = Parser( + "CREATE TABLE t (id KEYWORD, g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, n INT, PRIMARY KEY (id))" + ) match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"$other") + } + + private def resolved(sql: String): SingleSearch = + Parser(sql) match { + case Right(ss: SingleSearch) => + val r = ss.update(Some(schema)) + r.validateResolved() shouldBe Right(()) + r + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + /** The body `requestToElasticSearchRequest` sends. (`sqlQueryToAggregations`, the other door, + * parses its statement from text: no schema, so no type to normalise a comparison by.) + */ + private def body(sql: String, target: PainlessTarget): String = + requestToElasticSearchRequest(resolved(sql))( + now, + PainlessContextType.Query, + SearchBodySerializer.Default, + target + ).query + + private val venues: Seq[String] = Seq( + // a WHERE: a script query, alone and inside a bool + "SELECT id FROM t WHERE CAST(ts AS DATE) = d", + "SELECT id FROM t WHERE CURRENT_DATE > d AND n > 1", + // a script field + "SELECT id, CASE WHEN CURRENT_DATE > d THEN 1 ELSE 0 END AS c FROM t", + // an aggregation over an expression + "SELECT SUM(CASE WHEN CURRENT_DATE > d THEN 1 ELSE 0 END) AS s FROM t", + // the WHERE of an aggregate query + "SELECT g, COUNT(*) AS c FROM t WHERE CAST(ts AS DATE) = d GROUP BY g" + ) + + "a script comparing a date doc value" should + "carry the UTC normalisation when the client module gives Elasticsearch 6" in { + venues.foreach { sql => + withClue(s"[$sql] ") { + body(sql, PainlessTarget(6)) should include(normalisation) + } + } + } + + it should "be the body no target produces for Elasticsearch 7, 8 and 9" in { + venues.foreach { sql => + withClue(s"[$sql] ") { + val default = body(sql, PainlessTarget.Default) + default should not include normalisation + Seq(7, 8, 9).foreach(major => body(sql, PainlessTarget(major)) shouldBe default) + body(sql, PainlessTarget(6)).replace(normalisation, "") shouldBe default + } + } + } +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala index 5fdb8ecc4..22bfe40f7 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala @@ -25,7 +25,7 @@ import app.softnetwork.elastic.client.{ } import com.fasterxml.jackson.databind.JsonNode import app.softnetwork.elastic.client.result.ElasticResult -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.bridge.{ElasticSearchRequest, SearchBodySerializer} import app.softnetwork.elastic.sql.query.SingleSearch import io.searchbox.core.{MultiSearch, Search, SearchResult} @@ -71,6 +71,16 @@ trait JestSearchApi extends SearchApi with JestClientHelpers { */ implicit def searchBodySerializer: SearchBodySerializer = JestSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * Elasticsearch 6.8 hands a query script a `date` doc value as a `JodaCompatibleZonedDateTime`, + * so a comparison of one with any other date is normalised to UTC first. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget( + JestSearchBodySerializer.ElasticsearchMajor + ) + private[client] implicit def singleSearchToJsonQuery( sqlSearch: SingleSearch )(implicit diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientDateArithmeticSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientDateArithmeticSpec.scala new file mode 100644 index 000000000..a646db785 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 7031b951c..6789fcbf0 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, Value} +import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, PainlessTarget, Value} import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.policy.{EnrichPolicy, EnrichPolicyTask} @@ -1052,6 +1052,16 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel */ implicit def searchBodySerializer: SearchBodySerializer = RestHighLevelClientSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * Elasticsearch 6.8 hands a query script a `date` doc value as a `JodaCompatibleZonedDateTime`, + * so a comparison of one with any other date is normalised to UTC first. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget( + RestHighLevelClientSearchBodySerializer.ElasticsearchMajor + ) + override implicit def singleSearchToJsonQuery(sqlSearch: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala new file mode 100644 index 000000000..2dde42545 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 0092c4d47..7e9b7ab31 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, Value} +import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, PainlessTarget, Value} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.policy.{ EnrichPolicy, @@ -1104,6 +1104,15 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel */ implicit def searchBodySerializer: SearchBodySerializer = RestHighLevelClientSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * A `date` doc value is a `ZonedDateTime` here: scripts render as they always have. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget( + RestHighLevelClientSearchBodySerializer.ElasticsearchMajor + ) + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala new file mode 100644 index 000000000..2dde42545 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 9fac19776..2def9d106 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -38,7 +38,7 @@ import app.softnetwork.elastic.sql.policy.{ EnrichPolicyTaskStatus, EnrichPolicyType } -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.transform.{ Delay, @@ -1034,6 +1034,13 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { */ implicit def searchBodySerializer: SearchBodySerializer = JavaClientSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * A `date` doc value is a `ZonedDateTime` here: scripts render as they always have. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget(8) + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala new file mode 100644 index 000000000..de5ed4607 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 09681fda3..6129a13e8 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -37,7 +37,7 @@ import app.softnetwork.elastic.sql.policy.{ EnrichPolicyTaskStatus, EnrichPolicyType } -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.transform.{ @@ -1034,6 +1034,13 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { */ implicit def searchBodySerializer: SearchBodySerializer = JavaClientSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * A `date` doc value is a `ZonedDateTime` here: scripts render as they always have. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget(9) + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala new file mode 100644 index 000000000..de5ed4607 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 9543ac0eb..584ad3521 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -75,8 +75,29 @@ package object schema { } object IndexField { + + /** The type of a field elasticsql never DECLARED -- no `_meta` `data_type` -- read off its + * Elasticsearch mapping type: an index created by a bulk load or Logstash, or a field a bulk + * load added later to a table elasticsql created (dynamic mapping). + * + * πŸ”΄ THE declaration seam (the lead's ruling of 2026-10-05): such a `date` field holds + * INSTANTS + * -- Elasticsearch stores every temporal as one, and nobody said it was a calendar date -- so + * it is a TIMESTAMP, everywhere: DESCRIBE, the type a CTAS declares, comparisons, functions + * and date arithmetic all read it from here, and an `ALTER TABLE` that writes the table's + * `_meta` declares it TIMESTAMP, keeping its time. A field elasticsql DECLARED keeps its + * declaration, a DATE included. + * + * ⚠️ Deliberately NOT in `SQLTypes.apply`: that maps a TYPE NAME, the declared `DATE` as well + * as the mapping's `date`, and story 21.5 learned there that conflating the two rewrites the + * declaration of every DATE column (`ColumnMetaDiffSpec`). Only this seam knows that no + * declaration exists. + */ + private[schema] def undeclaredType(mappingType: String): String = + if (mappingType == "date") "timestamp" else mappingType + def apply(name: String, node: JsonNode, _meta: Option[ObjectValue]): IndexField = { - val tpe = _meta + val declaredType = _meta .flatMap { case m: ObjectValue => m.value.get("data_type") match { @@ -85,7 +106,10 @@ package object schema { } case _ => None } - .getOrElse(Option(node.get("type")).map(_.asText()).getOrElse("object")) + val tpe = + declaredType.getOrElse( + undeclaredType(Option(node.get("type")).map(_.asText()).getOrElse("object")) + ) val nullValue = Option(node.get("null_value")) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index a93f09664..469691ab7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -39,6 +39,55 @@ package object aggregate { } + object AggregateFunction { + + /** The type an AGGREGATE produces -- the metric Elasticsearch computes for a group -- as + * opposed to the type of the column it aggregates, which is what the aggregate identifier's + * `out` reports. + * + * πŸ”΄ ONE derivation for every consumer (#292): the date-arithmetic rules (`COUNT(d) - 1` is a + * number, never a date), the coercion a per-group calculation applies to a metric + * (`params.count_d` is a count, so it is never read as an instant), and the type rules that + * compare a metric once the column types are known (`HAVING COUNT(d) > 1`). Reading the + * column's type instead is what rendered `params.count_d.toInstant()`. + * + * - `COUNT` counts: BIGINT, whatever it counts. + * - `AVG` averages: DOUBLE. + * - `SUM` adds: the summed column's numeric type (whole stays whole, fractional stays + * fractional); a sum of anything else is the DOUBLE Elasticsearch answers -- a sum of + * dates is not a date. + * - `MIN`, `MAX`, `FIRST_VALUE`, `LAST_VALUE` return one of the values they aggregate: the + * REPORTED type of the aggregated chain, so a DATE stays a DATE and `MAX(CAST(ts AS + * DATE))` is a DATE. + * - every other aggregate answers its own declared type when it has one (STDDEV and the + * percentiles are DOUBLE, the ranking windows BIGINT), else the aggregated chain's. + * + * `None` when `aggregate` aggregates nothing. + */ + def outputTypeOf(aggregate: Identifier): Option[SQLType] = + aggregate.aggregateFunction.map { function => + def aggregated: SQLType = + aggregate + .withFunctions(aggregate.functions.filterNot(_.isInstanceOf[AggregateFunction])) + .reportedType + function match { + case COUNT | _: CountAgg => SQLTypes.BigInt + case AVG | _: AvgAgg => SQLTypes.Double + case SUM | _: SumAgg => + aggregated match { + case whole @ (SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | + SQLTypes.BigInt) => + whole + case fractional @ (SQLTypes.Real | SQLTypes.Double) => fractional + case _ => SQLTypes.Double + } + case MIN | MAX | _: MinAgg | _: MaxAgg | _: FirstValue | _: LastValue => aggregated + case other => + Some(other.out).filterNot(t => t.isUnknown || t == SQLTypes.Null).getOrElse(aggregated) + } + } + } + case object COUNT extends Expr("COUNT") with AggregateFunction with Window case object MIN extends Expr("MIN") with AggregateFunction with Window diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 4f3f8f1dd..27d88a835 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -18,6 +18,8 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ query, + renderedTypeOf, + ComparisonRule, Expr, GenericIdentifier, Identifier, @@ -39,6 +41,7 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTypes, SQLVarchar } +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.schema.Column import app.softnetwork.elastic.sql.query.{ @@ -282,6 +285,12 @@ package object cond { private val simpleOperand = """(?:[A-Za-z_][A-Za-z0-9_]*|-?\d+(?:\.\d+)?[lLfFdD]?|"(?:[^"\\]|\\.)*")""".r + /** How a simple `CASE WHEN ` compares (`Case.whenComparison`). */ + private[cond] sealed trait WhenComparison + private[cond] case object WhenAsResult extends WhenComparison + private[cond] case object WhenInstants extends WhenComparison + private[cond] case object WhenNumbers extends WhenComparison + case class NullIf(expr1: PainlessScript, expr2: PainlessScript) extends ConditionalFunction[SQLAny] { override def conditionalOp: ConditionalOp = NullIf @@ -346,7 +355,23 @@ package object cond { case other => other.out } - override def argTypes: List[SQLType] = args.map(argTypeOf) + /** What a string LITERAL argument is read as beside the other argument -- the ONE comparison + * rule (`ComparisonRule.readingOf`, the lead's ruling of 2026-10-05): `NULLIF(n, '3')` + * compares `n` with the number 3, `NULLIF(COALESCE(d, d2), '2024-01-31')` with a date. `None` + * for anything else, and for a literal that spells no value of the other's kind. + */ + private[cond] def secondReading: Option[ComparisonRule.Reading] = + ComparisonRule.readingOf(expr2, ComparisonRule.declaredTypeOf(expr1)) + + private[cond] def firstReading: Option[ComparisonRule.Reading] = + ComparisonRule.readingOf(expr1, ComparisonRule.declaredTypeOf(expr2)) + + /** A literal argument counts as what it is read as: `NULLIF(n, '3')` compares two numbers. */ + override def argTypes: List[SQLType] = + List( + firstReading.map(_.sqlType).getOrElse(argTypeOf(expr1)), + secondReading.map(_.sqlType).getOrElse(argTypeOf(expr2)) + ) /** πŸ”΄ `NULLIF` REPORTS THE TYPE OF ITS FIRST ARGUMENT, not the super type of both. * @@ -461,7 +486,11 @@ package object cond { def typeMismatchError: Option[String] = { val left = argTypeOf(expr1) val right = argTypeOf(expr2) - if (temporalAgainstText(left, right)) + // A literal the other argument READS (`secondReading`) is a value of that argument's kind: + // comparable by construction, and rendered as that value. Asked first, by the query seam + // and the DDL one alike, so a computed column and a query read `NULLIF(n, '3')` the same. + if (secondReading.isDefined || firstReading.isDefined) None + else if (temporalAgainstText(left, right)) // The resolver goes first so a MALFORMED literal is named as such (`'x'` is not a date for // this field's format) rather than reported as a bare type mismatch; a WELL-FORMED one // falls through to a message that says what is actually wrong -- the engine, not the SQL. @@ -617,8 +646,12 @@ package object cond { ): String = { callArgs match { case List(rendered0, rendered1) => - val arg0 = operand(rendered0, context) - val arg1 = operand(rendered1, context) + // A string literal argument is the value the other argument READS it as -- the number + // or the boolean it spells, rendered as one (`secondReading` / `firstReading`). + def scalar(reading: Option[ComparisonRule.Reading], rendered: String): String = + reading.flatMap(ComparisonRule.scalarPainless).getOrElse(rendered) + val arg0 = operand(scalar(firstReading, rendered0), context) + val arg1 = operand(scalar(secondReading, rendered1), context) // πŸ”΄ The SECOND argument is guarded too (issue #373, found by review) -- the same hole // `checkCase` had, one method up in this file. `NULLIF(CAST(d AS TIME), CAST(ts AS // TIME))` called the comparison with a null `arg1` for any document missing `ts`: @@ -630,35 +663,77 @@ package object cond { // answer is `a`. Short-circuiting `arg1 == null` to the NULL branch would invert that. def nullIf(comparison: String): String = s"$arg0 == null || ($arg1 != null && $comparison) ? null : $arg0" - val expr = - out match { - // πŸ”΄ `String.compareTo` takes a `String` and NOTHING else, so this arm is right only - // when BOTH operands render text -- and `leastCommonSuperType` answers VARCHAR for a - // MIXED pair too (`[BOOLEAN, KEYWORD]`, `[TIMESTAMP, KEYWORD]`). Without the guard - // `NULLIF(b, CAST(n AS KEYWORD))` and `NULLIF(d, CAST(i AS VARCHAR))` emitted - // `param1.compareTo(param2)` over a `Boolean` / `ZonedDateTime` receiver and threw - // PER DOCUMENT on ES 8.18.3 -- which in a computed column `ignoreFailure` SWALLOWS, - // so the column silently disappears. MEASURED: 32 corpus rows, all of them shapes - // that answered a value before the type fix of this same issue moved the supertype - // from the column's type to the chain's. They fall to the `==` arm below, which is - // what they emitted before and which Painless applies to any pair without throwing. - // - // The TEXT-against-NUMBER half of the same population is refused instead, by - // [[typeMismatchError]] -- `==` there would be a silent always-false. - case SQLTypes.Varchar if argTypes.forall(_.isText) => - nullIf(s"$arg0.compareTo($arg1) == 0") - // πŸ”΄ Keyed on what the RECEIVER's chain RENDERS, not on `out` (issue #373). - // `LocalTime` has no `isEqual`, and MEASURED on ES 8.18 before this, - // `NULLIF(CAST(d AS TIME), CAST('00:00:00' AS TIME))` failed the shard in a real - // `script_fields`. Neither `out` nor `expr1.out` says so: both are TIMESTAMP, the - // COLUMN's type -- `chainType` is the derivation that describes the rendering - // (#367), and it answers TIME. Two wrong keys were tried before measuring it. - case _: SQLTemporal if receiverIsTime => - nullIf(s"$arg0.compareTo($arg1) == 0") - case _: SQLTemporal => nullIf(s"$arg0.isEqual($arg1)") - // `==` is null-safe in Painless, so a numeric / boolean pair needs no guard. - case _ => s"$arg0 == $arg1 ? null : $arg0" + // Two temporals that do not render the same java.time type -- a DATE column's + // `ZonedDateTime` against a cast's `LocalDate` -- or a string literal READ as a temporal + // are compared as the instants they denote, a DATE as the instant its UTC day starts at + // (`ArithmeticExpression.comparedMillis`). `isEqual` failed between the first two + // (`NULLIF(COALESCE(d, d2), CAST('2024-01-31' AS DATE))`, all shards failed on 8.18.3); + // a literal is NEVER NULL, and ` != null` does not compile, so it is not + // guarded. + val temporalLiteral = Seq(firstReading, secondReading).exists { + case Some(_: ComparisonRule.TemporalReading) => true + case _ => false + } + def dateCarrying(t: SQLType): Boolean = + t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp + // (An ingest processor reads both arguments by their runtime class already -- the + // `processorInstant` arm below -- and its source is persisted: it keeps its bytes.) + val mixedTemporals = + !context.exists(_.isProcessor) && !receiverIsTime && + dateCarrying(ComparisonRule.declaredTypeOf(expr1)) && + dateCarrying(ComparisonRule.declaredTypeOf(expr2)) && { + val (java1, java2) = (argTypeOf(expr1), argTypeOf(expr2)) + !java1.isUnknown && !java2.isUnknown && java1 != java2 } + lazy val instants = + s"${ArithmeticExpression.comparedMillis(expr1, firstReading, arg0, context)} == " + + s"${ArithmeticExpression.comparedMillis(expr2, secondReading, arg1, context)}" + val expr = + if (secondReading.exists(_.isInstanceOf[ComparisonRule.TemporalReading])) + s"$arg0 == null || ($instants) ? null : $arg0" + else if (temporalLiteral) + s"$arg0 == null || ($arg1 != null && $instants) ? null : $arg0" + else if (mixedTemporals) nullIf(instants) + else + out match { + // πŸ”΄ `String.compareTo` takes a `String` and NOTHING else, so this arm is right only + // when BOTH operands render text -- and `leastCommonSuperType` answers VARCHAR for a + // MIXED pair too (`[BOOLEAN, KEYWORD]`, `[TIMESTAMP, KEYWORD]`). Without the guard + // `NULLIF(b, CAST(n AS KEYWORD))` and `NULLIF(d, CAST(i AS VARCHAR))` emitted + // `param1.compareTo(param2)` over a `Boolean` / `ZonedDateTime` receiver and threw + // PER DOCUMENT on ES 8.18.3 -- which in a computed column `ignoreFailure` SWALLOWS, + // so the column silently disappears. MEASURED: 32 corpus rows, all of them shapes + // that answered a value before the type fix of this same issue moved the supertype + // from the column's type to the chain's. They fall to the `==` arm below, which is + // what they emitted before and which Painless applies to any pair without throwing. + // + // The TEXT-against-NUMBER half of the same population is refused instead, by + // [[typeMismatchError]] -- `==` there would be a silent always-false. + case SQLTypes.Varchar if argTypes.forall(_.isText) => + nullIf(s"$arg0.compareTo($arg1) == 0") + // πŸ”΄ Keyed on what the RECEIVER's chain RENDERS, not on `out` (issue #373). + // `LocalTime` has no `isEqual`, and MEASURED on ES 8.18 before this, + // `NULLIF(CAST(d AS TIME), CAST('00:00:00' AS TIME))` failed the shard in a real + // `script_fields`. Neither `out` nor `expr1.out` says so: both are TIMESTAMP, the + // COLUMN's type -- `chainType` is the derivation that describes the rendering + // (#367), and it answers TIME. Two wrong keys were tried before measuring it. + case _: SQLTemporal if receiverIsTime => + nullIf(s"$arg0.compareTo($arg1) == 0") + // πŸ”΄ In an INGEST processor a column argument is the incoming document's raw JSON -- + // a string or epoch milliseconds -- so `isEqual` failed on every document (`String` + // has none) and `ignore_failure` stored NULL: `NULLIF(d, d2) - d2` was NULL on every + // row (found by review, measured on 8.18.3). Both are compared as the instants they + // denote, read by their runtime class (`SQLTypeUtils.processorInstant`); NULLIF still + // answers its FIRST argument as it came. + case _: SQLTemporal if context.exists(_.isProcessor) => + nullIf( + s"${SQLTypeUtils.processorInstant(arg0)}" + + s".isEqual(${SQLTypeUtils.processorInstant(arg1)})" + ) + case _: SQLTemporal => nullIf(s"$arg0.isEqual($arg1)") + // `==` is null-safe in Painless, so a numeric / boolean pair needs no guard. + case _ => s"$arg0 == $arg1 ? null : $arg0" + } context match { case Some(ctx) => ctx.addParam(LiteralParam(expr)) match { @@ -676,10 +751,23 @@ package object cond { for { _ <- expr1.validate() _ <- expr2.validate() - _ <- Validator.validateTypesMatching(expr1.out, expr2.out) } yield () } + /** The TYPE rules of `NULLIF`, asked once the column types are known (the lead's ruling of + * 2026-10-05): [[typeMismatchError]] (issue #382), then the ONE comparison rule + * (`ComparisonRule`) -- `NULLIF(a, b)` compares `a = b`, so it is judged as that comparison: + * on the types the two arguments are DECLARED with, a DATE comparable with a TIMESTAMP. + * `NULLIF(COALESCE(d, d2), CAST('2024-01-31' AS DATE))` compares two dates, and is accepted + * like `NULLIF(d, CAST('2025-01-01' AS DATE))` always was (the lead's ruling of 2026-09-23). + */ + override def typeError: Option[String] = + typeMismatchError.orElse { + ComparisonRule.mismatch(expr1, expr2).map { case (left, right) => + s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" + } + } + override def update(request: query.SingleSearch): NullIf = { this.copy( expr1 = expr1 match { @@ -735,19 +823,40 @@ package object cond { "Filter in WHERE, or compute the flag in a separate query." ) else if (conditions.isEmpty) Left("CASE WHEN requires at least one condition") - else if ( - expression.isEmpty && conditions.exists { case (cond, _) => cond.out != SQLTypes.Boolean } - ) - Left("CASE WHEN conditions must be of type BOOLEAN") - else if ( - expression.isDefined && conditions.exists { case (cond, _) => - !SQLTypeUtils.matches(cond.out, expression.get.out) - } - ) - Left("CASE WHEN conditions must be of the same type as the expression") else Right(()) } + /** The TYPE rules of the conditions, asked once the column types are known (the lead's ruling + * of 2026-10-05) and on what each operand RENDERS (`renderedTypeOf`): asked when the statement + * was parsed, `CASE WHEN flag THEN …` was refused because a BOOLEAN column is `Any` there. A + * type that is still unknown is accepted: Elasticsearch answers. + */ + override def typeError: Option[String] = + expression match { + case None if conditions.exists { case (cond, _) => + val t = renderedTypeOf(cond) + t != SQLTypes.Boolean && !t.isUnknown + } => + Some("CASE WHEN conditions must be of type BOOLEAN") + // Each value is compared with the expression by the ONE comparison rule (`ComparisonRule`): + // on their DECLARED types, a DATE comparable with a TIMESTAMP, a string literal read by + // what it is compared with -- `CASE n WHEN '3'`, `CASE d WHEN CAST('2024-01-31' AS DATE)`. + case Some(e) if conditions.exists { case (cond, _) => + ComparisonRule.mismatch(e, cond).isDefined + } => + Some("CASE WHEN conditions must be of the same type as the expression") + case _ => None + } + + /** The least common supertype of the BRANCHES' reported types -- the THEN results and the ELSE, + * never the `CASE ` operand -- as `COALESCE` reports its arms': `CASE … THEN d ELSE d2 + * END` is a DATE when both are. + */ + override def reportedType: SQLType = + SQLTypeUtils.leastCommonSuperType( + conditions.map { case (_, result) => result.reportedType } ++ default.map(_.reportedType) + ) + /** Whether the CASE EXPRESSION -- the receiver of every `WHEN` comparison -- renders a * `java.time.LocalTime`. `out` reports the CASE's result type and cannot say (issue #373). */ @@ -813,11 +922,124 @@ package object cond { // wrong as written. if (default.isEmpty) s"(def)($rendered)" else rendered + /** How `CASE WHEN …` compares its expression with each value -- the ONE comparison + * rule (`ComparisonRule`) decides what is compared; this decides how it renders. + * + * The historical rendering converts BOTH sides to `out`, the CASE's RESULT type, and is kept + * wherever that compares what SQL compares ([[WhenAsResult]]). It does not when: + * - the expression is a DATE or a TIMESTAMP and a value is a string literal read as one, the + * two render different java.time types, or the result is not a temporal: a `LocalDate` + * value against a column's `ZonedDateTime` failed the shard, and a numeric result compared + * `toEpochMilli()` with a `LocalDate` (`CASE d WHEN CAST('2024-01-31' AS DATE) THEN 1 ELSE + * 0 END` answered 0 for 2024-01-31) -- they are compared as the instants they denote + * ([[WhenInstants]]); + * - the expression is a number and the result is not: two numbers were compared as their + * strings (`"1.0"` against `"1"`) -- they are compared as numbers ([[WhenNumbers]]). An + * ingest processor keeps its (persisted) bytes unless a value is a string literal it now + * reads. + */ + private[this] def whenComparison( + expr: PainlessScript, + context: Option[PainlessContext] + ): WhenComparison = { + def dateCarrying(t: SQLType): Boolean = + t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp + val declared = ComparisonRule.declaredTypeOf(expr) + val readings = + conditions.map { case (cond, _) => ComparisonRule.readingOf(cond, declared) } + val processor = context.exists(_.isProcessor) + if (receiverIsTime) WhenAsResult + else if (dateCarrying(declared)) { + val temporalReading = + readings.exists(_.exists(_.isInstanceOf[ComparisonRule.TemporalReading])) + // a KNOWN mismatch only: with no schema attached a type is unknown, and the statement + // keeps the rendering it always had + val mismatch = + (!out.isInstanceOf[SQLTemporal] && !out.isUnknown) || + conditions.exists { case (cond, _) => + val rendered = javaTypeOf(cond) + !rendered.isUnknown && rendered != javaTypeOf(expr) + } + if (temporalReading || (!processor && mismatch)) WhenInstants else WhenAsResult + } else if (declared.isNumber && !out.isNumber && !out.isUnknown) { + if (!processor || readings.exists(_.isDefined)) WhenNumbers else WhenAsResult + } else WhenAsResult + } + + /** The JAVA type an operand renders (#384's `Identifier.renderedType`): a DATE column's doc + * value is a `ZonedDateTime`, a cast to DATE a `LocalDate`. + */ + private[this] def javaTypeOf(operand: PainlessScript): SQLType = operand match { + case i: Identifier => i.renderedType + case other => other.out + } + + /** A `THEN` result, converted to the CASE's type -- the one rendering both comparisons share. + */ + private[this] def whenResult( + cond: PainlessScript, + res: PainlessScript, + context: Option[PainlessContext] + ): String = { + val name = + cond match { + case e: Expression => e.identifier.name + case f: FunctionWithIdentifier => f.identifier.name + case i: Identifier => i.name + case _ => "" + } + boxWhenNoDefault(res match { + case i: Identifier if i.name == name && name.nonEmpty => + i.withNullable(false) + SQLTypeUtils.coerce(i, out, context) + case _ => + SQLTypeUtils.coerce(res, out, context) + }) + } + + /** The `WHEN` arms of a [[WhenInstants]] or [[WhenNumbers]] comparison. The expression is + * rendered ONCE (a rendering is not idempotent) and, for instants, converted once; every value + * is guarded like the historical candidate (`checkCase`), and a NULL on either side matches + * nothing -- SQL's `NULL = NULL` is unknown, where Painless's `null == null` is true. + */ + private[this] def comparedWhens( + expr: PainlessScript, + kind: WhenComparison, + ctx: PainlessContext, + context: Option[PainlessContext] + ): String = { + val declared = ComparisonRule.declaredTypeOf(expr) + def bind(rendered: String): String = ctx.addParam(LiteralParam(rendered)).getOrElse(rendered) + def instant(operand: PainlessScript, ref: String): String = + bind( + s"($ref != null ? (def)(${ArithmeticExpression.comparedMillis(operand, None, ref, context)}) : null)" + ) + val e = bind(expr.painless(context)) + val key = if (kind == WhenInstants) instant(expr, e) else e + conditions + .map { case (cond, res) => + val r = whenResult(cond, res, context) + val test = ComparisonRule.readingOf(cond, declared) match { + case Some(t: ComparisonRule.TemporalReading) => s"$key == ${t.epochMillis}L" + case Some(scalar) => + s"$key == ${ComparisonRule.scalarPainless(scalar).getOrElse(cond.painless(context))}" + case None => + val c = bind(cond.painless(context)) + val value = if (kind == WhenInstants) instant(cond, c) else c + s"$value != null && $key == $value" + } + s"$key != null && $test ? $r" + } + .mkString(" : ") + } + override def painless(context: Option[PainlessContext] = None): String = { context match { case Some(ctx) => var cases = expression match { + case Some(expr) if whenComparison(expr, context) != WhenAsResult => + comparedWhens(expr, whenComparison(expr, context), ctx, context) case Some(expr) => // case with expression to evaluate val e = SQLTypeUtils.coerce(expr, out, context) val expParam = ctx.addParam( @@ -835,7 +1057,22 @@ package object cond { i.name case _ => "" } - val c = SQLTypeUtils.coerce(cond, out, context) + // A string literal is the value the expression READS it as -- the number or + // the boolean it spells (`ComparisonRule.readingOf`), brought to the type the + // two are compared in like any other value. `CASE b WHEN 'true'` used to parse + // `"true"` as a long (`Long.parseLong`) and fail the shard. + val c = + ComparisonRule + .readingOf(cond, ComparisonRule.declaredTypeOf(expr)) + .flatMap(reading => + ComparisonRule + .scalarPainless(reading) + .map( + SQLTypeUtils + .coerce(_, reading.sqlType, out, nullable = false, context) + ) + ) + .getOrElse(SQLTypeUtils.coerce(cond, out, context)) val r = boxWhenNoDefault(res match { case i: Identifier if i.name == name && name.nonEmpty => @@ -968,14 +1205,21 @@ package object cond { } } - /** N-ary numeric reducer shared by `GREATEST` / `LEAST`. + /** N-ary reducer shared by `GREATEST` / `LEAST`, over numbers or over dates and timestamps. * * Emits a right-folded nested ternary so a NULL argument is skipped and the whole expression * yields NULL only when every argument is NULL: pairwise(x, y) = (x == null ? y : (y == null ? x * : Math.{max|min}(x, y))) * - * Output is `SQLNumeric` because `Math.max` / `Math.min` only operate on numeric primitives. The - * base type narrows to the widest input numeric. + * Over numbers the base type narrows to the widest input numeric. Over dates and timestamps (the + * lead's ruling of 2026-10-05: every SQL engine compares them, and they failed in Elasticsearch + * before) the same fold runs over the epoch milliseconds each argument denotes -- a DATE its UTC + * day's start, through the conversion date arithmetic uses + * (`ArithmeticExpression.instantMillis`) -- so dates compare as instants whatever each one + * renders as (a doc value, a `LocalDate`, the clock), and a NULL is skipped exactly as it is for + * numbers. The result is the instant that wins, a UTC `ZonedDateTime` (TIMESTAMP is what it + * renders); per group, where a date is the number its metric is, it is that number. Its reported + * type is its arguments' common one: DATE for dates, TIMESTAMP when they mix. */ sealed trait NumericReducer extends TransformFunction[SQLAny, SQLNumeric] @@ -994,37 +1238,69 @@ package object cond { override def inputType: SQLAny = SQLTypes.Any - override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(argTypes) match { - case n: SQLNumeric => n - case _ => outputType - } + /** Are the arguments dates and timestamps? Decided on what each one RENDERS (`renderedTypeOf`, + * the types the type rule reads), once the column types are known: until then a column is + * `Any`, and the reducer is the numeric one it always was. + */ + lazy val overDates: Boolean = values.exists(v => NumericReducer.dateCarrying(renderedTypeOf(v))) + + override def baseType: SQLType = + if (overDates) SQLTypes.Timestamp + else + SQLTypeUtils.leastCommonSuperType(argTypes) match { + case n: SQLNumeric => n + case _ => outputType + } override def sql: String = s"$operator(${values.map(_.sql).mkString(", ")})" + /** An aggregate among the arguments makes this a per-group calculation, as it makes an + * arithmetic over them one (`BinaryFunction.hasAggregation`): `GREATEST(MAX(d), MAX(d2))` + * beside a GROUP BY is computed per group, where it used to be refused as a non-aggregated + * field while `GREATEST(MAX(d), MAX(d2)) - MIN(d)` was accepted. + */ + override def hasAggregation: Boolean = values.exists(_.hasAggregation) + override def checkIfNullable: Boolean = false override def validate(): Either[String, Unit] = - if (values.isEmpty) Left(s"$operator requires at least one argument") - else - // Accept numeric args and still-unresolved args (SQLAny / NULL, which - // extends SQLAny): a bare field has no known type until it is resolved - // against the index mapping, so we must not reject it here. Only reject - // args whose type is *definitively* non-numeric (string, temporal, - // boolean, …) β€” those would emit Math.{max,min}() on a non-numeric and - // fail at ES runtime. - values.find { v => - v.out match { - case _: SQLNumeric => false - case _: SQLAny => false - case _ => true - } - } match { - case Some(nonNumeric) => - Left( - s"$operator requires numeric arguments but got ${nonNumeric.out} for ${nonNumeric.sql}" - ) - case None => Right(()) + if (values.isEmpty) Left(s"$operator requires at least one argument") else Right(()) + + /** Numbers, or dates and timestamps -- one kind, as every SQL engine requires -- asked once the + * column types are known (the lead's ruling of 2026-10-05) and on what each argument RENDERS + * (`renderedTypeOf`). An argument whose type is still unknown (SQLAny / NULL, which extends + * SQLAny) is accepted: Elasticsearch answers. An argument of any other kind (a string, a + * boolean, a TIME), or numbers mixed with dates, is refused by name. + */ + override def typeError: Option[String] = { + val known = values + .map(v => (v: PainlessScript) -> renderedTypeOf(v)) + .filterNot(_._2.isInstanceOf[SQLAny]) + val numbers = known.filter(_._2.isInstanceOf[SQLNumeric]) + val dates = known.filter(a => NumericReducer.dateCarrying(a._2)) + val offenders = + known.find(a => + !a._2.isInstanceOf[SQLNumeric] && !NumericReducer.dateCarrying(a._2) + ) match { + case Some(other) => Seq(other) + case None if numbers.nonEmpty && dates.nonEmpty => Seq(dates.head, numbers.head) + case None => Nil } + // each argument is NAMED by the type DESCRIBE shows (a DATE column RENDERS as a TIMESTAMP) + def named(arg: PainlessScript): String = (arg match { + case aggregate: Identifier if aggregate.isAggregation => renderedTypeOf(aggregate) + case other => other.reportedType + }).typeId + if (offenders.isEmpty) None + else + Some( + s"$operator requires numeric arguments, or date and timestamp arguments, but got " + + offenders.map { case (arg, _) => s"${named(arg)} for ${arg.sql}" }.mkString(" and ") + ) + } + + /** The least common supertype of the arguments' REPORTED types, as `COALESCE` reports. */ + override def reportedType: SQLType = SQLTypeUtils.leastCommonSuperType(reportedArgTypes) override def nullable: Boolean = values.forall(_.nullable) @@ -1049,40 +1325,97 @@ package object cond { case _ => false })) + // Pair each rendered arg with its nullability so non-nullable args (e.g. + // numeric literals, which render inline as primitives) are NOT guarded + // with `== null` β€” Painless rejects ` == null` at compile time. + // Right-fold pairwise: pairwise(a, pairwise(b, pairwise(c, …))). + // The combined sub-tree is itself nullable only when BOTH sides can be + // null, which lets us drop the guard on parent levels too. + private def fold(args: List[(String, Boolean)]): (String, Boolean) = + args match { + case Nil => + throw new IllegalArgumentException(s"$operator requires at least one argument") + case (s, n) :: Nil => (s.trim, n) + case (s, xNullable) :: rest => + val x = s.trim + val (y, yNullable) = fold(rest) + val expr = + (xNullable, yNullable) match { + case (true, true) => s"($x == null ? $y : ($y == null ? $x : $mathFn($x, $y)))" + case (true, false) => s"($x == null ? $y : $mathFn($x, $y))" + case (false, true) => s"($y == null ? $x : $mathFn($x, $y))" + case (false, false) => s"$mathFn($x, $y)" + } + // result is null only if every branch can be null + (expr, xNullable && yNullable) + } + override def toPainlessCall( callArgs: List[String], context: Option[PainlessContext] - ): String = { - // Pair each rendered arg with its nullability so non-nullable args (e.g. - // numeric literals, which render inline as primitives) are NOT guarded - // with `== null` β€” Painless rejects ` == null` at compile time. - // Right-fold pairwise: pairwise(a, pairwise(b, pairwise(c, …))). - // The combined sub-tree is itself nullable only when BOTH sides can be - // null, which lets us drop the guard on parent levels too. - def fold(args: List[(String, Boolean)]): (String, Boolean) = - args match { + ): String = + if (overDates) overDatesPainless(callArgs, context) + else + callArgs match { case Nil => throw new IllegalArgumentException(s"$operator requires at least one argument") - case (s, n) :: Nil => (s.trim, n) - case (s, xNullable) :: rest => - val x = s.trim - val (y, yNullable) = fold(rest) - val expr = - (xNullable, yNullable) match { - case (true, true) => s"($x == null ? $y : ($y == null ? $x : $mathFn($x, $y)))" - case (true, false) => s"($x == null ? $y : $mathFn($x, $y))" - case (false, true) => s"($y == null ? $x : $mathFn($x, $y))" - case (false, false) => s"$mathFn($x, $y)" - } - // result is null only if every branch can be null - (expr, xNullable && yNullable) + case x :: Nil => x + case _ => fold(callArgs.zip(values.map(nullableArgument(_, context))))._1 } - callArgs match { - case Nil => - throw new IllegalArgumentException(s"$operator requires at least one argument") - case x :: Nil => x - case _ => fold(callArgs.zip(values.map(nullableArgument(_, context))))._1 + /** The same fold over the epoch milliseconds each date denotes, then the instant that wins. + * + * Per document each argument is converted once, under a name, behind its own NULL guard (a + * NULL is skipped by the fold, never converted), and the winner is handed back as the UTC + * `ZonedDateTime` it denotes. Per group -- no context, or a view's per-group calculation -- a + * metric IS the epoch milliseconds Elasticsearch computed for its date, and may be NULL in a + * group filter: it is read through the same conversion (a DATE metric is its UTC day's start, + * as date arithmetic reads it per group: a `date` field accepts a date-time, and its MAX keeps + * that time), behind a NULL guard where it can be NULL, and the winner stays that number, as a + * date aggregate is there. A NULL literal can never win: it is left out. + */ + private def overDatesPainless( + callArgs: List[String], + context: Option[PainlessContext] + ): String = { + if (callArgs.isEmpty) + throw new IllegalArgumentException(s"$operator requires at least one argument") + val perDocument = context.filterNot(_.isTransform) + val millis = callArgs.zip(values).filterNot(a => ArithmeticExpression.nullLiteral(a._2)).map { + case (ref, argument) => + val nullable = nullableArgument(argument, context) + perDocument match { + case Some(ctx) => + val name = + if (ArithmeticExpression.isBareName(ref.trim)) ref.trim + else ctx.bindLocal(ref, "lv") + val converted = ArithmeticExpression.instantMillis(argument, name, context) + val guarded = + if (nullable) s"($name == null ? null : (def)($converted))" else converted + (ctx.bindLocal(guarded, "lv"), nullable) + case None if argument.isAggregation => + val metric = ref.trim + val converted = ArithmeticExpression.instantMillis(argument, metric, context) + // A NULL metric (a group filter's, over a group with none of its values) is handed + // back as it is -- the same NULL, which the fold skips. The group filter's gate + // (`MetricSelectorScript.disqualifyRendering`) refuses a rendering that spells + // `? null :` or reads `def`, so neither is written: the conditional is a `def` by its + // first branch, the metric itself. + if (converted == metric) (metric, nullable) + else if (nullable) (s"($metric == null ? $metric : $converted)", nullable) + else (converted, nullable) + case None => + (ArithmeticExpression.instantMillis(argument, ref.trim, context), nullable) + } + } + if (millis.isEmpty) return if (perDocument.isEmpty) "((def) null)" else "null" + val (winner, nullable) = fold(millis) + perDocument match { + case Some(ctx) => + val name = if (nullable) ctx.bindLocal(winner, "lv") else s"($winner)" + val instant = s"Instant.ofEpochMilli((long) $name).atZone(ZoneId.of('Z'))" + if (nullable) s"($name == null ? null : $instant)" else instant + case None => winner } } } @@ -1108,4 +1441,28 @@ package object cond { case other => other }) } + + object NumericReducer { + + /** A type that carries a date -- DATE, TIMESTAMP, DATETIME, or a temporal function's TEMPORAL + * -- which `GREATEST` / `LEAST` compare as instants. A TIME carries none. + */ + private[sql] def dateCarrying(t: SQLType): Boolean = + t == SQLTypes.Date || t == SQLTypes.Timestamp || t == SQLTypes.DateTime || + t == SQLTypes.Temporal + + /** Is this operand -- or the nameless identifier carrying it -- `GREATEST` / `LEAST` over + * dates? Per group its value is the epoch milliseconds of the date that wins + * (`ArithmeticExpression.epochMillis`). + */ + private[sql] def overDates(operand: PainlessScript): Boolean = operand match { + case reducer: NumericReducer => reducer.overDates + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (reducer: NumericReducer) :: Nil => reducer.overDates + case _ => false + } + case _ => false + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index 6690afcd0..9cf99f93e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -28,6 +28,7 @@ import app.softnetwork.elastic.sql.{ Updateable } import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression package object convert { @@ -112,7 +113,7 @@ package object convert { } case _ => // do nothing } - val ret = SQLTypeUtils.coerce(base, value.baseType, targetType, value.nullable, context) + val ret = SQLTypeUtils.coerce(base, sourceType, targetType, sourceNullable, context) val bloc = ret.startsWith("{") && ret.endsWith("}") val retWithBrackets = if (bloc) ret else s"{ $ret }" if (!safe) ret @@ -151,6 +152,46 @@ package object convert { } } + /** The arithmetic this conversion converts, when its operand IS one: `(n + x)::BIGINT`, whose + * operand is the nameless identifier carrying that one arithmetic. + */ + private def arithmetic: Option[ArithmeticExpression] = value match { + case a: ArithmeticExpression => Some(a) + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (a: ArithmeticExpression) :: Nil => Some(a) + case _ => None + } + case _ => None + } + + /** The type the conversion converts FROM: what an arithmetic operand COMPUTES (its own type, + * which a cast no longer overwrites -- `ArithmeticExpression.cast`), or the operand's type. + * + * πŸ”΄ The nameless identifier around an arithmetic folds its chain from the arithmetic's INPUT + * type, so it answered NUMERIC for `(n + x)`, and `NUMERIC -> BIGINT` has no conversion arm: + * the cast was silently ignored (`(n + x)::BIGINT` answered 3.25 on main). + */ + private def sourceType: SQLType = arithmetic.map(_.out).getOrElse(value.baseType) + + /** Can the value converted be NULL? Asked of the chain's innermost PRODUCER: the nameless + * identifier around an arithmetic answers `false` whatever its operands hold, so the + * conversion was emitted unguarded -- `((long) )` failed with `Cannot cast null to a + * primitive type`, `String.valueOf()` answered the text "null", and ` != 0` + * answered TRUE. Guarded, the value is bound once and the conversion applies to it when it is + * not NULL (`SQLTypeUtils.coerce`'s nullable form): `(v == null ? null : conversion)`. + */ + private def sourceNullable: Boolean = + value.nullable || (value match { + case i: Identifier if i.name.trim.isEmpty => + i.functions.lastOption.exists { + case a: ArithmeticExpression => a.nullable + case _ => false + } + case a: ArithmeticExpression => a.nullable + case _ => false + }) + override def roundingScript: Option[String] = DateMathRounding(targetType) override def dateMathScript: Boolean = isTemporal diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 89b330323..bd1d58e6c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -74,10 +74,10 @@ package object time { } } - override def validate(): Either[String, Unit] = interval.checkType(out) match { - case Left(err) => Left(err) - case Right(_) => Right(()) - } + /** The interval's unit against the type it applies to, asked once the column types are known + * (the lead's ruling of 2026-10-05), never when the statement is parsed. + */ + override def typeError: Option[String] = interval.checkType(out).left.toOption override def toPainless(base: String, idx: Int, context: Option[PainlessContext]): String = { context match { @@ -380,6 +380,20 @@ package object time { case _ => } super.toPainlessCall(callArgs, context) + // πŸ”΄ A DAY truncation of a date-only receiver is the receiver itself, and `truncatedTo` + // would fail on it: `DATE_TRUNC(ts, DAY) = '2024-01-31'` (or `> '…'`, or `= CAST(… AS + // DATE)`) narrows `ts` to a `LocalDate` to meet the date, and `LocalDate.truncatedTo` does + // not exist (all shards failed, MEASURED on 8.18.3). Rendered as the start of that day, as + // a date: the same value, through methods a `LocalDate` has -- and a `ZonedDateTime` + // receiver misjudged as date-only fails loudly on `atStartOfDay` rather than silently + // keeping its time. + // + // ⚠️ DAY only. HOUR, MINUTE and SECOND keep `truncatedTo`: on a receiver NARROWED to its + // date the time is already gone, and SQL compares `DATE_TRUNC(ts, HOUR)` with a date as + // the instant the date starts at, which a date cannot be compared as -- that comparison + // still fails loudly, rather than answering for every hour of the day. + case TimeUnit.DAYS if receiverIsDateOnly => + ".atStartOfDay(ZoneId.of('Z')).toLocalDate()" case _ => super.toPainlessCall(callArgs, context) } } @@ -610,12 +624,14 @@ package object time { * and `DATE_DIFF` read a calendar date through it too, off a row-level operand that is not a * column: a zoned value there may be Elasticsearch 6.8's doc value, which is no `java.time` * type until `withZoneSameInstant` (as in [[utcZoned]]) returns the `ZonedDateTime` it wraps. + * Date arithmetic (`ArithmeticExpression`) reads such an operand through it for the same + * reason: one conversion, shared. * * `ref` is read up to four times, so it is a name or an expression cast to `def` (the methods * below are resolved at runtime). NULL stays NULL: the MONTH, QUARTER and YEAR calls bind this * value in the script's prologue, which runs before the null guard. */ - private[time] def utcInstant(ref: String): String = + private[sql] def utcInstant(ref: String): String = s"($ref == null ? null : $ref instanceof LocalDate ? $ref.atStartOfDay(ZoneId.of('Z')) : $ref instanceof LocalDateTime ? $ref.atZone(ZoneId.of('Z')) : $ref.withZoneSameInstant(ZoneId.of('Z')))" /** A column a SEARCH script reads, as a genuine `java.time.ZonedDateTime` in UTC; NULL stays diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 6b904e023..692e39b73 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -18,10 +18,15 @@ package app.softnetwork.elastic.sql.operator.math import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.`type`._ -import app.softnetwork.elastic.sql.function.{BinaryFunction, TransformFunction} +import app.softnetwork.elastic.sql.function.{BinaryFunction, FunctionChain, TransformFunction} +import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction +import app.softnetwork.elastic.sql.function.cond.{functionsOf, NumericReducer} +import app.softnetwork.elastic.sql.function.time.DateDiff import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.query.NestedElement +import scala.util.Try + case class ArithmeticExpression( left: PainlessScript, operator: ArithmeticOperator, @@ -36,7 +41,24 @@ case class ArithmeticExpression( override def inputType: SQLNumeric = SQLTypes.Numeric override def outputType: SQLNumeric = SQLTypes.Numeric - override def applyType(in: SQLType): SQLType = in + /** Date arithmetic hands its OWN type to the chain it heads, so that a cast over it converts from + * what it computes: `(d - d2)::TIMESTAMP` is a number of days cast to TIMESTAMP, which is core's + * epoch-millisecond conversion, as for any other number. Every other arithmetic keeps handing on + * the type it is given, as it always has. + */ + override def applyType(in: SQLType): SQLType = dateArithmetic.flatMap(_.runtimeType).getOrElse(in) + + /** A cast OVER this expression converts what it computes; it does not change what it computes. + * + * πŸ”΄ A conversion's constructor casts its operand (`value.cast(targetType)`), and on an + * arithmetic that set `out` -- the type the arithmetic coerces its OWN operands to. So `(x * + * 2)::INT` truncated `x` BEFORE multiplying (1.6 * 2 answered 2, not 3), and `(n + x)::BIGINT` + * -- whose operands are nullable, so not coerced at all -- answered 3.25 while the conversion, + * reading the cast type back as its source, emitted nothing. MEASURED on main. The arithmetic + * now keeps its own type and the conversion applies core's cast rule to its RESULT + * (`convert.Conversion`), as SQL engines do: `(n + x)::BIGINT` is 3 for 1 + 2.25. + */ + override def cast(targetType: SQLType): SQLType = out override def sql: String = { val expr = s"${left.sql}$operator${right.sql}" @@ -70,10 +92,23 @@ case class ArithmeticExpression( * The non-`Identifier` arm keeps today's `_.out` exactly: a nested arithmetic renders what its * own `out` says, and a literal's `out` already carries any cast applied to it. */ - private def argTypeOf(operand: PainlessScript): SQLType = operand match { - case i: Identifier => i.chainType - case other => other.out - } + private def argTypeOf(operand: PainlessScript): SQLType = + literalNumber(operand) + .map(_.sqlType) + .getOrElse(operand match { + case i: Identifier => ArithmeticExpression.identifierType(i) + case other => other.out + }) + + /** A string LITERAL beside a NUMBER is the number it spells -- the ONE comparison rule's reading + * (`ComparisonRule.readingOf`), as PostgreSQL reads `n + '1'`: 4 for n = 3, where the + * concatenation `"31"` was answered. Beside anything else it stays what it is (a temporal beside + * it is date arithmetic, which reads it itself). + */ + private def literalNumber(operand: PainlessScript): Option[ComparisonRule.NumberReading] = + ComparisonRule + .readingOf(operand, ComparisonRule.declaredTypeOf(if (operand eq left) right else left)) + .collect { case number: ComparisonRule.NumberReading => number } override def argTypes: List[SQLType] = args.map(argTypeOf) @@ -125,11 +160,73 @@ case class ArithmeticExpression( * Measured on a clean clone at `origin/main`, so it is pre-existing and UNFILED. The documented * way to truncate is to compute the quotient into a column and cast THAT column. */ - override def baseType: SQLType = { - val folded = SQLTypeUtils.leastCommonSuperType(argTypes) - if (operator == DIVIDE && (folded.isNumber || folded.isUnknown)) SQLTypes.Double else folded + override def baseType: SQLType = dateArithmetic.flatMap(_.runtimeType).getOrElse { + val types = argTypes + // πŸ”΄ `+` and `-` answer UNKNOWN while an operand's type is unknown -- a column before its + // schema is attached, a field no schema describes. They are the two operators a DATE may + // take, so the fold could not know the answer: `d + 7` folded `[ANY, BIGINT]` to BIGINT, and + // `d + 7 > CURRENT_DATE` was refused as BIGINT against DATE before the column's type was ever + // read. `*`, `/` and `%` keep their fold: no date survives them. + if ((operator == ADD || operator == SUBTRACT) && types.exists(_.isUnknown)) SQLTypes.Any + else { + val folded = SQLTypeUtils.leastCommonSuperType(types) + if (operator == DIVIDE && (folded.isNumber || folded.isUnknown)) SQLTypes.Double else folded + } } + /** What the date-arithmetic rules make of this expression (see [[ArithmeticExpression]] 's + * companion), or `None` when neither operand is a DATE, a TIMESTAMP or a DATETIME -- every such + * expression keeps the derivation and the rendering it always had. + * + * πŸ”΄ Decided on the operands' DECLARED types, never on `out`. Elasticsearch stores DATE, + * TIMESTAMP and DATETIME alike as a `date`, so the type a query hands Painless (`runtimeType`) + * is TIMESTAMP for all three, and the difference between a DATE and a TIMESTAMP -- days between + * two dates, or a fraction of a day between two instants -- lives only in the declaration. + * Folding `out` is what typed `MAX(d) - MIN(d)` TIMESTAMP and answered it in milliseconds. + * + * Computed once per instance: `out` reads `baseType` on every call, and the operands of an + * instance do not change -- resolution (`update`) builds a new one. + */ + private[sql] lazy val dateArithmetic: Option[ArithmeticExpression.DateArithmetic] = + ArithmeticExpression.dateArithmetic(this) + + /** The type a consumer DECLARING a column for this expression reports: DATE for a date moved by + * whole days, although Painless renders it as the UTC instant that day starts at. + */ + override def reportedType: SQLType = dateArithmetic.flatMap(_.reportedType).getOrElse(out) + + /** The by-name refusal the DATE-ARITHMETIC rules give this expression, if any: a temporal operand + * under `*`, `/` or `%`, two temporals added, a number minus a temporal, or a temporal combined + * with something that is neither a number nor a temporal. + */ + def dateArithmeticError: Option[String] = + dateArithmetic.collect { case ArithmeticExpression.Refused(message) => message } + + /** The by-name refusal this expression earns once the types of its operands are known, if any: + * [[dateArithmeticError]], or -- for any other arithmetic -- two operands whose types do not + * match, the rule `validate()` used to apply when the statement was parsed, where every column + * is still `Any`. + * + * πŸ”΄ Asked AFTER the schema is attached (`SingleSearch.validateResolved`), never at parse time: + * no type is refused before the column types are known (the lead's ruling of 2026-10-05). Each + * operand's type is the one it RENDERS ([[ArithmeticExpression.identifierType]]), not `out`, + * which a resolved column answers with its column's type whatever function it wears. + */ + override def typeError: Option[String] = + dateArithmetic match { + case Some(_) => dateArithmeticError + case None => + Validator.validateTypesMatching(argTypeOf(left), argTypeOf(right)).left.toOption + } + + /** [[dateArithmeticError]], here and in every arithmetic among the operands, however deep. */ + private[sql] def dateArithmeticErrors: Seq[String] = + dateArithmeticError.toSeq ++ args.flatMap { + case nested: ArithmeticExpression => nested.dateArithmeticErrors + case chain: FunctionChain => ArithmeticExpression.typeErrorsOf(chain) + case _ => Nil + } + /** Is this the numeric division the ruling above governs? Keyed on the coercion TARGET rather * than on [[baseType]] so that an explicit `CAST(a / b AS INTEGER)` -- which sets `out` -- is * integer division again, with no arm of its own. @@ -215,10 +312,28 @@ case class ArithmeticExpression( * leaves the whole estate green, so nothing here distinguishes the two for a non-`Identifier` * operand. Measured, recorded on #382, and not smuggled in. */ - private def renderedType(operand: PainlessScript): SQLType = operand match { - case i: Identifier => i.chainType - case other => other.baseType - } + private def renderedType(operand: PainlessScript): SQLType = + literalNumber(operand) + .map(_.sqlType) + .getOrElse(operand match { + case i: Identifier => ArithmeticExpression.identifierType(i) + case other => other.baseType + }) + + /** An operand as it renders, a string literal read as a number ([[literalNumber]]) as that + * number. + */ + private def operandPainless( + operand: PainlessScript, + idx: Int, + context: Option[PainlessContext] + ): String = + literalNumber(operand) + .map(_.painless) + .getOrElse(operand match { + case t: TransformFunction[_, _] => t.toPainless("", idx + 1, context) + case _ => operand.painless(context) + }) /** πŸ”΄ A numeric coercion is skipped only over a NULLABLE operand, and the reason is exactly what * Painless refuses -- a primitive cast over a null-guarded expression, in BOTH directions, @@ -298,15 +413,32 @@ case class ArithmeticExpression( private def needsDoubleCast(target: SQLType): Boolean = floatingDivision(target) && !args.exists(operand => isFloating(emittedType(operand, target))) - override def validate(): Either[String, Unit] = { + /** STRUCTURE only: the operands. Every TYPE rule of this expression -- the date-arithmetic rules + * and the operand match alike -- is [[typeError]], asked once the column types are known (the + * lead's ruling of 2026-10-05: no refusal before the column types are known). Asked here, at + * parse time, it refused `d + 7 > CURRENT_DATE` as BIGINT against DATE before `d` had a type. + */ + override def validate(): Either[String, Unit] = for { _ <- left.validate() _ <- right.validate() - _ <- Validator.validateTypesMatching(left.out, right.out) } yield () - } - override def nullable: Boolean = left.nullable || right.nullable + /** Is an operand a NULL literal, bare or cast, in date arithmetic? Then the expression is NULL on + * every row, whatever the other operand holds. + */ + private def nullValued: Boolean = + dateArithmetic.exists { + case ArithmeticExpression.NullOperand => true + case _: ArithmeticExpression.Computed => + ArithmeticExpression.nullLiteral(left) || ArithmeticExpression.nullLiteral(right) + case _ => false + } + + /** A NULL literal beside a temporal makes the expression NULL on every row, whatever the other + * operand holds, so a consumer must guard it. + */ + override def nullable: Boolean = left.nullable || right.nullable || nullValued override def toPainless(base: String, idx: Int, context: Option[PainlessContext]): String = { context match { @@ -315,6 +447,12 @@ case class ArithmeticExpression( ctx.addParam(right) case _ => } + if (nullValued) return s"$base${nullOperandPainless(context)}" + dateArithmetic match { + case Some(computed: ArithmeticExpression.Computed) => + return s"$base${dateArithmeticPainless(computed, idx, context)}" + case _ => + } if (nullable) { // The coercion target, asked ONCE -- see `floatingDivision` for why that matters. val target = out @@ -331,10 +469,8 @@ case class ArithmeticExpression( else SQLTypeUtils.coerce(rendered, renderedType(operand), target, nullable = false, context) - def render(operand: PainlessScript): String = operand match { - case t: TransformFunction[_, _] => coerced(t.toPainless("", idx + 1, context), operand) - case _ => coerced(operand.painless(context), operand) - } + def render(operand: PainlessScript): String = + coerced(operandPainless(operand, idx, context), operand) /** A rendering that IS a Painless name needs no local of its own. */ def isBareName(e: String): Boolean = @@ -444,10 +580,179 @@ case class ArithmeticExpression( s"$base${painless(context)}" } - override def painless(context: Option[PainlessContext]): String = { + override def painless(context: Option[PainlessContext]): String = + if (nullValued) nullOperandPainless(context) + else + dateArithmetic match { + case Some(computed: ArithmeticExpression.Computed) => + dateArithmeticPainless(computed, 0, context) + case _ => numericPainless(context) + } + + /** A NULL literal beside a temporal: NULL. A `bucket_script` returns a Number, and a bare `null` + * is an Object to Painless -- `Cannot cast from [java.lang.Object] to [java.lang.Number]` at + * compile time (MEASURED, ES 8.18.3); typed `def`, it compiles and the group has no value, as it + * has for any NULL operand. + */ + private def nullOperandPainless(context: Option[PainlessContext]): String = { + val value = if (context.isEmpty) "((def) null)" else "null" + if (group) s"($value)" else value + } + + /** Date arithmetic, rendered in the milliseconds Elasticsearch stores a `date` in -- ONE formula + * per rule for every venue, which differ only in how an operand becomes a number of milliseconds + * and how a temporal result is handed back. + * + * - Per group (`bucket_script`, `bucket_selector`, a view's per-group calculation), which is + * every rendering without a context, an aggregate is the metric Elasticsearch computed: a + * number, epoch millis for a date. A temporal result is handed back as epoch millis too, + * which is what a date aggregate is there and what a `bucket_selector` compares a date with. + * - Per document, a column is the `date` value Elasticsearch hands the script (`toInstant()` + * exists on the `JodaCompatibleZonedDateTime` of 6.8 as on the `ZonedDateTime` of 7 and up), + * or, in an ingest processor, the raw value of the document, parsed as a CAST parses it. + * Anything else is read by its RUNTIME type (`DateDiff.utcInstant`): `CURRENT_DATE` is a + * `LocalDate` in a query and the current instant in a processor. A temporal result is the + * UTC `ZonedDateTime` it denotes -- the one temporal an ingest processor may assign to a + * field. + * + * A DATE operand is its UTC calendar day: the days between two dates are the days between their + * days, and a date moved by days starts at the start of its day. A NULL operand makes the result + * NULL, behind the same guard every arithmetic emits. + */ + private def dateArithmeticPainless( + shape: ArithmeticExpression.Computed, + idx: Int, + context: Option[PainlessContext] + ): String = { + import ArithmeticExpression.{ + isBareName, + DayMillis, + DaysBetween, + DaysShift, + FractionalDayMillis, + FractionalDaysBetween + } + + var declarations = "" + + def grouped(e: String): String = if (group) s"($e)" else e + + // the operand as it renders -- no coercion: the conversion below starts from what it IS + // A string literal that is a number counts as that number (`d + '1'`), so it renders as one. + def rendered(operand: PainlessScript): String = + (operand match { + case i: Identifier if i.functions.isEmpty => context.flatMap(_.get(i)) + case _ => ArithmeticExpression.numericString(operand).map(_._1) + }).getOrElse { + val r = operand match { + case t: TransformFunction[_, _] => t.toPainless("", idx + 1, context) + case _ => operand.painless(context) + } + if (PainlessOperandForm.placeable(r)) r else context.flatMap(_.get(operand)).getOrElse(r) + } + + // An operand that can be NULL when it runs: one whose own nullability says so, or one that + // READS a document field through a function. The nameless identifier around a `CASE`, a + // `COALESCE` or a `NULLIF` answers `nullable = false` whatever its branches hold, so it went + // unguarded and a NULL branch failed the script on `.toInstant()` instead of answering NULL. + // (An AGGREGATE is the metric Elasticsearch computed: a group with no value never runs the + // per-group script -- the `bucket_script`'s gap policy skips it -- so it keeps its own answer.) + def mayBeNull(operand: PainlessScript): Boolean = + operand.nullable || (operand match { + case i: Identifier if !i.isAggregation => + i.dependencies.nonEmpty || i.functions.exists(_.nullable) + case _ => false + }) + + // A string literal read as the temporal it spells beside the other operand (`d - + // '2024-01-31'`): the instant it denotes, known here, so it is a constant -- never NULL, never + // parsed per document. + def literalInstant(operand: PainlessScript): Option[Long] = + ArithmeticExpression + .temporalString(operand, if (operand eq left) right else left) + .map(_.epochMillis) + + // a nullable operand the guard reads is evaluated once, under a name + def named(operand: PainlessScript, fallback: String): String = + literalInstant(operand).map(millis => s"${millis}L").getOrElse { + val r = rendered(operand) + if (!mayBeNull(operand) || isBareName(r)) r + else + context match { + case Some(ctx) => ctx.bindLocal(r, "lv") + case None => + declarations += s"def $fallback = $r; " + fallback + } + } + + // Per group there is no document: every operand is a number already, and so is the result. + val perGroup = context.isEmpty + + // the UTC day of a temporal operand, and the instant it denotes -- a DATE its day's start -- + // in epoch milliseconds: the conversion GREATEST / LEAST over dates shares + def day(operand: PainlessScript, ref: String): String = + literalInstant(operand) + .map(millis => s"${Math.floorDiv(millis, 86400000L)}L") + .getOrElse(ArithmeticExpression.epochDay(operand, ref, context)) + + def instant(operand: PainlessScript, ref: String): String = + literalInstant(operand) + .map(millis => s"${millis}L") + .getOrElse(ArithmeticExpression.instantMillis(operand, ref, context)) + + def days(ref: String, fractional: Boolean): String = + if (fractional) s"Math.round($ref * $FractionalDayMillis)" else s"$ref * $DayMillis" + + // Per document, a whole number of days or of milliseconds is a `long` again; per group it stays + // the double a `bucket_script` returns anyway, and a `bucket_selector` compares as it is. + def integral(e: String): String = if (perGroup) e else s"((long) $e)" + + val l = named(left, s"lv$idx") + val r = named(right, s"rv$idx") + + val value = shape match { + case DaysBetween => integral(s"(${day(left, l)} - ${day(right, r)})") + case FractionalDaysBetween => + s"((${instant(left, l)} - ${instant(right, r)}) / $FractionalDayMillis)" + case DaysShift(temporalLeft, _, fractional) => + val ((temporal, t), n) = if (temporalLeft) ((left, l), r) else ((right, r), l) + val moved = + s"(${instant(temporal, t)} ${operator.painless(context)} ${days(n, fractional)})" + if (perGroup) moved else s"Instant.ofEpochMilli(${integral(moved)}).atZone(ZoneId.of('Z'))" + } + + // (a literal instant is a primitive constant: never NULL, and ` == null` does not compile) + val guards = + (if (mayBeNull(left) && literalInstant(left).isEmpty) List(s"$l == null") else Nil) ::: + (if (mayBeNull(right) && literalInstant(right).isEmpty) List(s"$r == null") else Nil) + val body = + if (guards.isEmpty) value else s"(${guards.mkString(" || ")}) ? null : (def)($value)" + s"$declarations${grouped(body)}" + } + + private def numericPainless(context: Option[PainlessContext]): String = { val target = out - val l = SQLTypeUtils.coerce(left, target, context) - val r = SQLTypeUtils.coerce(right, target, context) + // πŸ”΄ An aggregate is coerced FROM what it computes (`AggregateFunction.outputTypeOf`), never + // from the column it aggregates: per group it is the metric Elasticsearch already computed, a + // number, and `COUNT(d) - 1` used to read the DATE column's type and emit + // `params.count_d.toInstant().toEpochMilli() - 1`. + def coerced(operand: PainlessScript): String = operand match { + case metric: Identifier if metric.isAggregation => + SQLTypeUtils.coerce( + metric.painless(context), + ArithmeticExpression.identifierType(metric), + target, + nullable = false, + context + ) + case literal if literalNumber(literal).isDefined => + val number = literalNumber(literal).get + SQLTypeUtils.coerce(number.painless, number.sqlType, target, nullable = false, context) + case _ => SQLTypeUtils.coerce(operand, target, context) + } + val l = coerced(left) + val r = coerced(right) val core = s"$l ${operator.painless(context)} $r" /* The non-nullable rendering: both operands are literals or system values, so each is coerced * to `out` and a DOUBLE `out` is all it takes to make `SELECT 10 / 3` answer 3.333 instead of @@ -500,3 +805,350 @@ case class ArithmeticExpression( None } } + +/** The date-arithmetic rules: what `+` and `-` mean when an operand is a DATE, a TIMESTAMP or a + * DATETIME, and which combinations are type errors. + * + * | Expression | Result | + * |:-------------------------------------------------|:-------------------------------------------| + * | DATE - DATE | the calendar days between the two (BIGINT) | + * | TIMESTAMP - TIMESTAMP, or a DATE and a TIMESTAMP | the milliseconds between, in days (DOUBLE) | + * | DATE +/- a whole number n | the date n days later / earlier (DATE) | + * | DATE +/- a fractional number n | n days later / earlier, as an instant | + * | TIMESTAMP +/- a number n | n days later / earlier (TIMESTAMP) | + * | n + a temporal | the temporal + n | + * | a NULL literal beside a temporal | NULL | + * | two temporals added; a temporal under * / or %; | a type error, refused by name before | + * | n - a temporal; a temporal and a non-number | anything runs | + * + * They are the lead's ruling of 2026-10-04, aligned on the SQL engines a JDBC client knows: + * PostgreSQL, DuckDB, Oracle and Snowflake subtract two dates into days; PostgreSQL, DuckDB, + * Oracle and BigQuery move a date by a number of days; Oracle, whose DATE carries a time of day, + * answers fractional days, and moves a date by a fractional number of days into a time of day; + * PostgreSQL, DuckDB, Oracle, Trino, Snowflake and SQL Server refuse the other combinations. + * + * A temporal operand is recognised by its DECLARED type (`Token.reportedType`): a TIME, and an + * operand whose type is unknown (a column with no schema attached), leave the expression to the + * arithmetic it always was -- except under `*`, `/` and `%`, which refuse a temporal operand + * whatever the other one is. + */ +object ArithmeticExpression { + + /** One day, in the milliseconds Elasticsearch stores a `date` in. */ + private val DayMillis = "86400000L" + + /** The same, as a floating-point number, for a fraction of a day. */ + private val FractionalDayMillis = "86400000.0" + + /** What the date-arithmetic rules make of an expression. */ + private[sql] sealed trait DateArithmetic { + + /** The type the rendering produces (`Token.baseType`): a temporal result is rendered as the + * instant it denotes, so it is a TIMESTAMP whether it is reported as a DATE or not -- the same + * collapse `SQLTypeUtils.runtimeType` applies to a DATE column. + */ + def runtimeType: Option[SQLType] = None + + /** The type a consumer declaring a column reports (`Token.reportedType`). */ + def reportedType: Option[SQLType] = runtimeType + } + + /** A value the date-arithmetic rendering computes -- as opposed to a NULL, or a refusal. */ + private[sql] sealed trait Computed extends DateArithmetic + + /** DATE - DATE: the calendar days between the UTC days of the two dates. */ + private[sql] case object DaysBetween extends Computed { + override def runtimeType: Option[SQLType] = Some(SQLTypes.BigInt) + } + + /** A difference with an instant on one side at least: the milliseconds between, in days. */ + private[sql] case object FractionalDaysBetween extends Computed { + override def runtimeType: Option[SQLType] = Some(SQLTypes.Double) + } + + /** A temporal moved by a number of days, the number on either side of a `+`. + * + * @param temporalLeft + * whether the temporal is the LEFT operand + * @param calendarDate + * whether the temporal is a DATE, which is moved from the start of its day + * @param fractional + * whether the number of days has a fractional type, whose fraction becomes a time of day + */ + private[sql] final case class DaysShift( + temporalLeft: Boolean, + calendarDate: Boolean, + fractional: Boolean + ) extends Computed { + override def runtimeType: Option[SQLType] = Some(SQLTypes.Timestamp) + override def reportedType: Option[SQLType] = + Some(if (calendarDate && !fractional) SQLTypes.Date else SQLTypes.Timestamp) + } + + /** A NULL literal beside a temporal: NULL, with the type the expression derives otherwise. */ + private[sql] case object NullOperand extends DateArithmetic + + /** A combination no rule allows. */ + private[sql] final case class Refused(message: String) extends DateArithmetic + + /** One operand, as the rules see it. */ + private sealed trait Operand + private case object CalendarDate extends Operand + private case object Moment extends Operand + private final case class Days(fractional: Boolean) extends Operand + private case object NullLiteral extends Operand + private case object Undecided extends Operand + private case object NonNumber extends Operand + + private def temporal(operand: Operand): Boolean = operand == CalendarDate || operand == Moment + + /** The type an IDENTIFIER operand of an arithmetic computes: an aggregate is the metric it + * produces ([[AggregateFunction.outputTypeOf]]: `COUNT(d)` counts, whatever `d` is), anything + * else is what its chain renders (#382's `chainType`). + * + * πŸ”΄ The ONE derivation the arithmetic asks, for its own type and for the coercion of each + * operand alike (#292): reading the aggregated COLUMN's type typed `COUNT(d) - 1` as a TIMESTAMP + * and emitted `params.count_d.toInstant()` on the count. + */ + private[sql] def identifierType(i: Identifier): SQLType = renderedTypeOf(i) + + /** The type an operand is DECLARED with -- its REPORTED type, an aggregate's being the metric it + * produces: the ONE derivation the comparison rule reads too (`ComparisonRule.declaredTypeOf`). + * + * An Elasticsearch `date` field elasticsql never declared is a TIMESTAMP at the declaration seam + * itself (`schema.IndexField`), so it keeps its time here as everywhere else, and every chain + * over it follows: `MAX(ts)`, `COALESCE(ts, ts2)`, `CASE … THEN ts …`. + */ + private def declaredType(operand: PainlessScript): SQLType = + ComparisonRule.declaredTypeOf(operand) + + /** `NULL`, bare or cast: its VALUE is NULL in every venue. */ + private[sql] def nullLiteral(operand: PainlessScript): Boolean = operand match { + case Null => true + case i: Identifier => i.name.trim.isEmpty && i.functions.lastOption.contains(Null) + case _ => false + } + + /** A BARE `NULL`, which has no type of its own. A cast gives one -- `CAST(NULL AS DATE)` is a + * DATE, `CAST(NULL AS INT)` a whole number -- and a typed NULL is then judged by its type, + * whatever its value. + */ + private def bareNull(operand: PainlessScript): Boolean = operand match { + case Null => true + case i: Identifier => i.name.trim.isEmpty && i.functions == List(Null) + case _ => false + } + + /** A string LITERAL whose text is a number, as the Painless literal of that number and whether it + * is fractional (the lead's ruling of 2026-10-05) -- the ONE reader the comparison rule uses too + * (`ComparisonRule.numberOf`). + * + * PostgreSQL reads an untyped literal by what it is added to: `d + '1'` is the next day and `d - + * '1.5'` a day and a half earlier. A whole number is a `long` literal (`'99999999999'` does not + * fit an `int`), a fractional one the `double` it denotes. + */ + private[math] def numericString(operand: PainlessScript): Option[(String, Boolean)] = + ComparisonRule + .stringLiteral(operand) + .flatMap(ComparisonRule.numberOf) + .map(number => number.painless -> number.fractional) + + /** A string LITERAL read as the temporal it spells beside the temporal `other` -- the comparison + * rule's reading (`ComparisonRule.readingOf`), so `d - '2024-01-31'` is DATE minus DATE, as + * PostgreSQL reads it, and `ts - '2024-01-31T10:00:00Z'` TIMESTAMP minus TIMESTAMP. Beside + * anything else, or spelling no date, the literal is no temporal. + */ + private[math] def temporalString( + operand: PainlessScript, + other: PainlessScript + ): Option[ComparisonRule.TemporalReading] = + ComparisonRule.stringLiteral(operand).flatMap { _ => + operandOf(other) match { + case CalendarDate | Moment => + ComparisonRule.readingOf(operand, declaredType(other)).collect { + case t: ComparisonRule.TemporalReading => t + } + case _ => None + } + } + + private def operandOf(o: PainlessScript): Operand = + if (bareNull(o)) NullLiteral + else + numericString(o) match { + case Some((_, fractional)) => Days(fractional) + case None => + declaredType(o) match { + case SQLTypes.Date => CalendarDate + case SQLTypes.Timestamp | SQLTypes.DateTime | SQLTypes.Temporal => Moment + case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | SQLTypes.BigInt => + Days(fractional = false) + case SQLTypes.Double | SQLTypes.Real | SQLTypes.Numeric => Days(fractional = true) + // a TIME is outside these rules, and an unknown type cannot be judged + case t if t == SQLTypes.Time || t.isUnknown => Undecided + case _ => NonNumber + } + } + + /** One operand beside the other: a string literal that spells a date or a timestamp beside a + * temporal is that temporal ([[temporalString]]); anything else is what it is alone. + */ + private def operandOf(o: PainlessScript, other: PainlessScript): Operand = + temporalString(o, other) match { + case Some(t) if t.sqlType == SQLTypes.Date => CalendarDate + case Some(_) => Moment + case None => operandOf(o) + } + + private[math] def isCalendarDate(o: PainlessScript): Boolean = operandOf(o) == CalendarDate + + private[sql] def isBareName(e: String): Boolean = + e.nonEmpty && e.forall(c => c.isLetterOrDigit || c == '_') + + /** A TEMPORAL operand as the epoch milliseconds it denotes, in the venue `context` renders for -- + * the ONE conversion date arithmetic and `GREATEST` / `LEAST` over dates share (#292): + * + * - per group (`bucket_script`, `bucket_selector`: no context) an aggregate is the metric + * Elasticsearch computed, a number -- in a view's per-group calculation too -- and so is a + * value this module renders from metrics there: a date moved by days, `GREATEST` / `LEAST` + * over dates; + * - per document, a column is the `date` value Elasticsearch hands the script (`toInstant()` + * exists on the `JodaCompatibleZonedDateTime` of 6.8 as on the `ZonedDateTime` of 7 and up), + * or, in an ingest processor, whatever the incoming document holds -- the raw JSON string or + * number of a column, or what a function made of it -- read by its RUNTIME class + * (`SQLTypeUtils.processorInstant`), never by the type it is declared with; + * - anything else is read by its RUNTIME type (`DateDiff.utcInstant`): `CURRENT_DATE` is a + * `LocalDate` in a query and the current instant in a processor. + * + * `ref` is the operand's rendering: a name whenever the operand can be NULL (the caller guards + * it). + */ + private[sql] def epochMillis( + operand: PainlessScript, + ref: String, + context: Option[PainlessContext] + ): String = { + val perGroup = context.isEmpty + val processor = context.exists(_.isProcessor) + operand match { + case metric: Identifier + if metric.isAggregation && (perGroup || context.exists(_.isTransform)) => + ref + case nested: ArithmeticExpression + if nested.dateArithmetic.exists(_.isInstanceOf[DaysShift]) => + if (perGroup) ref else s"$ref.toInstant().toEpochMilli()" + case reducer + if (perGroup || context.exists(_.isTransform)) && NumericReducer.overDates(reducer) => + ref + case column: Identifier if column.name.trim.nonEmpty && column.functions.isEmpty => + if (processor) s"${SQLTypeUtils.processorInstant(ref)}.toInstant().toEpochMilli()" + else s"$ref.toInstant().toEpochMilli()" + case other => + context match { + case None => + s"${SQLTypeUtils.coerce(ref, other.baseType, SQLTypes.Timestamp, nullable = false, None)}.toInstant().toEpochMilli()" + case Some(ctx) => + val name = + if (isBareName(ref)) ref + else ctx.addParam(LiteralParam(ref)).getOrElse(s"((def) ($ref))") + if (processor) s"${SQLTypeUtils.processorInstant(name)}.toInstant().toEpochMilli()" + else s"${DateDiff.utcInstant(name)}.toInstant().toEpochMilli()" + } + } + } + + /** The UTC day a temporal operand falls in, in days since the epoch. `Math.floor` is the one + * floor Painless allows -- `Math.floorDiv` is on no major's allow-list -- and it is exact here: + * a date's milliseconds are far below 2^53, and so is every quotient's distance to an integer. + */ + private[sql] def epochDay( + operand: PainlessScript, + ref: String, + context: Option[PainlessContext] + ): String = + s"Math.floor(${epochMillis(operand, ref, context)} / $FractionalDayMillis)" + + /** The instant a temporal operand denotes, in epoch milliseconds: a DATE is the instant its UTC + * day starts at, so two dates compare -- and move -- by their days. + */ + private[sql] def instantMillis( + operand: PainlessScript, + ref: String, + context: Option[PainlessContext] + ): String = + if (isCalendarDate(operand)) s"${epochDay(operand, ref, context)} * $FractionalDayMillis" + else epochMillis(operand, ref, context) + + /** A temporal operand of a COMPARISON -- a simple `CASE`, `NULLIF` -- as the instant it denotes: + * [[instantMillis]], the conversion date arithmetic and `GREATEST` / `LEAST` share, so a DATE is + * the instant its UTC day starts at and equals a TIMESTAMP exactly at midnight, as SQL engines + * promote it. A string literal the comparison READS as a temporal (`ComparisonRule.readingOf`) + * is its instant, computed here rather than parsed per document. + */ + private[sql] def comparedMillis( + operand: PainlessScript, + reading: Option[ComparisonRule.Reading], + ref: String, + context: Option[PainlessContext] + ): String = + reading match { + case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" + case _ => instantMillis(operand, ref, context) + } + + private[math] def dateArithmetic(e: ArithmeticExpression): Option[DateArithmetic] = { + val l = operandOf(e.left, e.right) + val r = operandOf(e.right, e.left) + if (!temporal(l) && !temporal(r)) None + else + e.operator match { + case MULTIPLY | DIVIDE | MODULO => Some(refused(e)) + case ADD => + (l, r) match { + case (t, Days(fractional)) if temporal(t) => + Some(DaysShift(temporalLeft = true, t == CalendarDate, fractional)) + case (Days(fractional), t) if temporal(t) => + Some(DaysShift(temporalLeft = false, t == CalendarDate, fractional)) + case (NullLiteral, _) | (_, NullLiteral) => Some(NullOperand) + case (Undecided, _) | (_, Undecided) => None + case _ => Some(refused(e)) + } + case SUBTRACT => + (l, r) match { + case (CalendarDate, CalendarDate) => Some(DaysBetween) + case (a, b) if temporal(a) && temporal(b) => Some(FractionalDaysBetween) + case (t, Days(fractional)) if temporal(t) => + Some(DaysShift(temporalLeft = true, t == CalendarDate, fractional)) + case (NullLiteral, _) | (_, NullLiteral) => Some(NullOperand) + case (Undecided, _) | (_, Undecided) => None + case _ => Some(refused(e)) + } + } + } + + /** The type a refusal names an operand by: the one DESCRIBE shows for it. */ + private def typeName(o: PainlessScript): String = + if (bareNull(o)) SQLTypes.Null.typeId else declaredType(o).typeId + + private def refused(e: ArithmeticExpression): Refused = + Refused( + s"Type mismatch: operator ${e.operator.sql} cannot be applied to ${typeName(e.left)} and " + + s"${typeName(e.right)} in expression: ${e.sql.trim}; subtract two dates for the days between " + + "them, or add or subtract a number of days" + ) + + /** Every date-arithmetic type error in an expression, wherever an arithmetic sits in it. + * + * πŸ”΄ Asked of a SCHEMA-RESOLVED expression, as `NullIf.mismatchesOf` is: a computed column's + * resolved table (`schema.validateScriptReferences`) -- a statement asks every type rule at once + * (`SingleSearch.typeErrors`). Before a schema is attached a column's type is unknown, which + * these rules accept. `functionsOf` is the walk every such rule shares; an arithmetic nested + * directly in another is not a chain, so each arithmetic walks its own operands + * ([[ArithmeticExpression.dateArithmeticErrors]]). + */ + def typeErrorsOf(chain: FunctionChain): Seq[String] = + functionsOf(chain) + .collect { case a: ArithmeticExpression => a } + .flatMap(_.dateArithmeticErrors) + .distinct +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index bf3006f36..e2256373a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -491,6 +491,37 @@ package object sql { case object Transform extends PainlessContextType } + /** The Elasticsearch major a Painless script is rendered for. + * + * This module is version-agnostic, and a script renders the same on every major -- except where + * a major hands a script something the others do not. The major then reaches the rendering the + * way issue #222 hands the search-body serializer to the bridge: the CLIENT module of each major + * puts its own `PainlessTarget` in implicit scope of the bridge's conversions, and they give it + * to every [[PainlessContext]] they create. With none in scope ([[PainlessTarget.Default]] -- a + * bridge's own tests, a script rendered outside a client), a script renders as it does for the + * majors from 7 on. + * + * @param elasticsearchMajor + * the major the client module talks to, `None` when no client module gave one + */ + final case class PainlessTarget(elasticsearchMajor: Option[Int]) { + + /** Does this major hand a QUERY script a `date` doc value as a `JodaCompatibleZonedDateTime`? + * Elasticsearch 6.8 does, and a comparison between one and a genuine `ZonedDateTime` fails + * there with `Class.cast` (`query.Expression.painless`); 7 and up hand a `ZonedDateTime`. + */ + def jodaCompatibleDates: Boolean = elasticsearchMajor.exists(_ < 7) + } + + object PainlessTarget { + + /** No client module gave a major: the rendering of the majors from 7 on. */ + val Default: PainlessTarget = PainlessTarget(None) + + /** The target of a client module talking to `elasticsearchMajor`. */ + def apply(elasticsearchMajor: Int): PainlessTarget = PainlessTarget(Some(elasticsearchMajor)) + } + /** Is a rendered Painless fragment a single EXPRESSION, i.e. may it be placed where an operand * goes? [[PainlessOperandForm.placeable]] is the one owner of that rule. * @@ -722,8 +753,13 @@ package object sql { /** Context for painless scripts * @param context * the context type + * @param target + * the Elasticsearch major the script is rendered for ([[PainlessTarget]]) */ - case class PainlessContext(context: PainlessContextType = PainlessContextType.Query) { + case class PainlessContext( + context: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default + ) { // List of parameter keys private[this] var _keys: collection.mutable.Seq[PainlessParam] = collection.mutable.Seq.empty @@ -1213,6 +1249,14 @@ package object sql { override def painless(context: Option[PainlessContext]): String = "null" override def nullable: Boolean = true override def baseType: SQLType = SQLTypes.Null + + /** `NULL` is ONE object, shared by every statement this JVM parses, so a cast must not write + * its type into it: `CAST(NULL AS INT)` -- a conversion's constructor casts its operand -- + * retyped EVERY later `NULL` as an INT, and `NULLIF(s, NULL)` was then refused as "KEYWORD + * with INT" (MEASURED in one test JVM). A typed NULL takes its type from its conversion, which + * already carries it; the bare `NULL` keeps none. + */ + override def cast(targetType: SQLType): SQLType = out } case object ParamValue extends Value[String](null) with TokenRegex { @@ -1334,9 +1378,14 @@ package object sql { for { _ <- from.validate() _ <- to.validate() - _ <- Validator.validateTypesMatching(from.out, to.out) } yield () } + + /** The two bounds must match -- asked once the column types are known, like every TYPE rule + * (the lead's ruling of 2026-10-05), never when the statement is parsed. + */ + override def typeError: Option[String] = + Validator.validateTypesMatching(from.out, to.out).left.toOption } case class LiteralFromTo(override val from: StringValue, override val to: StringValue) @@ -1547,6 +1596,217 @@ package object sql { def update(request: SingleSearch): Source } + /** The ONE comparison rule (the lead's ruling of 2026-10-05): what a comparison compares, and + * whether it may. Every comparison asks it -- `=`, `<>`, `<`, `<=`, `>`, `>=`, `BETWEEN`, `IN`, + * a simple `CASE x WHEN v`, `NULLIF` and a CASE condition, in every clause -- and so does the + * arithmetic that meets a string literal. + * + * - An operand is judged by the type it is DECLARED with ([[ComparisonRule.declaredTypeOf]]): + * what the user wrote, never the runtime collapse of every temporal to TIMESTAMP that + * `renderedTypeOf` answers. A DATE and a TIMESTAMP are comparable + * (`SQLTypeUtils.comparableTemporals`): the DATE is the instant its UTC day starts at. + * - An untyped string LITERAL is read by what it is compared with, as PostgreSQL reads one + * ([[ComparisonRule.readingOf]]): beside a number, the number it spells; beside a temporal, + * the date or the timestamp it spells -- beside a DATE, a date-time literal is the date it + * names, as PostgreSQL's date input reads it; beside a BOOLEAN, the boolean it spells. Any + * other literal stays text, and the comparison is judged as such. + */ + object ComparisonRule { + + /** What a string literal is READ as. */ + sealed trait Reading { + def sqlType: SQLType + } + + /** The number a literal spells, as a Painless literal: a `long` for a whole number that fits + * one, the `double` it denotes otherwise. + */ + final case class NumberReading(painless: String, fractional: Boolean) extends Reading { + def sqlType: SQLType = if (fractional) SQLTypes.Double else SQLTypes.BigInt + } + + /** The date (`sqlType` DATE, `text` as `yyyy-MM-dd`) or the timestamp (`sqlType` TIMESTAMP, + * `text` an ISO date-time) a literal spells, and the instant it denotes in epoch milliseconds, + * UTC unless the literal names a zone; a date is the instant its day starts at. + */ + final case class TemporalReading(sqlType: SQLType, text: String, epochMillis: Long) + extends Reading + + /** The boolean a literal spells. */ + final case class BooleanReading(value: Boolean) extends Reading { + def sqlType: SQLType = SQLTypes.Boolean + } + + /** The text of an untyped STRING literal, bare or as the one function of a nameless identifier + * -- the parser wraps every operand in an identifier. + */ + def stringLiteral(token: Token): Option[String] = token match { + case s: StringValue => Some(s.value) + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (s: StringValue) :: Nil => Some(s.value) + case _ => None + } + case _ => None + } + + /** The type an operand is DECLARED with: an aggregate is the metric it produces + * (`AggregateFunction.outputTypeOf`), anything else the type it reports (`reportedType`: a + * DATE column, a COALESCE of DATE columns, `d + 1` are DATEs). + */ + def declaredTypeOf(operand: Token): SQLType = operand match { + case i: Identifier if i.isAggregation => + AggregateFunction.outputTypeOf(i).getOrElse(i.reportedType) + case p: PainlessScript => p.reportedType + case other => other.out + } + + /** A whole number, as PostgreSQL's integer input reads one. */ + private val WholeNumber = "[+-]?\\d+".r + + /** A decimal number, with an optional exponent; `NaN` and `Infinity` are not numbers here. */ + private val DecimalNumber = "[+-]?(?:\\d+\\.?\\d*|\\.\\d+)(?:[eE][+-]?\\d+)?".r + + /** STRICT: `2026-02-30` is no date (PostgreSQL refuses it), where the default resolver would + * read it as February 28. + */ + private val IsoDate = java.time.format.DateTimeFormatter + .ofPattern("uuuu-MM-dd") + .withResolverStyle(java.time.format.ResolverStyle.STRICT) + + private val DayMillis = 86400000L + + /** The number a text spells, if it spells one. */ + def numberOf(text: String): Option[NumberReading] = + text.trim match { + case s @ WholeNumber() => + Try(BigInt(s)).toOption.map { n => + if (n.isValidLong) NumberReading(s"${n}L", fractional = false) + else NumberReading(n.toDouble.toString, fractional = true) + } + case s @ DecimalNumber() => + Try(BigDecimal(s).toDouble).toOption + .filter(d => !d.isNaN && !d.isInfinite) + .map(d => NumberReading(d.toString, fractional = true)) + case _ => None + } + + /** The date a text spells -- `yyyy-MM-dd`, a `/` accepted for the `-` -- if it spells one. */ + private def dateOf(text: String): Option[java.time.LocalDate] = + Try(java.time.LocalDate.parse(text.trim.replace("/", "-"), IsoDate)).toOption + + /** The date-time an ISO text spells (a space accepted for the `T`), in UTC unless it names a + * zone, if it spells one. + */ + private def dateTimeOf(text: String): Option[java.time.ZonedDateTime] = + Try( + java.time.ZonedDateTime.parse( + text.trim.replace(" ", "T"), + java.time.format.DateTimeFormatter.ISO_DATE_TIME.withZone(java.time.ZoneOffset.UTC) + ) + ).toOption + + private def dateReading(date: java.time.LocalDate): TemporalReading = + TemporalReading(SQLTypes.Date, date.toString, date.toEpochDay * DayMillis) + + /** The temporal a text spells beside an operand of type `against`, read AS that operand's type, + * as PostgreSQL reads an untyped literal: + * - beside a DATE, the date it names -- a date-time text's date, as PostgreSQL's date input + * reads `'2024-01-31T10:00:00Z'` as 2024-01-31; + * - beside any other temporal, the TIMESTAMP it denotes -- a date-only text is the instant + * its UTC day starts at, so `ts = '2024-01-31'` holds at midnight only. Its `text` is then + * always a full ISO date-time, which is what the `CAST(… AS TIMESTAMP)` conversion parses. + */ + def temporalOf(text: String, against: SQLType): Option[TemporalReading] = { + val date = dateOf(text) + if (against == SQLTypes.Date) + date.orElse(dateTimeOf(text).map(_.toLocalDate)).map(dateReading) + else + date + .map { d => + TemporalReading(SQLTypes.Timestamp, s"${d}T00:00:00Z", d.toEpochDay * DayMillis) + } + .orElse { + dateTimeOf(text).map { zoned => + TemporalReading( + SQLTypes.Timestamp, + text.trim.replace(" ", "T"), + zoned.toInstant.toEpochMilli + ) + } + } + } + + /** The boolean a text spells, as PostgreSQL's boolean input reads one. */ + private def booleanOf(text: String): Option[BooleanReading] = + text.trim.toLowerCase match { + case "t" | "true" | "y" | "yes" | "on" | "1" => Some(BooleanReading(value = true)) + case "f" | "false" | "n" | "no" | "off" | "0" => Some(BooleanReading(value = false)) + case _ => None + } + + /** What `literal` is read as beside an operand DECLARED `against`, if it is an untyped string + * literal that spells a value of that kind; `None` otherwise -- it stays text. An operand of + * UNKNOWN type (no schema attached) reads no literal: nothing decides what it should be. + */ + def readingOf(literal: Token, against: => SQLType): Option[Reading] = + stringLiteral(literal).flatMap { text => + against match { + case _: SQLNumeric => numberOf(text) + case t: SQLTemporal if t != SQLTypes.Time && t != SQLTypes.Temporal => + temporalOf(text, t) + case SQLTypes.Temporal => temporalOf(text, SQLTypes.Timestamp) + case _: SQLBool => booleanOf(text) + case _ => None + } + } + + /** The type `operand` is compared AS beside `other`: what a literal reads as, the declared type + * of anything else. + */ + def comparedType(operand: Token, other: Token): SQLType = + readingOf(operand, declaredTypeOf(other)) + .map(_.sqlType) + .getOrElse(declaredTypeOf(operand)) + + /** May two compared types be compared? */ + def comparable(left: SQLType, right: SQLType): Boolean = + SQLTypeUtils.matches(left, right) || SQLTypeUtils.comparableTemporals(left, right) + + /** The two compared types, when they cannot be compared. */ + def mismatch(left: Token, right: Token): Option[(SQLType, SQLType)] = { + val l = comparedType(left, right) + val r = comparedType(right, left) + if (comparable(l, r)) None else Some((l, r)) + } + + /** The Painless a NUMBER or a BOOLEAN reading renders: the literal itself. */ + def scalarPainless(reading: Reading): Option[String] = reading match { + case NumberReading(painless, _) => Some(painless) + case BooleanReading(value) => Some(value.toString) + case _ => None + } + } + + /** The type a TYPE RULE reads for an operand once the column types are known: the one the operand + * RENDERS, never the one a resolved column reports through `out`. + * + * πŸ”΄ ONE derivation for every rule the lead's ruling of 2026-10-05 moved to the post-resolution + * seam, and for the arithmetic that types itself from its operands (#292). After the schema is + * attached `out` answers the COLUMN's type whatever function the operand wears -- `CAST(n AS + * INT)` over a KEYWORD column says KEYWORD, `COUNT(d)` says TIMESTAMP -- so a rule reading it + * would refuse `CAST(s AS INT) IN (1, 2)` and `HAVING COUNT(d) > 1`. So: + * - an AGGREGATE is the metric it produces (`AggregateFunction.outputTypeOf`); + * - any other identifier is what its chain renders (#382's `chainType`); + * - anything else -- a literal, a function, a nested arithmetic -- its own `out`. + */ + def renderedTypeOf(operand: Token): SQLType = operand match { + case i: Identifier if i.isAggregation => + AggregateFunction.outputTypeOf(i).getOrElse(i.chainType) + case i: Identifier => i.chainType + case other => other.out + } + sealed trait Identifier extends TokenValue with Source @@ -2236,7 +2496,8 @@ package object sql { * `runtimeType(declaration)`. A bare `DATE` column is reported as `DATE`, which is what the * user wrote and what DuckDB answers for the same column. */ - override def reportedLeafType: SQLType = col.map(_.dataType).getOrElse(super.reportedLeafType) + override def reportedLeafType: SQLType = + col.map(_.dataType).getOrElse(super.reportedLeafType) def update(request: SingleSearch): Identifier = query.Having.aliasedAggregate(this) match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala index 1ea0e14cc..c0042f9ce 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala @@ -17,12 +17,12 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} -import app.softnetwork.elastic.sql.function.{Function, FunctionN} +import app.softnetwork.elastic.sql.function.Function object Validator { def validateChain(functions: List[Function]): Either[String, Unit] = { - // validate function chain type compatibility + // every function of the chain, for its own syntax and structure functions match { case Nil => return Right(()) case _ => @@ -31,12 +31,10 @@ object Validator { case Nil => // ok case errors => return Left(errors.map { case Left(err) => err }.mkString("\n")) } - val funcs = functions.collect { case f: FunctionN[_, _] => f } - funcs.sliding(2).foreach { - case Seq(f1, f2) => - validateTypesMatching(f2.outputType, f1.inputType) - case _ => // ok - } + // ⚠️ No chain-link TYPE check here. One used to sit at this point and its result was + // DISCARDED (`foreach` over `validateTypesMatching`), so it never refused anything; it is not + // moved to the post-resolution seam either, because it never applied (the lead's ruling of + // 2026-10-05 moves the type checks that refuse, and adds none). Right(()) } @@ -50,7 +48,22 @@ object Validator { } trait Validation { + + /** The rules of SYNTAX and STRUCTURE, checked when the statement is parsed. */ def validate(): Either[String, Unit] = Right(()) + + /** This node's TYPE rule alone, if it breaks it: asked once the column types are known -- after + * the schema is attached (`SingleSearch.validateResolved`) -- and never at parse time. + * + * πŸ”΄ The lead's ruling of 2026-10-05: no refusal before the column types are known. At parse + * time every column is `Any`, so a type rule asked there either accepts what it cannot judge or + * refuses on a type it guessed: `d + 7 > CURRENT_DATE` was refused as BIGINT against DATE + * because `d + 7` folded `[ANY, BIGINT]` to BIGINT. Every TYPE refusal waits for the types, a + * refusal certain whatever the column's type (`x * CURRENT_DATE`) included; parsing checks + * syntax and structure only. With no schema at all, nothing is refused on types and + * Elasticsearch answers. + */ + def typeError: Option[String] = None } case class ValidationError(message: String) extends Exception(message) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index c548263e2..cd0a2a218 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -36,8 +36,8 @@ import app.softnetwork.elastic.sql.function.cond.{ } import app.softnetwork.elastic.sql.function.convert.Conversion import app.softnetwork.elastic.sql.function.geo.Distance -import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql._ import scala.annotation.tailrec @@ -85,6 +85,33 @@ sealed trait Criteria extends Updateable with PainlessScript { // operand of an equality, where the arm above sees only the equality itself. Found by review. }) ++ referencedIdentifiers.flatMap(Case.conditionsOf).flatMap(_.temporalComparisonErrors) + /** Every TYPE refusal of this criteria tree once the column types are known: each comparison's + * own rules ([[temporalComparisonErrors]]' TIME rule, then `Validation.typeError`), and every + * type rule of the functions its operands apply (`chainTypeErrors`), a CASE nested in an operand + * included. + * + * πŸ”΄ Asked of a SCHEMA-RESOLVED statement only (`SingleSearch.validateResolved`): the lead's + * ruling of 2026-10-05 -- no refusal before the column types are known. The same walk as + * [[temporalComparisonErrors]], so a criteria type that one reaches, this one reaches too. + * + * `scripted`: whether every comparison of this tree runs as a SCRIPT -- a CASE condition does, + * wherever the CASE sits. A `WHERE` / `HAVING` clause's own comparison of a BARE column with a + * literal runs as a query on the column's mapping instead, which Elasticsearch judges itself + * (`Expression.comparisonError`). + */ + def typeErrors: Seq[String] = typeErrors(scripted = false) + + def typeErrors(scripted: Boolean): Seq[String] = ((this match { + case Predicate(left, _, right, _, _) => left.typeErrors(scripted) ++ right.typeErrors(scripted) + case e: Expression => + e.temporalComparisonError.toSeq ++ e.comparisonError(scripted).toSeq + case relation: ElasticRelation => relation.criteria.typeErrors(scripted) + case _ => Nil + }) ++ referencedIdentifiers.flatMap(chainTypeErrors) ++ + referencedIdentifiers + .flatMap(Case.conditionsOf) + .flatMap(_.typeErrors(scripted = true))).distinct + def dependencies: Seq[Identifier] = this match { case Predicate(left, _, right, _, _) => left.dependencies ++ right.dependencies case c: Expression => c.dependencies @@ -780,7 +807,34 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { def painlessOp: String = effectiveOperator.painless(None) def painlessValue(context: Option[PainlessContext]): String = - maybeValue.map(painlessOperand(_, context)).getOrElse("") + maybeValue match { + case Some(value) => readingPainless(literalReading, value, context) + case None => "" + } + + /** One operand as it is compared: a string literal by what the comparison reads it as + * ([[literalReading]] -- the number, the boolean, or the temporal it spells, the last converted + * exactly as `CAST('…' AS DATE)` / `CAST('…' AS TIMESTAMP)` converts it), anything else as it + * renders. + */ + protected def readingPainless( + reading: Option[ComparisonRule.Reading], + token: Token, + context: Option[PainlessContext] + ): String = + reading match { + case Some(t: ComparisonRule.TemporalReading) => + SQLTypeUtils.coerce( + s""""${t.text}"""", + SQLTypes.Varchar, + t.sqlType, + nullable = false, + context + ) + case Some(scalar) => + ComparisonRule.scalarPainless(scalar).getOrElse(painlessOperand(token, context)) + case None => painlessOperand(token, context) + } /** One operand's Painless rendering. A `Token` that is not a [[PainlessScript]] has only its SQL * spelling to offer -- which is why `BetweenExpr` must go through this and not `toString`: a @@ -909,11 +963,47 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { * right-hand type: this predicate's target type, and `check`'s per-type dispatch. */ protected def valueType: SQLType = - maybeValue match { - case Some(id: Identifier) => id.chainType - case Some(v: PainlessScript) => v.out - case Some(other) => other.out - case None => SQLTypes.Any + literalReading + .map(_.sqlType) + .getOrElse(maybeValue match { + case Some(id: Identifier) => id.chainType + case Some(v: PainlessScript) => v.out + case Some(other) => other.out + case None => SQLTypes.Any + }) + + /** What a STRING literal on the right is READ as -- the ONE comparison rule + * (`ComparisonRule.readingOf`), against the type the left operand is DECLARED with: `COALESCE(n, + * 0) = '3'` compares a number with the number 3, `d + 1 > '2024-01-01'` a date with a date, as + * SQL engines read an untyped literal by what it is compared with. + * + * A temporal is rendered by the very ` -> ` conversion a `CAST` uses + * ([[painlessValue]]) and reconciled with the left operand like any typed temporal; a number or + * a boolean renders as one. + * + * It decides only what a SCRIPT renders: a bare column compared with a literal in a `WHERE` runs + * as a query on the column's mapping, which never renders this comparison. + * + * A comparison that reads an AGGREGATE is a group filter, and a group filter reads every metric + * as a NUMBER -- a date metric as epoch milliseconds -- so while the metric's type is still + * unknown (before the schema is attached, or when none ever is) a literal is read there as the + * number or the instant it spells: nothing else can be compared with a metric. `HAVING MAX(n) > + * '4'` therefore renders the same comparison before and after resolution; the type rule judges + * it once the type is known. + */ + protected def literalReading: Option[ComparisonRule.Reading] = maybeValue.flatMap(readingFor) + + /** What `value` -- the right-hand side, a `BETWEEN` bound, an `IN` element -- is read as against + * this comparison's left operand ([[literalReading]]). + */ + protected def readingFor(value: Token): Option[ComparisonRule.Reading] = + ComparisonRule.stringLiteral(value).flatMap { text => + val declared = ComparisonRule.declaredTypeOf(identifier) + ComparisonRule.readingOf(value, declared).orElse { + if (identifier.hasAggregation && declared.isUnknown) + ComparisonRule.numberOf(text).orElse(ComparisonRule.temporalOf(text, SQLTypes.Timestamp)) + else None + } } /** The JAVA type the RIGHT-hand side renders β€” what `Identifier.renderedType` answers for the @@ -1279,6 +1369,13 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { value: String, op: Operator ): String = { + // A bucket pipeline reads every metric as a NUMBER -- a date aggregate, and the alias of a + // per-group date calculation (`MAX(d) + 1 AS x ... HAVING x > ...`), arrive as epoch + // milliseconds -- and [[bucketPipelineCheck]] converts a temporal value to the same unit, so the + // two are compared as numbers. The `java.time` methods below would be called on a `Double` + // (`dynamic method [java.lang.Double, isAfter/1] not found`). + val readsMetric = + isAggregation || hasBucket || (context.isEmpty && (referencesBucketMetric || reducesDateMetrics)) op match { case _: ComparisonOperator => // πŸ”΄ Story BIDC-8 AD-9 β€” dispatch on the EFFECTIVE operator, i.e. with a `NOT` folded in, @@ -1300,7 +1397,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar => return s"$param.compareTo($value) < 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isBefore($value)" case _ => } @@ -1308,7 +1405,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar => return s"$param.compareTo($value) > 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isAfter($value)" case _ => } @@ -1331,7 +1428,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // spellings are otherwise identical for a `LocalTime`, nanoseconds included. case SQLTypes.Varchar | SQLTypes.Time => return s"$param.compareTo($value) == 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isEqual($value)" case _ => } @@ -1339,7 +1436,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar | SQLTypes.Time => return s"$param.compareTo($value) != 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isEqual($value) == false" case _ => } @@ -1347,7 +1444,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar => return s"$param.compareTo($value) >= 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isBefore($value) == false" case _ => } @@ -1355,7 +1452,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar => return s"$param.compareTo($value) <= 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isAfter($value) == false" case _ => } @@ -1546,19 +1643,45 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { /** The comparison body of the bucket-pipeline rendering, `param` (= `params.`) against * the right-hand side. BETWEEN and IN, whose `painless` never goes through `check`, override it. */ - protected def bucketPipelineCheck(param: String): String = { + protected def bucketPipelineCheck(param: String): String = + check(None, param, bucketPipelineValue) + + /** The right-hand side as a group filter compares it with a metric: a NUMBER, epoch milliseconds + * for a temporal -- the unit a date metric arrives in. + */ + protected def bucketPipelineValue: String = literalReading match { + // a string literal read as a date or a timestamp ([[literalReading]]): its instant, computed + // once, here, rather than parsed per bucket + case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" + case _ => bucketPipelineOperand + } + + private def bucketPipelineOperand: String = { val rhs = painlessValue(None) - val value = maybeValue match { - case Some(v) if operator.isInstanceOf[ComparisonOperator] && !v.isAggregation => + maybeValue match { + // A date-arithmetic value already renders epoch milliseconds here, the number a per-group + // date is (`HAVING MAX(d) > CAST('2024-03-01' AS DATE) - 30`): it is compared as it stands. + case Some(v) + if operator.isInstanceOf[ComparisonOperator] && !v.isAggregation && + !Expression.carriesArithmetic(v) => v.out match { - case SQLTypes.Date => s"$rhs.truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli()" + // πŸ”΄ A DATE value is read through `getLong(ChronoField.EPOCH_DAY)`, which every + // `java.time` value with a date answers, whatever it renders: a `LocalDate` + // (`CAST('2024-01-01' AS DATE)`), which has neither `truncatedTo` nor `toInstant` -- + // `HAVING MAX(d) > CAST('2024-01-01' AS DATE)` failed to compile in Elasticsearch + // (`member method [java.time.LocalDate, truncatedTo/1] not found`, MEASURED on 8.18.3) + // -- or a UTC `ZonedDateTime` (date arithmetic, the clock). No `instanceof` and no NULL + // arm: the group-filter gate refuses both, and the value is a literal or the clock, + // never NULL. A TIMESTAMP value (a `ZonedDateTime`) was compared with the metric's + // number as it stood; it is converted like a DATETIME. + case SQLTypes.Date => s"(($rhs).getLong(ChronoField.EPOCH_DAY) * 86400000L)" case SQLTypes.Time => s"$rhs.truncatedTo(ChronoUnit.SECONDS).toInstant().toEpochMilli()" - case SQLTypes.DateTime => s"$rhs.toInstant().toEpochMilli()" - case _ => rhs + case SQLTypes.DateTime => s"$rhs.toInstant().toEpochMilli()" + case SQLTypes.Timestamp => s"($rhs).toInstant().toEpochMilli()" + case _ => rhs } case _ => rhs } - check(None, param, value) } /** True when this predicate reads a bucket-pipeline metric: an aggregate, or the alias of a @@ -1570,9 +1693,20 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { def referencesBucketMetric: Boolean = identifier.isAggregation || (identifier.hasAggregation && identifier.fieldAlias.isDefined) + /** `GREATEST` / `LEAST` over date AGGREGATES, inline in a group filter (`HAVING GREATEST(MAX(d), + * MAX(d2)) > CAST('2024-02-01' AS DATE)`): per group they render the epoch milliseconds of the + * date that wins (`NumericReducer`), the number a date aggregate is there, so they are compared + * as one -- the value converted to the same unit ([[bucketPipelineCheck]]). Compared as a + * `java.time` value, the group filter called `isAfter` on a `Double`. + */ + private def reducesDateMetrics: Boolean = + NumericReducer.overDates(identifier) && identifier.hasAggregation + override def painless(context: Option[PainlessContext]): String = { // A context-free rendering of an aggregate predicate is a bucket-pipeline rendering. if (context.isEmpty && referencesBucketMetric) return bucketPipelinePainless + if (context.isEmpty && reducesDateMetrics) + return s"$painlessNot(${bucketPipelineCheck(identifier.painless(None))})" val (chainRendering, innerLeft) = leftOperand(context) // The right-hand side is rendered ONCE: `painlessValue` can register a parameter on the context, @@ -1671,6 +1805,38 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { val leftNullable = readsDocumentField(Some(identifier)) val rightNullable = readsDocumentField(maybeValue) + /** πŸ”΄ Elasticsearch 6.8 hands a QUERY script a `date` doc value as a + * `JodaCompatibleZonedDateTime`, and its `isBefore` / `isAfter` / `isEqual` against a genuine + * `ZonedDateTime` -- which every function, every literal and date arithmetic produce -- fail + * the shard with `Class.cast` (MEASURED on 6.8.23, in both directions: `d > d2 - 1` and + * `CURRENT_DATE > d`). So each operand that READS a document and is compared as an instant is + * normalised to the `ZonedDateTime` it denotes, in UTC -- the conversion `DateDiff.utcInstant` + * already relies on (`withZoneSameInstant`), a no-op on a `ZonedDateTime` already in UTC. + * + * Only where the comparison is between INSTANTS (`comparisonTargetType` TIMESTAMP): an operand + * narrowed to a `LocalDate` or a `LocalTime` has no zone, and normalising one is what broke + * five comparisons in the first attempt at this. Only a QUERY script: an ingest processor + * parses its operands into `ZonedDateTime`s itself, and a view's transform reads its own + * index. The operand is a name or a call chain here, so the conversion is appended to it; it + * is never NULL there -- every route below tests it against NULL first. + * + * Only for Elasticsearch 6.8 ([[PainlessTarget.jodaCompatibleDates]], the major the client + * module gives the bridge): 7 and up hand a genuine `ZonedDateTime`, and there the conversion + * cost 2 to 4% of a per-document filter for nothing (MEASURED on 8.18.3), so the scripts of + * those majors stay what they were. + */ + val utcComparison: Boolean = + context.exists(c => !c.isProcessor && !c.isTransform && c.target.jodaCompatibleDates) && + valueType.isInstanceOf[SQLTemporal] && !isAggregation && !hasBucket && + comparisonTargetType == SQLTypes.Timestamp + def instantOperand(rendered: SQLType, readsDocument: Boolean): Boolean = + utcComparison && readsDocument && + (rendered == SQLTypes.Timestamp || rendered == SQLTypes.Date) + val leftUtc = instantOperand(operandRenderedType, leftNullable) + val rightUtc = instantOperand(valueRenderedType, rightNullable) + def utc(operand: String, normalise: Boolean): String = + if (normalise) s"$operand.withZoneSameInstant(ZoneId.of('Z'))" else operand + context match { case Some(ctx) => ctx.get(identifier) match { @@ -1705,9 +1871,9 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // `ABS(amount)` already used, and the reason those were never affected. case Some(p) if !rightNullable && p == chainRendering => if (identifier.nullable) - return s"$p == null ? false : $painlessNot(${check(context, p, rightPromoted)})" + return s"$p == null ? false : $painlessNot(${check(context, utc(p, leftUtc), utc(rightPromoted, rightUtc))})" else - return s"$painlessNot(${check(context, p, rightPromoted)})" + return s"$painlessNot(${check(context, utc(p, leftUtc), utc(rightPromoted, rightUtc))})" case _ => } case _ => @@ -1771,7 +1937,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { if (rightNullable) Some(s"$rightRef == null") else None ).flatten if (guards.nonEmpty && context.nonEmpty) - return s"(${guards.mkString(" || ")} ? false : ($painlessNot(${check(context, leftRef, promotedRight(rightRef, context))})))" + return s"(${guards.mkString(" || ")} ? false : ($painlessNot(${check(context, utc(leftRef, leftUtc), utc(promotedRight(rightRef, context), rightUtc))})))" // πŸ”΄ INVARIANT (story BIDC-8, review L-11): a statement sequence is legal ONLY with no // `PainlessContext`. `Criteria.painless` is spliced into larger expressions -- a `Predicate` // joins two renderings with `&&`/`||`, and a `CASE` condition captures one as @@ -1780,43 +1946,88 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // route is the HAVING / bucket-pipeline one, which is never spliced. Gated by // `PainlessOperandFormSpec`, which FAILS (14 shapes) if `bindLocal` stops hoisting. if (identifier.nullable) - return s"def left = $leftRef; left == null ? false : $painlessNot(${check(context, "left", rightPromoted)})" - s"$painlessNot${check(context, leftRef, rightPromoted)}" + return s"def left = $leftRef; left == null ? false : $painlessNot(${check(context, utc("left", leftUtc), utc(rightPromoted, rightUtc))})" + s"$painlessNot${check(context, utc(leftRef, leftUtc), utc(rightPromoted, rightUtc))}" } - override def validate(): Either[String, Unit] = { + /** STRUCTURE only: the two operands. The TYPE rule is [[typeError]], asked once the column types + * are known (the lead's ruling of 2026-10-05). + */ + override def validate(): Either[String, Unit] = for { _ <- identifier.validate() _ <- maybeValue match { - case Some(v) => - v.validate() match { - case Left(err) => Left(s"$err in expression: $this") - case Right(_) => - // πŸ”΄ Issue #384, item 4 β€” the types COMPARED are the ones the two operands END UP - // with, not the ones their COLUMNS have. `identifier.out` is the column's type, so - // this rule was wrong in BOTH directions: it accepted `CAST(ts AS TIME) > ts` - // (TIMESTAMP against TIMESTAMP, the cast invisible) and rejected - // `CAST(ts AS TIME) = CAST('07:00:00' AS TIME)` (TIMESTAMP against TIME, two - // `LocalTime`s that compare perfectly well). It is the same `out`-vs-`chainType` - // lie issue #367 fixed for EMISSION, still live in VALIDATION -- and `chainType` - // and `valueType` are the two derivations that already answer it for each side. - Validator.validateTypesMatching(identifier.chainType, valueType) match { - // πŸ”΄ ...and a comparison additionally accepts any two DATE-CARRYING temporals, - // because it is the one caller that RECONCILES them (issue #384, item 2). See - // `SQLTypeUtils.comparableTemporals` for why that allowance is NOT in `matches`. - case Left(_) if SQLTypeUtils.comparableTemporals(identifier.chainType, valueType) => - Right(()) - case Left(_) => - Left( - s"Type mismatch: '${identifier.chainType.typeId}' is not compatible with '${valueType.typeId}' in expression: $this" - ) - case Right(_) => Right(()) - } - } - case _ => Right(()) + case Some(v) => v.validate().left.map(err => s"$err in expression: $this") + case _ => Right(()) } } yield () + + /** The comparison's TYPE rule, asked once the column types are known, as a `WHERE` / `HAVING` + * clause asks it ([[comparisonError]]). + */ + override def typeError: Option[String] = comparisonError(scripted = false) + + /** The comparison's TYPE rule -- the ONE comparison rule (`ComparisonRule`, the lead's ruling of + * 2026-10-05) -- asked once the column types are known. + * + * πŸ”΄ It used to run when the statement was PARSED, where every column is `Any`: it refused `d + + * 7 > CURRENT_DATE` as BIGINT against DATE before `d` had a type, and could not see `s = n` at + * all. Asked after resolution it reads the types the two operands are DECLARED with, and + * accepts: + * - two matching types, and a DATE against a TIMESTAMP, judged on their declarations, never on + * the runtime collapse of every temporal to TIMESTAMP (`SQLTypeUtils.comparableTemporals`; a + * comparison RECONCILES them, issue #384); + * - a STRING literal read by what it is compared with: `COALESCE(n, 0) = '3'` is a number + * against 3, `d + 1 > '2024-01-01'` a date against a date (`ComparisonRule.readingOf`); + * - in a `WHERE` or `HAVING` clause's own comparison (`scripted = false`), any LITERAL against + * a BARE column: that comparison is a query Elasticsearch runs against the column's mapping, + * which coerces the literal (`n = '5'`, `d >= '2024-01-01'`) or refuses it itself, as it + * always has. A CASE condition is a script, so the rule judges it like any other comparison. + */ + def comparisonError(scripted: Boolean): Option[String] = + maybeValue.flatMap { value => + if (!scripted && Expression.literalAgainstBareColumn(identifier, value)) None + else + ComparisonRule.mismatch(identifier, value).map { case (left, right) => + s"Type mismatch: '${left.typeId}' is not compatible with '${right.typeId}' in expression: $this" + } + } +} + +object Expression { + + /** Does a value carry arithmetic -- itself, or as a function of its chain? In a bucket pipeline, + * date arithmetic renders the epoch milliseconds a per-group date is. + */ + private[query] def carriesArithmetic(value: Token): Boolean = value match { + case _: ArithmeticExpression => true + case i: Identifier => i.functions.exists(_.isInstanceOf[ArithmeticExpression]) + case _ => false } + + /** A column named with no function applied to it. */ + private[query] def bareColumn(token: Token): Boolean = token match { + case i: Identifier => i.name.trim.nonEmpty && i.functions.isEmpty + case _ => false + } + + /** A literal value, bare or as the one function of a nameless identifier. */ + private[query] def literal(token: Token): Boolean = token match { + case _: Value[_] => true + case i: Identifier => + i.name.trim.isEmpty && (i.functions match { + case (_: Value[_]) :: Nil => true + case _ => false + }) + case _ => false + } + + /** A LITERAL against a BARE column, on either side: the comparison runs as a query on the + * column's own mapping, and Elasticsearch coerces the literal or refuses it -- core never + * refused it, column types or not. + */ + private[query] def literalAgainstBareColumn(left: Token, right: Token): Boolean = + (bareColumn(left) && literal(right)) || (literal(left) && bareColumn(right)) } case class GenericExpression( @@ -2082,20 +2293,46 @@ case class InExpr[R, +T <: Value[R]]( for { _ <- identifier.validate() _ <- values.validate() - _ <- { - val elementType = values.out match { - case a: SQLArray => a.elementType - case other => other - } - Validator - .validateTypesMatching(identifier.out, elementType) - .left - .map(_ => - s"Type mismatch: '${identifier.out.typeId}' is not compatible with '${elementType.typeId}' in expression: $this" - ) - } } yield () + /** Each element against the operand, by the ONE comparison rule (`ComparisonRule`), asked once + * the column types are known (the lead's ruling of 2026-10-05): `x IN (a, b)` is `x = a OR x = + * b`, so `COALESCE(n, 0) IN ('3', '10')` compares a number with two numbers, and `COALESCE(d, + * d2) IN ('2024-01-31', '2024-02-01')` a date with two dates. Judged on the operand's DECLARED + * type: `CAST(s AS INT) IN (1, 2)` over a KEYWORD `s` compares numbers. + * + * In a `WHERE` / `HAVING` clause's own IN over a BARE column (`scripted = false`), a `terms` + * query on the column's own mapping, which coerces the list or refuses it itself, as it always + * has. + */ + override def comparisonError(scripted: Boolean): Option[String] = + if (!scripted && Expression.bareColumn(identifier)) None + else + values.values + .flatMap(element => ComparisonRule.mismatch(identifier, element)) + .headOption + .map { case (left, right) => + s"Type mismatch: '${left.typeId}' is not compatible with '${right.typeId}' in expression: $this" + } + + /** What every element is READ as (`readingFor`), when each one is a string literal the operand + * reads -- the list is then the numbers, the dates, the timestamps or the booleans they spell. + */ + private def elementReadings: Option[Seq[ComparisonRule.Reading]] = + if (values.values.isEmpty) None + else { + val readings = values.values.map(readingFor) + if (readings.forall(_.isDefined)) Some(readings.flatten) else None + } + + /** The type the list is compared IN when its elements are read ([[elementReadings]]), as the + * right-hand side of a single comparison is; the list's own type otherwise, as it always was. + */ + override protected def valueType: SQLType = + elementReadings + .map(rs => SQLTypeUtils.leastCommonSuperType(rs.map(_.sqlType).toList)) + .getOrElse(super.valueType) + /** `[v1, v2].contains()` -- the LIST owns `contains`, not the element. * * πŸ”΄ Story BIDC-8 (review L-8). The old override rendered @@ -2124,9 +2361,36 @@ case class InExpr[R, +T <: Value[R]]( // // A non-numeric list keeps `contains`: for strings `equals` is what IN means, and the rendering // is one term rather than N. - if (numericList) - values.values.map(v => s"$param == ${v.painless(context)}").mkString("(", " || ", ")") - else s"$value.contains($param)" + // + // A list of string literals the operand READS (`elementReadings`) is compared element by + // element, as the single comparison `x = ` compares: numbers and booleans with `==`, + // temporals through the shared per-type dispatch (`isEqual`), each element brought to the + // operand's java.time type first. `contains` asked a `ZonedDateTime` whether it EQUALS the + // string "2024-01-31" -- never, so `COALESCE(d, d2) IN ('2024-01-31', …)` matched no row. + elementReadings match { + case Some(readings) => + values.values + .zip(readings) + .map { + case (literal, t: ComparisonRule.TemporalReading) => + val rendered = readingPainless(Some(t), literal, context) + val target = comparisonTargetType + val element = + if ( + t.sqlType != target && target.isInstanceOf[SQLTemporal] && + !context.exists(_.isProcessor) + ) + SQLTypeUtils.coerce(rendered, t.sqlType, target, nullable = false, context) + else rendered + super.check(context, param, element, EQ) + case (element, reading) => + s"$param == ${readingPainless(Some(reading), element, context)}" + } + .mkString("(", " || ", ")") + case None if numericList => + values.values.map(v => s"$param == ${v.painless(context)}").mkString("(", " || ", ")") + case None => s"$value.contains($param)" + } /** True when every element of the list is a number β€” the case `List.contains` gets wrong. */ private def numericList: Boolean = @@ -2138,9 +2402,20 @@ case class InExpr[R, +T <: Value[R]]( // measured live, `MAX(age) IN (40, 50)` returned no bucket. Painless `==` promotes numerics and // uses `equals` for strings. The NOT is NOT rendered here: `IN` has no negated operator, so // `painlessNot` renders the `!` for it (review M-6) and `bucketPipelinePainless` already wraps - // this call in it -- negating twice was a double negation. + // this call in it -- negating twice was a double negation. A string element is the number, or + // the epoch milliseconds, it is read as -- a metric is a number (`readingFor`). override protected def bucketPipelineCheck(param: String): String = - values.values.map(v => s"$param == ${v.painless(None)}").mkString(" || ") + values.values + .map { v => + val element = readingFor(v) match { + case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" + case Some(scalar) => + ComparisonRule.scalarPainless(scalar).getOrElse(v.painless(None)) + case None => v.painless(None) + } + s"$param == $element" + } + .mkString(" || ") } @@ -2524,10 +2799,60 @@ case class BetweenExpr( for { _ <- identifier.validate() _ <- fromTo.validate() - _ <- Validator.validateTypesMatching(identifier.out, fromTo.out) } yield () } + /** Each bound against the operand, by the ONE comparison rule (`ComparisonRule`), asked once the + * column types are known (the lead's ruling of 2026-10-05): `x BETWEEN a AND b` is `x >= a AND x + * <= b`, so a bound is compared with the operand, never with the other bound -- `COALESCE(d, d2) + * BETWEEN '2024-01-01' AND '2024-02-01'` compares a date with two dates, and `d BETWEEN CAST(… + * AS DATE) AND CAST(… AS TIMESTAMP)` a date with a date and a timestamp. + * + * In a `WHERE` / `HAVING` clause's own BETWEEN over a BARE column (`scripted = false`), a + * `range` query on the column's own mapping, which coerces the bounds or refuses them itself, as + * it always has: only the bounds' own match is asked there. + */ + override def comparisonError(scripted: Boolean): Option[String] = + if (!scripted && Expression.bareColumn(identifier)) fromTo.typeError + else + Seq(fromTo.from, fromTo.to) + .flatMap(bound => ComparisonRule.mismatch(identifier, bound)) + .headOption + .map { case (left, right) => + s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" + } + + /** The SQL type of a bound as it is compared: its reading, or its own type -- the question + * `valueType` answers for the right-hand side of a single comparison. + */ + private def boundType(bound: Token): SQLType = + readingFor(bound) + .map(_.sqlType) + .getOrElse(bound match { + case id: Identifier => id.chainType + case other => other.out + }) + + /** The JAVA type a bound renders -- the question `valueRenderedType` answers for a single one. */ + private def boundRenderedType(bound: Token): SQLType = + readingFor(bound) + .map(_.sqlType) + .getOrElse(bound match { + case id: Identifier => id.renderedType + case other => other.out + }) + + /** The type the bounds are compared IN -- what the shared dispatch ([[check]]) and the operand's + * target type ([[comparisonTargetType]]) read for the right-hand side of a single comparison. + */ + override protected def valueType: SQLType = + SQLTypeUtils.leastCommonSuperType(List(boundType(fromTo.from), boundType(fromTo.to))) + + override protected def valueRenderedType: SQLType = + SQLTypeUtils.leastCommonSuperType( + List(boundRenderedType(fromTo.from), boundRenderedType(fromTo.to)) + ) + /** Two comparisons, `param >= from && param <= to`, each rendered by the SHARED per-type * dispatch. * @@ -2539,6 +2864,13 @@ case class BetweenExpr( * [[Expression.check]] knows. Overriding `check` puts BETWEEN on the ONE guarded emission path * and reuses that dispatch via the explicit `GE` / `LE` operators. The NOT is rendered by * `painlessNot` (`BETWEEN` has no negated operator). + * + * Each bound is rendered as the comparison reads it (a string literal by its reading: the + * number, or the temporal, it spells) and, like the right-hand side of a single comparison, + * brought to the operand's java.time type when the two differ: a DATE bound beside a TIMESTAMP + * operand is the instant its day starts at. Compared as rendered, a string bound was compared as + * TEXT (`String.valueOf().compareTo("2024-02-01")`, right only while the two spellings + * happened to sort alike) and a `LocalDate` bound failed against a `ZonedDateTime`. */ override protected def check( context: Option[PainlessContext], @@ -2546,17 +2878,35 @@ case class BetweenExpr( value: String, op: Operator ): String = { - val from = painlessOperand(fromTo.from, context) - val to = painlessOperand(fromTo.to, context) + def bound(b: Token): String = { + val rendered = readingPainless(readingFor(b), b, context) + val from = boundRenderedType(b) + val to = comparisonTargetType + if ( + from != to && from.isInstanceOf[SQLTemporal] && to.isInstanceOf[SQLTemporal] && + !context.exists(_.isProcessor) + ) + SQLTypeUtils.coerce(rendered, from, to, nullable = false, context) + else rendered + } + val from = bound(fromTo.from) + val to = bound(fromTo.to) s"(${super.check(context, param, from, GE)} && ${super.check(context, param, to, LE)})" } // The guarded bucket form of ` BETWEEN a AND b` -- two comparisons, not a chained // `a <= p <= b` (Painless rejects `boolean <= int`). The NOT is NOT rendered here: `BETWEEN` has // no negated operator, so `painlessNot` renders the `!` (review M-6) and - // `bucketPipelinePainless` already wraps this call in it. - override protected def bucketPipelineCheck(param: String): String = - s"$param >= ${fromTo.from} && $param <= ${fromTo.to}" + // `bucketPipelinePainless` already wraps this call in it. A string bound is the number, or the + // epoch milliseconds, it is read as -- a metric is a number (`readingFor`). + override protected def bucketPipelineCheck(param: String): String = { + def bound(b: Token): String = readingFor(b) match { + case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" + case Some(scalar) => ComparisonRule.scalarPainless(scalar).getOrElse(b.toString) + case None => b.toString + } + s"$param >= ${bound(fromTo.from)} && $param <= ${bound(fromTo.to)}" + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index f511370e2..537667c4a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.operator.{AND, OR, SetOperator, UNION} +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.schema.{ sqlConfig, validateScriptReferences, @@ -34,8 +35,8 @@ import app.softnetwork.elastic.sql.schema.{ Table => Schema, TableType } -import app.softnetwork.elastic.sql.function.{FunctionChain, FunctionUtils} -import app.softnetwork.elastic.sql.function.cond.{Case, NullIf} +import app.softnetwork.elastic.sql.function.{BinaryFunction, FunctionChain, FunctionUtils} +import app.softnetwork.elastic.sql.function.cond.{functionsOf, Case, NumericReducer} import app.softnetwork.elastic.sql.function.aggregate.WindowFunction import app.softnetwork.elastic.sql.function.time.DateDiff import app.softnetwork.elastic.sql.policy.{EnrichPolicy, EnrichPolicyType} @@ -63,6 +64,26 @@ import scala.util.Try package object query { + /** Every TYPE refusal of the functions reachable from `chain` (`functionsOf`), each one's own + * `Validation.typeError`, an arithmetic nested directly in another included -- it is an operand, + * not a chain, so `functionsOf` does not reach it. + * + * πŸ”΄ Asked of a SCHEMA-RESOLVED expression only: the lead's ruling of 2026-10-05, no refusal + * before the column types are known. ONE walk for every function rule moved off `validate()` + * (`ArithmeticExpression`, `NULLIF`, `CASE`, `GREATEST` / `LEAST`, the `INTERVAL` family). + */ + private[elastic] def chainTypeErrors(chain: FunctionChain): Seq[String] = { + def arithmetic(a: ArithmeticExpression): Seq[String] = + a.typeError.toSeq ++ a.args.flatMap { + case nested: ArithmeticExpression => arithmetic(nested) + case _ => Nil + } + functionsOf(chain).flatMap { + case a: ArithmeticExpression => arithmetic(a) + case f => f.typeError.toSeq + }.distinct + } + /** The `SingleSearch`es a statement EMBEDS β€” the ONE place these arms are spelled. * * Every closure-shaped question ("does this statement need the relational engine?", "does it @@ -1025,7 +1046,7 @@ package object query { case relation: ElasticRelation => walk(relation.criteria) case leaf => leaf.referencedIdentifiers - .filter(id => !id.isAggregation && id.hasAggregation) + .filter(SingleSearch.readsPerGroupCalculation) .map(leaf -> _) } having.flatMap(_.criteria).toSeq.flatMap(walk) @@ -1763,12 +1784,11 @@ package object query { * (`SearchApi.resolveWithSchema`), which is the same seam #306 uses to attach the schema in * the first place. * - * ⚠️ It is NOT a second `validate()`. Running the whole of `validate()` against resolved types - * would surface every pre-existing disagreement between a column's DECLARED type and its - * chain's β€” including `CAST(ts AS TIME) = CAST('07:00:00' AS TIME)`, which works today and - * which `Expression.validate`'s `identifier.out` (the COLUMN's type, not the chain's) already - * rejects when it is asked after resolution. This method carries exactly the rules that were - * measured, and each one names the shape it refuses. + * ⚠️ It is NOT a second `validate()`: `validate()` checks syntax and structure only, and this + * method asks the TYPE rules ([[typeErrors]]). Each of them reads the type an operand RENDERS + * (`renderedTypeOf`: the chain's, or an aggregate's own output type), never a column's + * declared type alone -- `CAST(ts AS TIME) = CAST('07:00:00' AS TIME)` compares a TIME with a + * TIME -- and each one names the shape it refuses. */ def validateResolved(): Either[String, Unit] = // πŸ”΄ Issue #389 / F4 -- the representability gate runs AGAIN here, on the RESOLVED statement. @@ -1783,15 +1803,33 @@ package object query { // which is issue #250's family. That throw stays as the last-resort invariant; this is what // makes it unreachable in practice. having.map(_.unrepresentable).getOrElse(Nil).headOption.map(u => Left(u.message)).getOrElse { - ((where.flatMap(_.criteria).toSeq ++ having.flatMap(_.criteria).toSeq ++ - select.fields.flatMap(f => Case.conditionsOf(f.identifier)) ++ + typeErrors.headOption.map(Left(_)).getOrElse(Right(())) + } + + /** Every TYPE refusal of this RESOLVED statement, by name, in clause order. + * + * πŸ”΄ The lead's ruling of 2026-10-05: no refusal before the column types are known. Every TYPE + * rule that `validate()` used to apply when the statement was parsed -- where every column is + * `Any` -- is asked here instead, on the real types, alongside the rules that always needed + * them: a comparison's ([[Criteria.typeErrors]]: TIME against a date, the operand match, `IN`, + * `BETWEEN`), and every function's ([[chainTypeErrors]]: date arithmetic and the operand + * match, `NULLIF`, `CASE`, `GREATEST` / `LEAST`, `INTERVAL`). + * + * With NO schema attached nothing calls this, and nothing is refused on types: Elasticsearch + * answers. `SearchApi.resolveWithSchema` calls it through [[validateResolved]]; so does a + * `CREATE TABLE ... AS SELECT` before it creates -- or, with `OR REPLACE`, deletes -- its + * target. + */ + def typeErrors: Seq[String] = + // A WHERE / HAVING clause's own comparison may be a QUERY on a column's mapping; a CASE + // condition is always a script (`Criteria.typeErrors`). + ((where.flatMap(_.criteria).toSeq ++ having.flatMap(_.criteria).toSeq) + .flatMap(_.typeErrors(scripted = false)) ++ + (select.fields.flatMap(f => Case.conditionsOf(f.identifier)) ++ orderBy.toSeq.flatMap(_.sorts.flatMap(s => Case.conditionsOf(s.field))) ++ groupBy.toSeq.flatMap(_.buckets.flatMap(b => Case.conditionsOf(b.identifier)))) - .flatMap(_.temporalComparisonErrors) ++ - scriptedExpressions.flatMap(NullIf.mismatchesOf)).headOption - .map(Left(_)) - .getOrElse(Right(())) - } + .flatMap(_.typeErrors(scripted = true)) ++ + scriptedExpressions.flatMap(chainTypeErrors)).distinct /** Every expression of this statement that can be emitted as Painless, as the chain it is. * @@ -1849,11 +1887,12 @@ package object query { // Arithmetic over aggregates written INLINE in HAVING (`HAVING MAX(x) - MIN(x) > 3`) has no // aggregation to read from and was silently dropped. Alias it in SELECT and reference the // alias (`... AS d ... HAVING d > 3`), which IS supported (Having.resolveAggregateAliases). + // `GREATEST` / `LEAST` inline are no arithmetic (`SingleSearch.readsPerGroupCalculation`). having .flatMap(_.criteria) .map(_.referencedIdentifiers) .getOrElse(Nil) - .find(id => !id.isAggregation && id.hasAggregation && id.fieldAlias.isEmpty) match { + .find(id => id.fieldAlias.isEmpty && SingleSearch.readsPerGroupCalculation(id)) match { case Some(id) => Left( s"HAVING cannot combine aggregates arithmetically inline (${id.sql}); alias the expression in SELECT and reference the alias" @@ -2330,6 +2369,32 @@ package object query { object SingleSearch { + /** Does a HAVING reference read a per-group calculation the way an arithmetic over aggregates + * is read? The ONE predicate the inline-arithmetic refusal (`validate()`) and a materialized + * view's rule 4 (`havingBucketScriptRefs`) share (#292). + * + * - ALIASED (`SELECT GREATEST(MAX(a), MAX(b)) AS m ... HAVING m > 1`, once + * `Having.resolveAggregateAliases` has put the item in place), it reads the SELECT item's + * `bucket_script` by name: any expression over aggregates, `GREATEST` / `LEAST` over them + * included. + * - INLINE, it is an ARITHMETIC over aggregates, the shape the group filter cannot read. + * `GREATEST` / `LEAST` report the aggregates of their arguments + * (`NumericReducer.hasAggregation`, so that a SELECT computes them per group), but inline + * they are a function over aggregates, which the group filter reads through its operands + * (issue #389) -- alone or under an arithmetic, exactly as before they reported any. + */ + private[query] def readsPerGroupCalculation(id: Identifier): Boolean = { + def carriesAggregate(token: Token): Boolean = token match { + case _: NumericReducer => false + case binary: BinaryFunction[_, _, _] => + carriesAggregate(binary.left) || carriesAggregate(binary.right) + case i: Identifier => i.isAggregation || i.functions.exists(carriesAggregate) + case other => other.hasAggregation + } + !id.isAggregation && + (if (id.fieldAlias.isDefined) id.hasAggregation else id.functions.exists(carriesAggregate)) + } + /** Name of the synthetic single-bucket aggregation the bridge emits for a `HAVING` with no * `GROUP BY` (see [[SingleSearch.wholeTableHaving]]). It exists only so that the * `bucket_selector` has a multi-bucket parent to hang from -- Elasticsearch rejects one inside @@ -2516,7 +2581,9 @@ package object query { case None => Right(()) } _ <- MultiSearch.branchArity(requests) - _ <- MultiSearch.branchTypes(requests) + // ⚠️ No `branchTypes` here: it is a TYPE rule, asked once each branch's schema is attached + // (`SearchApi.resolveWithSchema(MultiSearch)`), never when the statement is parsed (the + // lead's ruling of 2026-10-05). } yield () } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index c1fd29936..3b2ec46a1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -25,6 +25,7 @@ import app.softnetwork.elastic.sql.function.{ FunctionWithIdentifier } import app.softnetwork.elastic.sql.function.cond.{Case, NullIf} +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.serialization._ @@ -788,6 +789,78 @@ package object schema { .filter(path => projection.find(path).isEmpty) .distinct + /** The schema an index PATTERN or a comma list (`FROM logs-*`) answers with, so it is typed as + * the single index is: the union of its indices' columns, merged FIELD BY FIELD (the lead's + * ruling of 2026-10-05). + * + * - A field every index that maps it maps the SAME way -- the same type (a field no index + * declared is typed at the declaration seam, `schema.IndexField`: an undeclared `date` is a + * TIMESTAMP, like a declared one), the same date `format` (the one mapping option a + * statement's meaning depends on: a temporal literal is read against it) -- keeps that type, + * whatever the indices disagree on elsewhere. A field one index maps and another does not is + * the one index's. + * - A field two indices map DIFFERENTLY gets no type (`SQLTypes.Any`): a type cannot be chosen + * between them, so nothing is refused on it and Elasticsearch answers for it, as it does + * with no schema at all (the ruling's rule 4). It stays IN the schema: the name exists, and + * a subquery's scope must know it does. + * - Sub-fields (multi-fields, object properties) merge by the same rule, under their parent. + * + * πŸ”΄ Found by review, MEASURED on Elasticsearch 8.18.3: `SELECT g, MAX(d) - MIN(d) FROM zwild_* + * GROUP BY g` answered 2592000000 (milliseconds) where the single index answered 30 (days) -- + * with no schema attached a pattern was typed `Any` throughout, and the date-arithmetic rules + * could not see a date. A first merge attached NOTHING as soon as the indices disagreed on one + * field, which left every other field of such a pattern in that state. + * + * @return + * the merged schema, and every field the indices disagree on, by path, with the types they + * give it + */ + private[elastic] def mergedSchema( + name: String, + members: Seq[(String, Schema)] + ): (Schema, Seq[String]) = { + def format(c: Column): Option[Value[_]] = c.options.get("format") + def sameMapping(a: Column, b: Column): Boolean = + a.dataType == b.dataType && format(a) == format(b) + val conflicts = scala.collection.mutable.LinkedHashMap.empty[String, Set[String]] + def merge(into: List[Column], columns: List[Column], parent: String): List[Column] = { + val merged = scala.collection.mutable.LinkedHashMap(into.map(c => c.name -> c): _*) + columns.foreach { column => + val path = s"$parent${column.name}" + merged.get(column.name) match { + case None => merged.put(column.name, column) + case Some(first) => + val multiFields = merge(first.multiFields, column.multiFields, s"$path.") + if (sameMapping(first, column)) + merged.put(column.name, first.copy(multiFields = multiFields)) + else { + conflicts.put( + path, + conflicts.getOrElse(path, Set(first.dataType.typeId)) + column.dataType.typeId + ) + merged.put( + column.name, + first.copy( + dataType = SQLTypes.Any, + options = first.options - "format", + multiFields = multiFields + ) + ) + } + } + } + merged.values.toList + } + val columns = + members.foldLeft(List.empty[Column]) { case (acc, (_, schema)) => + merge(acc, schema.columns, "") + } + val disagreements = conflicts.toSeq.map { case (path, types) => + s"'$path' (${types.filterNot(_ == SQLTypes.Any.typeId).toSeq.sorted.mkString(", ")})" + } + (Table(name, columns = columns.sortBy(_.name)).update(), disagreements) + } + /** Reject a computed column whose expression reads a column the table does not declare, or hands * a column to a function that cannot take its declared type. * @@ -874,6 +947,27 @@ package object schema { case errors => return Left(errors.mkString("; ")) } + // Date arithmetic refused by name -- `d * 2`, `d + d2`, `2 - d` -- for the same two reasons as + // the NULLIF rule above: it needs the RESOLVED table, and no reference population, so it sits + // above the `isRegular` carve-out too. An ingest script would otherwise fail per document, and + // `ignore_failure` would swallow it into an absent column. + def arithmeticErrors(column: Column): Seq[String] = + column.script + .filterNot(_.materialized) + .toSeq + .flatMap(_.validationExpr) + .flatMap { + case chain: FunctionChain => ArithmeticExpression.typeErrorsOf(chain) + case _ => Nil + } + .map(reason => s"Column '${column.path}': $reason") ++ + column.multiFields.flatMap(arithmeticErrors) + + schema.columns.flatMap(arithmeticErrors).distinct match { + case Nil => + case errors => return Left(errors.mkString("; ")) + } + if (!schema.isRegular) return Right(()) def columnErrors(column: Column): Seq[String] = @@ -898,6 +992,14 @@ package object schema { case Some(referenced) => consumer.collect { case fn: FunctionN[_, _] => fn }.toSeq.flatMap { fn => if (acceptableOperand(referenced.dataType, fn.inputType)) Nil + // A DATE, TIMESTAMP or DATETIME operand of `+` / `-` is date arithmetic, which the + // rule above judges on BOTH operands -- a NUMERIC input type cannot say whether + // `d - d2` is legal. A TIME is outside those rules and keeps this check. + else if ( + fn.isInstanceOf[ArithmeticExpression] && referenced.dataType.isTemporal && + referenced.dataType != SQLTypes.Time + ) + Nil else Seq( s"Column '${column.path}' applies ${fn.sql} to '${id.path}', which is " + diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index d56ca35be..63be3e49a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -25,6 +25,7 @@ import app.softnetwork.elastic.sql.{ PainlessScript } import app.softnetwork.elastic.sql.`type`.SQLTypes._ +import app.softnetwork.elastic.sql.function.time.DateDiff object SQLTypeUtils { @@ -422,8 +423,24 @@ object SQLTypeUtils { s"((${painlessType(t)}) $expr)" // ---- NUMERIC <-> TEMPORAL ---- + // A number cast to a TIMESTAMP is the instant that many milliseconds from the epoch -- + // core's rule, and it holds for EVERY numeric source, not only a BIGINT one: with the + // BIGINT arm alone `CAST(x AS TIMESTAMP)` over a DOUBLE, and `(ts - ts2)::TIMESTAMP` (a + // DOUBLE number of days), fell to the identity fallback and answered the number itself -- + // a cast that silently did nothing. A fractional number of milliseconds is truncated, as + // every other fractional-to-integral cast here is. case (SQLTypes.BigInt, SQLTypes.Timestamp | SQLTypes.DateTime) => s"Instant.ofEpochMilli($expr).atZone(ZoneId.of('Z'))" + case ( + SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int, + SQLTypes.Timestamp | SQLTypes.DateTime + ) => + s"Instant.ofEpochMilli(((long) $expr)).atZone(ZoneId.of('Z'))" + case ( + SQLTypes.Double | SQLTypes.Real | SQLTypes.Numeric, + SQLTypes.Timestamp | SQLTypes.DateTime + ) => + s"Instant.ofEpochMilli(((long) $expr)).atZone(ZoneId.of('Z'))" case (SQLTypes.Timestamp | SQLTypes.DateTime, SQLTypes.BigInt) => s"$expr.toInstant().toEpochMilli()" @@ -719,10 +736,19 @@ object SQLTypeUtils { val parsed = declared match { // A DATE is written date-only, and `ZonedDateTime.parse` REFUSES a date with no time // (measured), so it is parsed as a `LocalDate` and then given the UTC start of day. + // + // πŸ”΄ ...unless it is longer than a date: Elasticsearch accepts a date-time string into a + // `date` field declared DATE, and the strict `yyyy-MM-dd` parse threw on it -- which a + // computed column's `ignore_failure: true` stored as NULL (`CASE WHEN .. THEN d ELSE d2 END - + // d`, MEASURED on Elasticsearch 8.18.3). Such a value is the instant it spells, given its UTC + // day, as `processorInstant` reads it and as the query venue's doc value is. case SQLTypes.Date => + val dateTime = coerce(expr, SQLTypes.Varchar, SQLTypes.Timestamp, nullable = false, None) Some( + s"($expr.length() > 10 ? $dateTime.withZoneSameInstant(ZoneId.of('Z'))" + + ".truncatedTo(ChronoUnit.DAYS) : " + coerce(expr, SQLTypes.Varchar, SQLTypes.Date, nullable = false, None) + - ".atStartOfDay(ZoneId.of('Z'))" + ".atStartOfDay(ZoneId.of('Z')))" ) case SQLTypes.DateTime | SQLTypes.Timestamp | SQLTypes.Temporal => Some(coerce(expr, SQLTypes.Varchar, SQLTypes.Timestamp, nullable = false, None)) @@ -734,6 +760,31 @@ object SQLTypeUtils { parsed.map(p => s"($expr instanceof String ? $p : $epoch)") } + /** An INGEST operand as the UTC instant it denotes, decided by its RUNTIME class: a string is + * parsed by its shape -- a date at the start of its UTC day, a date-time as the instant it + * spells, through the very ` -> ` conversions a `CAST` uses -- a number is + * epoch milliseconds, and anything else is a temporal a function already made + * (`DateDiff.utcInstant`). + * + * πŸ”΄ Found by review, measured on Elasticsearch 8.18.3: an operand that is not a bare column + * (`COALESCE(d, d2) - d2`, `NULLIF(d, d2) - d2`) holds the raw JSON string, and reading it as a + * temporal threw on every document -- which a computed column's `ignore_failure: true` stored as + * NULL on every row. And a DATE column holding a date-time string (Elasticsearch accepts one) + * failed the strict `yyyy-MM-dd` parse the same way. Both now read the instant the value spells, + * as the query venue's doc value does, and a DATE operand is then floored to its UTC day. + * + * `ref` is read up to five times: it is a name. + */ + private[sql] def processorInstant(ref: String): String = { + val dateTime = coerce(ref, SQLTypes.Varchar, SQLTypes.Timestamp, nullable = false, None) + val date = + coerce(ref, SQLTypes.Varchar, SQLTypes.Date, nullable = false, None) + + ".atStartOfDay(ZoneId.of('Z'))" + s"($ref == null ? null : $ref instanceof String ? ($ref.length() > 10 ? $dateTime : $date) : " + + s"$ref instanceof Number ? Instant.ofEpochMilli($ref).atZone(ZoneId.of('Z')) : " + + s"${DateDiff.utcInstant(ref)})" + } + private val painlessPrimitives: Set[String] = Set("byte", "short", "int", "long", "float", "double", "boolean", "char") diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index c092bdb25..62b65dc65 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1175,8 +1175,18 @@ class ParserSpec extends AnyFlatSpec with Matchers { } it should "reject GREATEST with definitively non-numeric args" in { + // once the column types are known -- never when the statement is parsed (the lead's ruling of + // 2026-10-05: no TYPE refusal before the column types are known) val result = Parser("SELECT GREATEST('a', 'b') AS hi FROM products") - result.isLeft shouldBe true + result.isRight shouldBe true + result match { + case Right(single: SingleSearch) => + single + .update(Some(Table("products", columns = Nil))) + .validateResolved() + .isLeft shouldBe true + case other => fail(s"$other") + } } // Story 14.6 β€” case-insensitivity of the Epic-14 keywords (AC 4): every new @@ -1505,7 +1515,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { .map(_.source) .getOrElse("") should include( """def param1 = ctx.birthdate; - |def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); + |def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(" ", "T"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); |def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); |def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); |def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); @@ -1521,16 +1531,16 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(schema.defaultPipeline.ddl) val json = schema.defaultPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" val indexMappings = schema.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"id":{"data_type":"INT","not_null":"true","comment":"user identifier"},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"birthdate":{"data_type":"DATE","not_null":"false"},"age":{"data_type":"INT","not_null":"false","script":{"sql":"TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5"}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4"}}}}},"type":"regular"}}""".stripMargin + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"id":{"data_type":"INT","not_null":"true","comment":"user identifier"},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"birthdate":{"data_type":"DATE","not_null":"false"},"age":{"data_type":"INT","not_null":"false","script":{"sql":"TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5"}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4"}}}}},"type":"regular"}}""".stripMargin val indexSettings = schema.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = schema.defaultPipelineNode println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala index 46a5420f1..6c73e51e1 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala @@ -236,15 +236,22 @@ class SetOperatorSpec extends AnyFlatSpec with Matchers { } "Branch types" should "be validated where both sides are typeable" in { - rejects("SELECT 1 AS n FROM t UNION SELECT 'a' AS n FROM u", "compatible types at column 1") - rejects( - "SELECT CAST(x AS BIGINT) AS v FROM t UNION ALL SELECT CAST(y AS VARCHAR) AS v FROM u", - "compatible types at column 1" - ) + // A TYPE rule, asked once each branch's schema is attached (`SearchApi.resolveWithSchema`), + // never when the statement is parsed (the lead's ruling of 2026-10-05). + def branchTypes(sql: String): Either[String, Unit] = + MultiSearch.branchTypes(parse(sql).requests) + Seq( + "SELECT 1 AS n FROM t UNION SELECT 'a' AS n FROM u", + "SELECT CAST(x AS BIGINT) AS v FROM t UNION ALL SELECT CAST(y AS VARCHAR) AS v FROM u" + ).foreach { sql => + withClue(s"[$sql] ") { + branchTypes(sql).swap.getOrElse("") should include("compatible types at column 1") + } + } // numeric family - Parser("SELECT 1 AS n FROM t UNION SELECT 2.5 AS n FROM u").isRight shouldBe true - // a bare column is `Any` at parse time β€” the core seam re-checks with schemas attached - Parser("SELECT a FROM t UNION SELECT 'x' FROM u").isRight shouldBe true + branchTypes("SELECT 1 AS n FROM t UNION SELECT 2.5 AS n FROM u") shouldBe Right(()) + // a bare column is `Any` until its schema is attached β€” the core seam checks with schemas + branchTypes("SELECT a FROM t UNION SELECT 'x' FROM u") shouldBe Right(()) } it should "validate a set operation INSIDE a derived table one level down" in { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/perf/ResolutionCostProbe.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/perf/ResolutionCostProbe.scala new file mode 100644 index 000000000..ff566ae8a --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/perf/ResolutionCostProbe.scala @@ -0,0 +1,529 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.perf + +import app.softnetwork.elastic.sql.function.aggregate.BucketScriptAggregation +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{ + CreateTable, + MetricSelectorScript, + MultiSearch, + SingleSearch +} +import app.softnetwork.elastic.sql.schema.{Column, Table} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} + +import scala.util.{Failure, Success, Try} + +/** Per-statement cost of everything a statement pays before Elasticsearch sees it, STAGE BY STAGE + * -- the companion of [[ParseCostProbe]] (parsing only) and [[EmissionCostProbe]] (parsing and one + * predicate's translation). + * + * Each iteration is one request's worth of work on a FRESH parse, timed per stage: + * + * - `parse` -- `Parser(sql)`, syntax and structure (`validate()`); + * - `attach` -- `updateAll` with the corpus schemas, what `SearchApi.resolveWithSchema` does + * once the mapping is read; + * - `validate` -- `validateResolved()`, the rules that need the column types (for a set + * operation, every branch's, then the branch type check); + * - `emit` -- the Painless the bridge renders from the statement: every script field, the WHERE + * criteria (query context), every per-group calculation (`bucket_script`) and the HAVING + * selector; + * + * and the `total` of the four. A statement refused at a stage stops there, as a request does, and + * says so in its verdict; a DDL statement is parsed, and its computed columns' scripts read. + * + * Corpus: [[ParseCostProbe]] 's statements, [[EmissionCostProbe]] 's, the TYPE rules that are + * asked once the column types are known (comparisons, IN, BETWEEN, NULLIF, CASE, GREATEST / LEAST, + * intervals, arithmetic, set operations -- their true mismatches and the statements they used to + * refuse although well typed), and the date-arithmetic request bodies. Then, as one block, the + * date-arithmetic POPULATION: every operand against every operand for `+ - * / %`, a few + * iterations each, reported as the sum of the per-statement medians per stage. + * + * Same protocol as the other probes: a global warm-up, a per-statement discarded warm-up, then + * `runs` timed iterations, reported as medians in microseconds. It uses only what the base already + * has, so the SAME file runs in a control tree. + * + * {{{ + * sbt "sql/Test/runMain app.softnetwork.elastic.sql.perf.ResolutionCostProbe" + * sbt "sql/Test/runMain app.softnetwork.elastic.sql.perf.ResolutionCostProbe 100 300 out.tsv" + * }}} + * + * With an output path, the per-statement table is written there as TSV and the population block + * beside it (`.population.tsv`), so two trees can be compared mechanically. + * + * ==This probe NEVER asserts== + * It prints and exits (issues #269/#270: a timing assertion is a CI-flake liability). Compare two + * trees by running it in each, INTERLEAVED, in one session -- an absolute number from another + * session is noise. + */ +object ResolutionCostProbe { + + val DefaultWarmup: Int = 100 + val DefaultRuns: Int = 300 + val PopulationWarmup: Int = 2 + val PopulationRuns: Int = 5 + + private def table(name: String, columns: (String, SQLType)*): Table = + Table(name, columns = columns.map { case (n, t) => Column(n, t) }.toList) + + private val schemas: Map[String, Table] = Map( + "t" -> table( + "t", + "id" -> SQLTypes.Keyword, + "g" -> SQLTypes.Keyword, + "s" -> SQLTypes.Keyword, + "name" -> SQLTypes.Keyword, + "other" -> SQLTypes.Keyword, + "num" -> SQLTypes.Keyword, + "n" -> SQLTypes.Int, + "m" -> SQLTypes.Int, + "x" -> SQLTypes.Double, + "amount" -> SQLTypes.Double, + "d" -> SQLTypes.Date, + "d2" -> SQLTypes.Date, + "ts" -> SQLTypes.Timestamp, + "ts2" -> SQLTypes.Timestamp, + "b" -> SQLTypes.Boolean + ), + "orders" -> table( + "orders", + "id" -> SQLTypes.Keyword, + "name" -> SQLTypes.Keyword, + "category" -> SQLTypes.Keyword, + "amount" -> SQLTypes.Double, + "created_at" -> SQLTypes.Timestamp, + "status" -> SQLTypes.Keyword, + "country" -> SQLTypes.Keyword, + "city" -> SQLTypes.Keyword, + "label" -> SQLTypes.Keyword, + "customer_id" -> SQLTypes.Keyword + ), + "customers" -> table( + "customers", + "id" -> SQLTypes.Keyword, + "cid" -> SQLTypes.Int, + "region" -> SQLTypes.Keyword, + "name" -> SQLTypes.Keyword + ), + "emp" -> table( + "emp", + "name" -> SQLTypes.Keyword, + "salary" -> SQLTypes.Double, + "department" -> SQLTypes.Keyword + ) + ) + + /** [[EmissionCostProbe]]'s predicate shapes (its list is private to it). */ + private val EmissionStatements: List[(String, String)] = List( + "plain column, numeric" -> "SELECT id FROM t WHERE n > 1", + "plain column, keyword" -> "SELECT id FROM t WHERE name = 'x'", + "plain column, temporal" -> "SELECT id FROM t WHERE d = CAST('2025-01-31' AS DATE)", + "folded function (YEAR)" -> "SELECT id FROM t WHERE YEAR(d) = 2025", + "folded function (DATE_TRUNC)" -> + "SELECT id FROM t WHERE DATE_TRUNC(d, MONTH) = CAST('2025-01-01' AS DATE)", + "expression function (WEEKDAY)" -> "SELECT id FROM t WHERE WEEKDAY(d) = 0", + "expression function (DATE_FORMAT)" -> "SELECT id FROM t WHERE DATE_FORMAT(d, 'yyyy') = '2025'", + "expression function (LAST_DAY)" -> + "SELECT id FROM t WHERE LAST_DAY(d) = CAST('2025-01-31' AS DATE)", + "conversion (CAST)" -> "SELECT id FROM t WHERE CAST(num AS INT) = 9", + "conversion (TRY_CAST)" -> "SELECT id FROM t WHERE TRY_CAST(num AS INT) = 9", + "conditional (ISNULL)" -> "SELECT id FROM t WHERE ISNULL(name) = true", + "conditional (NULLIF, cast)" -> "SELECT id FROM t WHERE NULLIF(CAST(num AS BIGINT), 0) > 1", + "conditional (NULLIF, function)" -> "SELECT id FROM t WHERE NULLIF(UPPER(name), 'X') = 'A'", + "string function (UPPER)" -> "SELECT id FROM t WHERE UPPER(name) = 'A'", + "math function (ABS)" -> "SELECT id FROM t WHERE ABS(amount) > 10", + "division, two integer columns" -> "SELECT id FROM t WHERE n / m > 1", + "division, literal divisor" -> "SELECT id FROM t WHERE amount / 2 > 1", + "IN, numeric" -> "SELECT id FROM t WHERE CAST(num AS BIGINT) IN (9, 12)", + "IN, strings" -> "SELECT id FROM t WHERE UPPER(name) IN ('A', 'B')", + "both sides" -> "SELECT id FROM t WHERE YEAR(d) = YEAR(ts)", + "composite AND/OR" -> + "SELECT id FROM t WHERE WEEKDAY(d) = 0 AND UPPER(name) = 'A' OR ABS(amount) > 10" + ) + + /** The TYPE rules asked once the column types are known: one true mismatch per rule, then the + * statements that are well typed -- the ones parse-time rules used to refuse, a literal against + * a bare column, functions over dates. + */ + private val TypeRuleStatements: List[(String, String)] = List( + "comparison, literal mismatch" -> "SELECT id FROM t WHERE n + 1 > 'abc'", + "comparison, column mismatch" -> "SELECT id FROM t WHERE s = n", + "comparison, TIME" -> "SELECT id FROM t WHERE CAST(ts AS TIME) > ts", + "IN, mismatch" -> "SELECT id FROM t WHERE CAST(n AS INT) IN ('a', 'b')", + "IN (HAVING), mismatch" -> "SELECT g FROM t GROUP BY g HAVING COUNT(n) IN ('a', 'b')", + "BETWEEN bounds, mismatch" -> "SELECT id FROM t WHERE n BETWEEN 1 AND 'a'", + "BETWEEN operand, mismatch" -> "SELECT id FROM t WHERE CAST(s AS INT) BETWEEN 'a' AND 'b'", + "NULLIF, mismatch" -> "SELECT NULLIF(s, 1) AS v FROM t", + "NULLIF, kinds" -> "SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP)) AS v FROM t", + "CASE WHEN, non-boolean" -> "SELECT CASE WHEN n THEN 1 ELSE 0 END AS v FROM t", + "CASE , mismatch" -> "SELECT CASE n WHEN 'a' THEN 1 ELSE 0 END AS v FROM t", + "GREATEST, strings" -> "SELECT GREATEST('a', 'b') AS v FROM t", + "LEAST, dates" -> "SELECT LEAST(d, d2) AS v FROM t", + "date arithmetic, clock" -> "SELECT CURRENT_DATE * 2 AS v FROM t", + "date arithmetic, columns" -> "SELECT d + d2 AS v FROM t", + "arithmetic, column mismatch" -> "SELECT n + s AS v FROM t", + "arithmetic, literal mismatch" -> "SELECT 1 + 'abc' AS v FROM t", + "set operation, mismatch" -> "SELECT 1 AS n FROM t UNION ALL SELECT 'a' AS n FROM t", + "set operation, agreeing" -> "SELECT id FROM t UNION ALL SELECT g FROM t", + "well typed, CURRENT_DATE - d" -> "SELECT id FROM t WHERE CURRENT_DATE - d > 30", + "well typed, d + 7" -> "SELECT id FROM t WHERE d + 7 > CURRENT_DATE", + "well typed, NOW() - ts" -> "SELECT id FROM t WHERE NOW() - ts > 1", + "well typed, CAST - d" -> "SELECT id FROM t WHERE CAST(ts AS DATE) - d = 0", + "well typed, d + 1 = DATE" -> "SELECT id FROM t WHERE d + 1 = CAST('2024-02-01' AS DATE)", + "well typed, d + 1 > literal" -> "SELECT id FROM t WHERE d + 1 > '2024-01-01'", + "well typed, HAVING alias" -> + "SELECT g, MAX(d) + 1 AS v FROM t GROUP BY g HAVING v > CAST('2024-01-01' AS DATE)", + "well typed, CASE WHEN b" -> "SELECT CASE WHEN b THEN 1 ELSE 0 END AS v FROM t", + "literal vs column, =" -> "SELECT id FROM t WHERE n = '5'", + "literal vs column, date" -> "SELECT id FROM t WHERE d >= '2024-01-01'", + "literal vs column, IN" -> "SELECT id FROM t WHERE n IN ('1', '2')", + "literal vs column, BETWEEN" -> "SELECT id FROM t WHERE n BETWEEN '1' AND '9'", + "interval over a date" -> "SELECT id FROM t WHERE DATE_ADD(d, INTERVAL 1 DAY) > ts", + "COALESCE against the clock" -> "SELECT id FROM t WHERE COALESCE(d, d2) > CURRENT_DATE", + "CASE, many branches" -> + ("SELECT CASE WHEN n > 10 THEN 'a' WHEN n > 5 THEN 'b' WHEN b THEN 'c' ELSE 'd' END AS v " + + "FROM t WHERE x > 1 AND s IN ('p', 'q') AND d BETWEEN '2024-01-01' AND '2024-12-31'") + ) + + /** The request bodies date arithmetic renders: rows, comparisons against an instant, the clock in + * a per-group calculation, a cast over arithmetic, an aggregate's own type, a DDL computed + * column. The typed NULLs come last: on a tree where `CAST(NULL AS ..)` retypes the shared NULL + * literal, nothing after them can be affected. + */ + private val DateArithmeticStatements: List[(String, String)] = List( + "row, d + 1" -> "SELECT d + 1 AS v FROM t", + "row, ts - 1.5" -> "SELECT ts - 1.5 AS v FROM t", + "row, d - d2" -> "SELECT d - d2 AS v FROM t", + "row, ts - ts2" -> "SELECT ts - ts2 AS v FROM t", + "row, d + '1'" -> "SELECT d + '1' AS v FROM t", + "row, COALESCE(d, d2) - d" -> "SELECT COALESCE(d, d2) - d AS v FROM t", + "row, NULLIF(d, d2) + 1" -> "SELECT NULLIF(d, d2) + 1 AS v FROM t", + "row, CASE - d" -> "SELECT CASE WHEN n > 0 THEN d ELSE d2 END - d AS v FROM t", + "where, d + 1 > ts" -> "SELECT id FROM t WHERE d + 1 > ts", + "where, ts > d + 1" -> "SELECT id FROM t WHERE ts > d + 1", + "where, CAST(d) >= ts" -> "SELECT id FROM t WHERE CAST(d AS TIMESTAMP) >= ts", + "where, LAST_DAY(d) = ts" -> "SELECT id FROM t WHERE LAST_DAY(d) = ts", + "where, d - d2 > 3" -> "SELECT id FROM t WHERE d - d2 > 3", + "group, CURRENT_DATE - MAX(d)" -> "SELECT g, CURRENT_DATE - MAX(d) AS v FROM t GROUP BY g", + "group, NOW() - MAX(ts)" -> "SELECT g, NOW() - MAX(ts) AS v FROM t GROUP BY g", + "group, DATEDIFF(clock, MAX)" -> "SELECT g, DATEDIFF(CURRENT_DATE, MAX(d)) AS v FROM t GROUP BY g", + "group, MAX(d) - MIN(d)" -> "SELECT g, MAX(d) - MIN(d) AS v FROM t GROUP BY g", + "group, COUNT(d) - 1" -> "SELECT g, COUNT(d) - 1 AS v FROM t GROUP BY g", + "group, MAX(d) - AVG(n)" -> "SELECT g, MAX(d) - AVG(n) AS v FROM t GROUP BY g", + "having, alias of MAX - MIN" -> "SELECT g, MAX(d) - MIN(d) AS k FROM t GROUP BY g HAVING k > 1", + "cast, (n + x)::BIGINT" -> "SELECT (n + x)::BIGINT AS v FROM t", + "cast, (d + 1)::TIMESTAMP" -> "SELECT (d + 1)::TIMESTAMP AS v FROM t", + "cast, (x * 2)::INT" -> "SELECT (x * 2)::INT AS v FROM t", + "DDL, computed date columns" -> + ("CREATE TABLE ev2 (id INT, d DATE, d2 DATE, gap INT SCRIPT AS (d - d2), " + + "next DATE SCRIPT AS (COALESCE(d, d2) + 1), PRIMARY KEY (id))"), + "typed NULL, CAST(NULL AS DATE) - d" -> "SELECT CAST(NULL AS DATE) - d AS v FROM t", + "typed NULL, d + CAST(NULL AS INT)" -> "SELECT d + CAST(NULL AS INT) AS v FROM t" + ) + + val Statements: List[(String, String, String)] = + ParseCostProbe.Statements.map { case (l, s) => ("parse", l, s) } ++ + EmissionStatements.map { case (l, s) => ("emission", l, s) } ++ + TypeRuleStatements.map { case (l, s) => ("type rules", l, s) } ++ + DateArithmeticStatements.map { case (l, s) => ("date arithmetic", l, s) } + + /** The date-arithmetic population: every operand on both sides of every operator. */ + private val PopulationOperands: List[(String, Boolean)] = List( + "d" -> false, + "ts" -> false, + "MAX(d)" -> true, + "MAX(ts)" -> true, + "2" -> false, + "1.5" -> false, + "n" -> false, + "x" -> false, + "CURRENT_DATE" -> false, + "NOW()" -> false, + "NULL" -> false, + "CASE WHEN n > 0 THEN d ELSE d2 END" -> false, + "COALESCE(d, d2)" -> false, + "COALESCE(ts, d)" -> false, + "NULLIF(d, d2)" -> false, + "GREATEST(n, 2)" -> false, + "LEAST(x, 1.5)" -> false, + "AVG(n)" -> true, + "SUM(n)" -> true, + "COUNT(d)" -> true, + "'1'" -> false, + "'1.5'" -> false, + "'x'" -> false, + // typed NULLs last, for the reason given on `DateArithmeticStatements` + "CAST(NULL AS DATE)" -> false, + "CAST(NULL AS INT)" -> false + ) + + val Population: List[String] = + for { + (l, la) <- PopulationOperands + (r, ra) <- PopulationOperands + op <- List("+", "-", "*", "/", "%") + } yield + if (la || ra) s"SELECT g, $l $op $r AS x FROM t GROUP BY g" + else s"SELECT $l $op $r AS x FROM t" + + private def queryContext(): PainlessContext = PainlessContext(context = PainlessContextType.Query) + + /** What the bridge renders from a resolved statement, as Painless text. */ + private def emission(s: SingleSearch): Int = { + var n = 0 + s.scriptFields.filterNot(_.isAggregation).foreach { f => + val ctx = queryContext() + val body = f.painless(Some(ctx)) + n += (ctx.toString + body).length + } + s.where.flatMap(_.criteria).foreach { c => + val ctx = queryContext() + val body = c.painless(Some(ctx)) + n += (ctx.toString + body).length + } + s.select.fields.filter(_.isBucketScript).foreach { f => + n += BucketScriptAggregation(f.identifier).update(s).identifier.painless(None).length + } + s.having.flatMap(_.criteria).foreach(c => n += MetricSelectorScript.metricSelector(c).length) + n + } + + /** One timed iteration: nanoseconds per stage (`-1` = not reached) and the verdict. */ + private final case class Iteration( + parse: Long, + attach: Long, + validate: Long, + emit: Long, + verdict: String, + sink: Int + ) { + def total: Long = parse + math.max(attach, 0L) + math.max(validate, 0L) + math.max(emit, 0L) + } + + private def iteration(sql: String): Iteration = { + val t0 = System.nanoTime() + val parsed = Parser(sql) + val t1 = System.nanoTime() + parsed match { + case Left(_) => Iteration(t1 - t0, -1, -1, -1, "parse-refused", 0) + case Right(single: SingleSearch) => + val attached = Try(single.updateAll(schemas)) + val t2 = System.nanoTime() + attached match { + case Failure(e) => Iteration(t1 - t0, t2 - t1, -1, -1, s"attach-throws", e.hashCode()) + case Success(resolved) => + val verdict = Try(resolved.validateResolved()) + val t3 = System.nanoTime() + verdict match { + case Success(Right(())) => + val emitted = Try(emission(resolved)) + val t4 = System.nanoTime() + Iteration( + t1 - t0, + t2 - t1, + t3 - t2, + t4 - t3, + if (emitted.isSuccess) "ok" else "emit-throws", + emitted.getOrElse(0) + ) + case Success(Left(_)) => + Iteration(t1 - t0, t2 - t1, t3 - t2, -1, "resolve-refused", 0) + case Failure(_) => Iteration(t1 - t0, t2 - t1, t3 - t2, -1, "validate-throws", 0) + } + } + case Right(multi: MultiSearch) => + val attached = Try(multi.requests.map(_.updateAll(schemas))) + val t2 = System.nanoTime() + attached match { + case Failure(e) => Iteration(t1 - t0, t2 - t1, -1, -1, "attach-throws", e.hashCode()) + case Success(branches) => + val verdict = Try( + branches + .map(_.validateResolved()) + .collectFirst { case l @ Left(_) => l } + .getOrElse(MultiSearch.branchTypes(branches)) + ) + val t3 = System.nanoTime() + verdict match { + case Success(Right(())) => + val emitted = Try(branches.map(emission).sum) + val t4 = System.nanoTime() + Iteration( + t1 - t0, + t2 - t1, + t3 - t2, + t4 - t3, + if (emitted.isSuccess) "ok" else "emit-throws", + emitted.getOrElse(0) + ) + case Success(Left(_)) => + Iteration(t1 - t0, t2 - t1, t3 - t2, -1, "resolve-refused", 0) + case Failure(_) => Iteration(t1 - t0, t2 - t1, t3 - t2, -1, "validate-throws", 0) + } + } + case Right(ddl: CreateTable) => + val scripts = Try(ddl.schema.columns.flatMap(_.script).map(_.source.length).sum) + val t2 = System.nanoTime() + Iteration(t1 - t0, -1, -1, t2 - t1, "ddl", scripts.getOrElse(0)) + case Right(other) => Iteration(t1 - t0, -1, -1, -1, "other", other.hashCode()) + } + } + + private def median(values: Seq[Long]): Long = + if (values.isEmpty || values.forall(_ < 0)) -1L + else { + val sorted = values.filter(_ >= 0).sorted.toArray + val n = sorted.length + if (n % 2 == 1) sorted(n / 2) else (sorted(n / 2 - 1) + sorted(n / 2)) / 2 + } + + private def p95(values: Seq[Long]): Long = { + val sorted = values.sorted.toArray + sorted(math.min(sorted.length - 1, (sorted.length.toLong * 95L / 100L).toInt)) + } + + /** Locale-independent: this machine's default locale renders a decimal comma. */ + private def micros(nanos: Long): String = + if (nanos < 0) "-" + else String.format(java.util.Locale.ROOT, "%.1f", java.lang.Double.valueOf(nanos / 1000.0)) + + private final case class Row( + group: String, + label: String, + verdict: String, + parse: Long, + attach: Long, + validate: Long, + emit: Long, + total: Long, + totalP95: Long + ) + + private def measure( + group: String, + label: String, + sql: String, + warmup: Int, + runs: Int + ): (Row, Int) = { + var sink = 0 + for (_ <- 1 to warmup) sink ^= iteration(sql).sink + val samples = (1 to runs).map(_ => iteration(sql)) + samples.foreach(s => sink ^= s.sink) + val totals = samples.map(_.total) + ( + Row( + group, + label, + samples.head.verdict, + median(samples.map(_.parse)), + median(samples.map(_.attach)), + median(samples.map(_.validate)), + median(samples.map(_.emit)), + median(totals), + p95(totals) + ), + sink + ) + } + + def main(args: Array[String]): Unit = { + val warmup = if (args.length > 0) args(0).toInt else DefaultWarmup + val runs = if (args.length > 1) args(1).toInt else DefaultRuns + val out = if (args.length > 2) Some(args(2)) else None + if (warmup < 0 || runs < 1) { + throw new RuntimeException( + s"ResolutionCostProbe: warmup must be >= 0 and runs >= 1 (got warmup=$warmup runs=$runs)" + ) + } + + // Global warm-up: every object initialised and the hot paths compiled before any timing. + var sink = 0 + for (_ <- 1 to warmup; (_, _, sql) <- Statements) sink ^= iteration(sql).sink + + val rows = Statements.map { case (group, label, sql) => + val (row, s) = measure(group, label, sql, warmup, runs) + sink ^= s + row + } + + val header = + f"${"group"}%-16s ${"statement"}%-36s ${"verdict"}%-16s ${"parse"}%8s ${"attach"}%8s ${"validate"}%9s ${"emit"}%8s ${"total"}%9s ${"p95"}%9s" + println() + println(s"ResolutionCostProbe - warmup=$warmup runs=$runs, medians in microseconds") + println(header) + println("-" * header.length) + rows.foreach { r => + println( + f"${r.group}%-16s ${r.label.take(36)}%-36s ${r.verdict}%-16s ${micros(r.parse)}%8s ${micros( + r.attach + )}%8s ${micros(r.validate)}%9s ${micros(r.emit)}%8s ${micros(r.total)}%9s ${micros(r.totalP95)}%9s" + ) + } + println("-" * header.length) + def sum(f: Row => Long): Long = rows.map(f).filter(_ >= 0).sum + println( + f"${"SUM of medians"}%-70s ${micros(sum(_.parse))}%8s ${micros(sum(_.attach))}%8s ${micros( + sum(_.validate) + )}%9s ${micros(sum(_.emit))}%8s ${micros(sum(_.total))}%9s" + ) + + // The population block: a few iterations per statement, summed per stage. + val population = Population.map { sql => + val (row, s) = measure("population", sql, sql, PopulationWarmup, PopulationRuns) + sink ^= s + row + } + def popSum(f: Row => Long): Long = population.map(f).filter(_ >= 0).sum + val verdicts = + population.groupBy(_.verdict).map { case (v, rs) => s"$v=${rs.size}" }.toSeq.sorted + println() + println( + s"population: ${population.size} statements (${verdicts.mkString(", ")}), " + + s"$PopulationRuns runs each, SUM of medians in microseconds: parse ${micros(popSum(_.parse))}, " + + s"attach ${micros(popSum(_.attach))}, validate ${micros(popSum(_.validate))}, " + + s"emit ${micros(popSum(_.emit))}, total ${micros(popSum(_.total))}" + ) + println() + + out.foreach { path => + def write(file: String, lines: Seq[Row]): Unit = { + val sb = new StringBuilder + sb.append( + "group\tstatement\tverdict\tparse_ns\tattach_ns\tvalidate_ns\temit_ns\ttotal_ns\ttotal_p95_ns\n" + ) + lines.foreach { r => + sb.append( + s"${r.group}\t${r.label}\t${r.verdict}\t${r.parse}\t${r.attach}\t${r.validate}\t${r.emit}\t${r.total}\t${r.totalP95}\n" + ) + } + java.nio.file.Files.write( + java.nio.file.Paths.get(file), + sb.toString.getBytes(java.nio.charset.StandardCharsets.UTF_8) + ) + } + write(path, rows) + write(s"$path.population.tsv", population) + println(s"ResolutionCostProbe: wrote $path and $path.population.tsv") + } + + // Keep `sink` observable so the JIT cannot eliminate the work as dead code. + if (sink == Int.MinValue) println("unreachable sink guard") + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala new file mode 100644 index 000000000..0315ee595 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala @@ -0,0 +1,432 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** The date-arithmetic rules, as the TYPE every `+ - * / %` expression with a temporal operand + * gets, or the refusal by name it earns -- asked of a generated population once its schema is + * attached, and NEVER when the statement is parsed: no type is refused before the column types are + * known (the lead's ruling of 2026-10-05). + * + * The expectation is written HERE, from the ruling, and never read from the implementation: + * + * | Expression | Type | + * |:-------------------------------------------------------------------|:--------------------------------| + * | DATE - DATE | BIGINT | + * | TIMESTAMP - TIMESTAMP, a DATE and a TIMESTAMP | DOUBLE | + * | DATE +/- a whole number | DATE | + * | DATE +/- a fractional number | TIMESTAMP | + * | TIMESTAMP +/- a number | TIMESTAMP | + * | a number + a temporal | as the temporal + it | + * | a temporal and NULL (+ or -) | not refused (the value is NULL) | + * | temporal + temporal, a temporal under * / %, a number - a temporal | refused by name | + * | a temporal and anything that is neither a number nor a temporal | refused by name | + * + * What an operand IS for these rules: + * - `NOW()` is a DATETIME, which these rules read as an instant, like a TIMESTAMP; + * - an aggregate is the metric it produces: `COUNT` a BIGINT, `AVG` a DOUBLE, `SUM` its column's + * number, `MIN` / `MAX` what they aggregate; + * - `CASE`, `COALESCE`, `GREATEST` and `LEAST` are the least common supertype of their branches + * or arguments, `NULLIF` its first argument -- `GREATEST` / `LEAST` over dates included, a + * DATE when every argument is one, a TIMESTAMP when they mix; + * - a typed NULL is its type (`CAST(NULL AS DATE)` is a DATE), a bare `NULL` has none; + * - a string literal that is a number is that number (`'1'`, `'1.5'`); one that spells a date or + * a timestamp is, beside a temporal, that temporal read as the other operand's type (`d - + * '2024-01-31'` is DATE minus DATE, as PostgreSQL reads it) and a string beside anything else; + * any other is a string; + * - a `date` field elasticsql never declared (`u`, an index created by a bulk load) holds + * instants: a TIMESTAMP, typed so at the declaration seam -- DESCRIBE shows it so too. + */ +class DateArithmeticTypeSpec extends AnyFlatSpec with Matchers { + + private val schema: SchemaTable = SchemaTable( + "ev", + columns = List( + Column("g", SQLTypes.Keyword), + Column("d", SQLTypes.Date), + Column("d2", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("n", SQLTypes.Int), + Column("x", SQLTypes.Double), + // an Elasticsearch `date` with no elasticsql declaration (no `_meta` data type), typed at + // the declaration seam (`schema.IndexField`) + Index("raw", """{"mappings":{"properties":{"u":{"type":"date"}}},"settings":{}}""").schema + .find("u") + .getOrElse(fail("no undeclared column u")) + ) + ) + + private val ddlColumns = "g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, n INT, x DOUBLE" + + /** What the ruling sees in an operand. */ + private sealed trait Kind + private case object DateKind extends Kind + private case object InstantKind extends Kind + private case object Whole extends Kind + private case object Fractional extends Kind + private case object NullKind extends Kind + private case object TextKind extends Kind + + /** A string literal that spells a date or a timestamp: what it IS depends on the other operand. + */ + private case object TemporalText extends Kind + + /** One operand of the population: its SQL, what the ruling sees in it, the type name a refusal + * names it by, whether it is an aggregate, and whether a computed column can hold it (the DDL + * venue declares every column, so it has no undeclared one, and no aggregate). + */ + private final case class Operand( + sql: String, + kind: Kind, + typeName: String, + aggregate: Boolean = false, + ddl: Boolean = true + ) + + private val operands: Seq[Operand] = Seq( + Operand("d", DateKind, "DATE"), + Operand("ts", InstantKind, "TIMESTAMP"), + Operand("MAX(d)", DateKind, "DATE", aggregate = true, ddl = false), + Operand("MAX(ts)", InstantKind, "TIMESTAMP", aggregate = true, ddl = false), + Operand("2", Whole, "BIGINT"), + Operand("1.5", Fractional, "DOUBLE"), + Operand("n", Whole, "INT"), + Operand("x", Fractional, "DOUBLE"), + Operand("CURRENT_DATE", DateKind, "DATE"), + Operand("NOW()", InstantKind, "DATETIME"), + Operand("NULL", NullKind, "NULL"), + // ---- the fix round's extension (E1) ---- + Operand("u", InstantKind, "TIMESTAMP", ddl = false), + Operand("MAX(u)", InstantKind, "TIMESTAMP", aggregate = true, ddl = false), + Operand("CASE WHEN n > 0 THEN d ELSE d2 END", DateKind, "DATE"), + Operand("COALESCE(d, d2)", DateKind, "DATE"), + Operand("COALESCE(ts, d)", InstantKind, "TIMESTAMP"), + Operand("NULLIF(d, d2)", DateKind, "DATE"), + Operand("GREATEST(n, 2)", Whole, "BIGINT"), + Operand("LEAST(x, 1.5)", Fractional, "DOUBLE"), + Operand("AVG(n)", Fractional, "DOUBLE", aggregate = true, ddl = false), + Operand("SUM(n)", Whole, "INT", aggregate = true, ddl = false), + Operand("COUNT(d)", Whole, "BIGINT", aggregate = true, ddl = false), + Operand("CAST(NULL AS DATE)", DateKind, "DATE"), + Operand("CAST(NULL AS INT)", Whole, "INT"), + Operand("'1'", Whole, "VARCHAR"), + Operand("'1.5'", Fractional, "VARCHAR"), + Operand("'x'", TextKind, "VARCHAR"), + // ---- fix round 2 (G1): GREATEST / LEAST over dates and timestamps ---- + Operand("GREATEST(d, d2)", DateKind, "DATE"), + Operand("LEAST(ts, d)", InstantKind, "TIMESTAMP"), + Operand("GREATEST(MAX(d), MAX(d2))", DateKind, "DATE", aggregate = true, ddl = false), + // ---- fix round 3 (T1): a string literal that spells a date or a timestamp ---- + Operand("'2024-01-31'", TemporalText, "VARCHAR"), + Operand("'2024-01-31T10:00:00Z'", TemporalText, "VARCHAR") + ) + + private val operators: Seq[String] = Seq("+", "-", "*", "/", "%") + + private sealed trait Expected + private final case class Typed(t: SQLType) extends Expected + private case object NullValued extends Expected + private case object Refused extends Expected + private case object NotDateArithmetic extends Expected + + private def temporal(k: Kind): Boolean = k == DateKind || k == InstantKind + private def number(k: Kind): Boolean = k == Whole || k == Fractional + + private def shifted(t: Kind, n: Kind): Typed = + Typed(if (t == DateKind && n == Whole) SQLTypes.Date else SQLTypes.Timestamp) + + /** What an operand IS beside the other: a literal that spells a temporal is, beside a temporal, + * that temporal read as the other's type -- the comparison rule's reading, as PostgreSQL reads + * an untyped literal -- and a string beside anything else. + */ + private def beside(k: Kind, other: Kind): Kind = + if (k != TemporalText) k else if (temporal(other)) other else TextKind + + /** The ruling, restated. */ + private def expected(op: String, left: Kind, right: Kind): Expected = { + val l = beside(left, right) + val r = beside(right, left) + if (!temporal(l) && !temporal(r)) NotDateArithmetic + else + op match { + case "*" | "/" | "%" => Refused + case "+" => + (l, r) match { + case (t, n) if temporal(t) && number(n) => shifted(t, n) + case (n, t) if number(n) && temporal(t) => shifted(t, n) + case (NullKind, _) | (_, NullKind) => NullValued + case _ => Refused + } + case "-" => + (l, r) match { + case (DateKind, DateKind) => Typed(SQLTypes.BigInt) + case (a, b) if temporal(a) && temporal(b) => Typed(SQLTypes.Double) + case (t, n) if temporal(t) && number(n) => shifted(t, n) + case (NullKind, _) | (_, NullKind) => NullValued + case _ => Refused + } + } + } + + private final case class Case(op: String, l: Operand, r: Operand) { + val expression: String = s"${l.sql} $op ${r.sql}" + val outcome: Expected = expected(op, l.kind, r.kind) + val aggregate: Boolean = l.aggregate || r.aggregate + def sql: String = + if (aggregate) s"SELECT g, $expression AS x FROM ev GROUP BY g" + else s"SELECT $expression AS x FROM ev" + } + + private val population: Seq[Case] = + for { + op <- operators + l <- operands + r <- operands + } yield Case(op, l, r) + + /** The refusal names the operator, both operand types and the remedy, and nothing internal. */ + private def namesTheRefusal(c: Case, message: String): Boolean = + message.contains(s"operator ${c.op} ") && + message.contains(s"${c.l.typeName} and ${c.r.typeName}") && + message.contains("subtract two dates for the days between them") && + message.contains("add or subtract a number of days") && + !message.matches("(?s).*(? Some(s"refused at parse (${err.msg}): no TYPE is refused at parse") + case Right(ss: SingleSearch) => + val resolved = ss.update(Some(schema)) + val item = resolved.select.fields + .find(_.fieldAlias.exists(_.alias == "x")) + .getOrElse(fail(s"[${c.sql}] no x item")) + (c.outcome, resolved.validateResolved()) match { + case (Refused, Left(message)) => + if (namesTheRefusal(c, message)) None else Some(s"refused as [$message]") + case (Refused, Right(_)) => + Some(s"accepted, reported ${item.identifier.reportedType}, expected a refusal") + case (Typed(t), Right(_)) => + val reported = item.identifier.reportedType + if (reported == t) None else Some(s"reported $reported, expected $t") + case (NullValued, Right(_)) => None + case (NotDateArithmetic, Right(_)) => + if (ArithmeticExpression.typeErrorsOf(item.identifier).isEmpty) None + else Some("a date-arithmetic refusal for an expression with no temporal") + // a type rule other than the date-arithmetic one (`'x' + 2`), asked once types are known + case (NotDateArithmetic, Left(message)) if !dateArithmeticRefusal(message) => None + case (other, Left(message)) => Some(s"refused as [$message], expected $other") + } + case Right(other) => Some(s"parsed as ${other.getClass.getSimpleName}") + } + + "every + - * / % with a DATE, TIMESTAMP or DATETIME operand" should + "get the ruling's type or be refused by name, once its schema is attached -- never at parse" in { + // Non-vacuity, over the material: every kind on both sides of every operator, and every + // outcome of the ruling present. + population.size shouldBe operators.size * operands.size * operands.size + val outcomes = population.map(_.outcome) + outcomes.count(_ == Refused) should be > 100 + outcomes.collect { case Typed(t) => t }.toSet shouldBe + Set(SQLTypes.BigInt, SQLTypes.Double, SQLTypes.Date, SQLTypes.Timestamp) + outcomes should contain(NullValued) + outcomes should contain(NotDateArithmetic) + + val mismatches = population.flatMap(c => mismatch(c).map(m => s"[${c.expression}] $m")) + info( + s"date arithmetic, query venue: ${population.size} expressions " + + s"(${outcomes.count(_ == Refused)} refused, ${outcomes.count(_.isInstanceOf[Typed])} typed, " + + s"${outcomes.count(_ == NullValued)} NULL, ${outcomes.count(_ == NotDateArithmetic)} " + + s"without a temporal); mismatches: ${mismatches.size}" + ) + withClue(mismatches.take(30).mkString("\n", "\n", "\n")) { + mismatches shouldBe empty + } + } + + /** The DDL venue, as `CREATE TABLE` runs it: `Parser` -> `CreateTable.validate()` -> + * `validateScriptReferences` on the RESOLVED table -- its column types are declared in the + * statement itself. A computed column cannot hold an aggregate, so the population is the + * row-level operands, and every column is declared. + */ + private def ddlVerdict(expression: String, dataType: String): Either[String, String] = + Parser(s"CREATE TABLE ev ($ddlColumns, c $dataType SCRIPT AS ($expression))") match { + case Right(ct: CreateTable) => + Right( + ct.schema.columns + .find(_.name == "c") + .flatMap(_.script) + .map(_.source) + .getOrElse(fail(s"no script processor for [$expression]")) + ) + case Right(other) => fail(s"[$expression] expected a CreateTable, got $other") + case Left(err) => Left(err.msg) + } + + "a computed column over date arithmetic" should + "be created with the ruling's type, or refused by name" in { + val ddlOperands = operands.filter(_.ddl) + val ddlPopulation = population.filter(c => c.l.ddl && c.r.ddl) + ddlPopulation.size shouldBe operators.size * ddlOperands.size * ddlOperands.size + val mismatches = ddlPopulation.flatMap { c => + val dataType = c.outcome match { + case Typed(t) => t.typeId + case _ => "BIGINT" + } + val verdict = ddlVerdict(c.expression, dataType) + ((c.outcome, verdict) match { + case (Refused, Left(message)) => + if (message.startsWith("Column 'c': ") && namesTheRefusal(c, message)) None + else Some(s"refused as [$message]") + case (Refused, Right(source)) => Some(s"created [$source], expected a refusal") + case (Typed(_) | NullValued, Right(_)) => None + case (NotDateArithmetic, Right(_)) => None + case (NotDateArithmetic, Left(message)) if !dateArithmeticRefusal(message) => None + case (other, Left(message)) => Some(s"refused as [$message], expected $other") + }).map(m => s"[${c.expression}] $m") + } + info( + s"date arithmetic, DDL venue: ${ddlPopulation.size} computed columns; mismatches: ${mismatches.size}" + ) + withClue(mismatches.take(30).mkString("\n", "\n", "\n")) { + mismatches shouldBe empty + } + } + + private def resolvedItem(sql: String): (SingleSearch, Field) = + Parser(sql) match { + case Right(ss: SingleSearch) => + val resolved = ss.update(Some(schema)) + resolved -> resolved.select.fields + .find(_.fieldAlias.exists(_.alias == "x")) + .getOrElse(fail(s"[$sql] no x item")) + case other => fail(s"[$sql] $other") + } + + "a cast over date arithmetic" should "keep the type it casts to" in { + Seq( + "(d - d)::TIMESTAMP" -> SQLTypes.Timestamp, + "(d + 1)::TIMESTAMP" -> SQLTypes.Timestamp, + "(ts - ts)::BIGINT" -> SQLTypes.BigInt, + "(d + 1.5)::DATE" -> SQLTypes.Date, + "(MAX(d) - MIN(d))::DOUBLE" -> SQLTypes.Double + ).foreach { case (expression, t) => + val sql = + if (expression.contains("MAX")) s"SELECT g, $expression AS x FROM ev GROUP BY g" + else s"SELECT $expression AS x FROM ev" + val (resolved, item) = resolvedItem(sql) + withClue(s"[$expression] ") { + resolved.validateResolved() shouldBe Right(()) + item.identifier.reportedType shouldBe t + } + } + } + + "INTERVAL arithmetic and DATEDIFF" should "keep the types they had" in { + Seq( + "d - INTERVAL 1 DAY" -> SQLTypes.Timestamp, + "ts + INTERVAL 2 HOUR" -> SQLTypes.Timestamp, + "DATEDIFF(d, d)" -> SQLTypes.Numeric, + "TIMESTAMPDIFF(DAY, d, ts)" -> SQLTypes.Numeric + ).foreach { case (expression, t) => + Parser(s"SELECT $expression AS x FROM ev") match { + case Right(ss: SingleSearch) => + val resolved = ss.update(Some(schema)) + withClue(s"[$expression] ") { + resolved.validateResolved() shouldBe Right(()) + resolved.select.fields.head.identifier.reportedType shouldBe t + } + case other => fail(s"[$expression] $other") + } + } + } + + "a date field elasticsql never declared" should + "be a TIMESTAMP everywhere, from the declaration seam on" in { + // DESCRIBE reads the column's data type: the seam types it TIMESTAMP (the lead's ruling of + // 2026-10-05), while a declared DATE keeps its declaration + schema.find("u").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) + schema.find("d").map(_.dataType) shouldBe Some(SQLTypes.Date) + resolvedItem("SELECT u AS x FROM ev")._2.identifier.reportedType shouldBe SQLTypes.Timestamp + resolvedItem("SELECT d + 1 AS x FROM ev")._2.identifier.reportedType shouldBe SQLTypes.Date + // date arithmetic: a number of days keeps its time of day, a difference is fractional days + resolvedItem("SELECT u + 1 AS x FROM ev")._2.identifier.reportedType shouldBe + SQLTypes.Timestamp + resolvedItem("SELECT u - d AS x FROM ev")._2.identifier.reportedType shouldBe SQLTypes.Double + resolvedItem( + "SELECT g, MAX(u) - MIN(u) AS x FROM ev GROUP BY g" + )._2.identifier.reportedType shouldBe + SQLTypes.Double + // and a refusal names it as DESCRIBE does + resolvedItem("SELECT u * 2 AS x FROM ev")._1 + .validateResolved() + .swap + .getOrElse("") should include( + "cannot be applied to TIMESTAMP and BIGINT" + ) + } + + "CASE, GREATEST and LEAST" should "report the least common supertype of their branches" in { + resolvedItem( + "SELECT CASE WHEN n > 0 THEN d ELSE d2 END AS x FROM ev" + )._2.identifier.reportedType shouldBe SQLTypes.Date + resolvedItem( + "SELECT CASE WHEN n > 0 THEN d ELSE ts END AS x FROM ev" + )._2.identifier.reportedType shouldBe SQLTypes.Timestamp + resolvedItem("SELECT GREATEST(n, x) AS x FROM ev")._2.identifier.reportedType shouldBe + SQLTypes.Double + resolvedItem("SELECT LEAST(n, 2) AS x FROM ev")._2.identifier.reportedType shouldBe + SQLTypes.BigInt + // GREATEST / LEAST over dates and timestamps (the lead's ruling of 2026-10-05): a DATE for + // dates, a TIMESTAMP when they mix -- and a date difference of one and a date is days + Seq( + "SELECT GREATEST(d, d2) AS x FROM ev" -> SQLTypes.Date, + "SELECT LEAST(d, ts) AS x FROM ev" -> SQLTypes.Timestamp, + "SELECT GREATEST(d, CURRENT_DATE) AS x FROM ev" -> SQLTypes.Date, + "SELECT g, GREATEST(MAX(d), MAX(d2)) AS x FROM ev GROUP BY g" -> SQLTypes.Date, + "SELECT GREATEST(d, d2) - d AS x FROM ev" -> SQLTypes.BigInt, + "SELECT LEAST(d, ts) - d AS x FROM ev" -> SQLTypes.Double + ).foreach { case (sql, t) => + val (resolved, item) = resolvedItem(sql) + withClue(s"[$sql] ") { + resolved.validateResolved() shouldBe Right(()) + item.identifier.reportedType shouldBe t + } + } + // numbers and dates do not mix, as in every SQL engine + resolvedItem("SELECT GREATEST(d, n) AS x FROM ev")._1 + .validateResolved() + .swap + .getOrElse("") should include( + "GREATEST requires numeric arguments, or date and timestamp arguments, but got DATE for d " + + "and INT for n" + ) + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/HavingAggregateResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/HavingAggregateResolutionSpec.scala index c29949b95..c15f814a0 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/HavingAggregateResolutionSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/HavingAggregateResolutionSpec.scala @@ -1,6 +1,8 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.SQLTypes import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -101,12 +103,32 @@ class HavingAggregateResolutionSpec extends AnyFlatSpec with Matchers { ).foreach { sql => withClue(s"[$sql] ") { Parser(sql).isRight shouldBe true + resolved(sql).validateResolved() shouldBe Right(()) } } - // A genuine mismatch stays loud, and names the ELEMENT types. - val msg = rejection("SELECT id FROM t GROUP BY id HAVING COUNT(x) IN ('a', 'b')") + // A genuine mismatch stays loud, and names the ELEMENT types -- once the column types are + // known (the lead's ruling of 2026-10-05: no TYPE refusal when the statement is parsed). + val sql = "SELECT id FROM t GROUP BY id HAVING COUNT(x) IN ('a', 'b')" + Parser(sql).isRight shouldBe true + val msg = + resolved(sql).validateResolved().swap.getOrElse(fail(s"expected a refusal for [$sql]")) msg should include("Type mismatch") msg should include("'BIGINT' is not compatible with 'VARCHAR'") - msg should not startWith Parser.InternalParseFailure } + + private val schema = SchemaTable( + "t", + columns = List( + Column("id", SQLTypes.Int), + Column("x", SQLTypes.Int), + Column("name", SQLTypes.Keyword), + Column("createdAt", SQLTypes.Timestamp) + ) + ) + + private def resolved(sql: String): SingleSearch = + Parser(sql) match { + case Right(single: SingleSearch) => single.update(Some(schema)) + case other => fail(s"[$sql] $other") + } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala index 7fc3bea04..f9258cd97 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala @@ -373,17 +373,42 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr * operation to types [ZonedDateTime] and [ZonedDateTime]`). This project's standing rule: a * parse fix must not upgrade a loud failure into a silent corruption β€” or, as here, into a shard * error naming neither the column nor the SQL. + * + * ⚠️ The arithmetic caller is no longer one of them: a DATE minus a TIMESTAMP is date + * arithmetic, which reconciles its operands by its own rules (`ArithmeticExpression`, the lead's + * ruling of 2026-10-04) and answers the fractional days between them -- the row below. */ - "the date-carrying allowance" should "not leak to callers that cannot reconcile" in { - forAll( - Table( - "sql", - "SELECT name FROM t WHERE CAST(d AS DATE) BETWEEN CAST('2025-01-01' AS DATE) " + - "AND CAST('2025-12-31T00:00:00Z' AS TIMESTAMP)", - "SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP)) AS x FROM t", - "SELECT CAST(d AS DATE) - CAST(ts AS TIMESTAMP) AS x FROM t" - ) - ) { sql => withClue(s"[$sql] ")(Parser(sql).isLeft shouldBe true) } + "the date-carrying allowance" should "reach BETWEEN and NULLIF, which now reconcile the pair" in { + // The ONE comparison rule (the lead's ruling of 2026-10-05): a DATE and a TIMESTAMP are + // comparable wherever a comparison is made, judged on the declared types. BETWEEN brings each + // bound to the operand's java.time type, as a single comparison does, and NULLIF compares the + // instants the two denote -- neither calls `isBefore` / `isEqual` across a `LocalDate` and a + // `ZonedDateTime`, which is what failed the shard. RUN in `DateArithmeticSpec`. + resolvedVerdict( + "SELECT name FROM t WHERE CAST(d AS DATE) BETWEEN CAST('2025-01-01' AS DATE) " + + "AND CAST('2025-12-31T00:00:00Z' AS TIMESTAMP)" + ) shouldBe Right(()) + resolvedVerdict("SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP)) AS x FROM t") shouldBe + Right(()) + // a TIME is no date-carrying temporal: still refused once the types are known, never at parse + val time = "SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIME)) AS x FROM t" + Parser(time).isRight shouldBe true + resolvedVerdict(time).isLeft shouldBe true + } + + /** The post-resolution verdict of a statement: parsed, its schema attached, `validateResolved`. + */ + private def resolvedVerdict(sql: String): Either[String, Unit] = Parser(sql) match { + case Right(ss: SingleSearch) => ss.update(Some(schema)).validateResolved() + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + "a DATE minus a TIMESTAMP" should "be date arithmetic, the fractional days between them" in { + Parser("SELECT CAST(d AS DATE) - CAST(ts AS TIMESTAMP) AS x FROM t") match { + case Right(ss: SingleSearch) => + ss.select.fields.head.identifier.reportedType shouldBe SQLTypes.Double + case other => fail(s"$other") + } } "two operands cast to different date-carrying types" should "be comparable" in { @@ -403,11 +428,12 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr withClue(s"->\n$emitted\n")(emitted should include("atStartOfDay")) } - /** The other half of the same line: a TIME cast against a date-carrying cast is refused at PARSE, - * where both types are known without a schema. (A TIME against a bare COLUMN is only knowable - * after resolution β€” that is what `validateResolved` below is for.) + /** The other half of the same line: a TIME cast against a date-carrying cast is refused -- once + * the column types are known, like every TYPE rule (the lead's ruling of 2026-10-05); it used to + * be refused at PARSE, where both types are known without a schema. (A TIME against a bare + * COLUMN is only knowable after resolution β€” that is what `validateResolved` below is for.) */ - "a TIME cast against a date-carrying cast" should "be refused at parse" in { + "a TIME cast against a date-carrying cast" should "be refused once the types are known" in { forAll( Table( "sql", @@ -416,12 +442,15 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr "SELECT name FROM t WHERE CAST(ts AS TIMESTAMP) > CAST(ts AS TIME)" ) ) { sql => - withClue(s"[$sql] ")(Parser(sql).isLeft shouldBe true) + withClue(s"[$sql] ") { + Parser(sql).isRight shouldBe true + resolvedVerdict(sql).isLeft shouldBe true + } } // and TIME against TIME stays legal - Parser( + resolvedVerdict( "SELECT name FROM t WHERE CAST(ts AS TIME) > CAST('07:00:00' AS TIME)" - ).isRight shouldBe true + ) shouldBe Right(()) } /** πŸ”΄ `Expression.validate()` compares the types the two operands END UP with, not the ones their @@ -436,7 +465,9 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr * It is observable ONLY against a schema-resolved statement, which is what this asserts and why * the corpus could not falsify it. */ - "validate() on a schema-resolved statement" should "read the chain's types, not the column's" in { + "the comparison TYPE rule on a schema-resolved statement" should "read the chain's types, not the column's" in { + // `typeErrors`, not `validate()`: the TYPE rule moved off `validate()` (which checks syntax and + // structure only) to the post-resolution seam -- the lead's ruling of 2026-10-05. def criteriaOf(sql: String): Criteria = Parser(sql) match { case Right(ss: SingleSearch) => ss.update(Some(schema)).where.flatMap(_.criteria).getOrElse(fail(s"[$sql] no criteria")) @@ -446,11 +477,11 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr withClue("TIME = TIME ") { criteriaOf( "SELECT name FROM t WHERE CAST(ts AS TIME) = CAST('07:00:00' AS TIME)" - ).validate() shouldBe Right(()) + ).typeErrors shouldBe empty } // a TIME against a date-carrying column -- this was ACCEPTED (TIMESTAMP vs TIMESTAMP) withClue("TIME > TIMESTAMP ") { - criteriaOf("SELECT name FROM t WHERE CAST(ts AS TIME) > ts").validate().isLeft shouldBe true + criteriaOf("SELECT name FROM t WHERE CAST(ts AS TIME) > ts").typeErrors should not be empty } // and the date family stays comparable, resolved or not forAll( @@ -462,7 +493,12 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr "SELECT name FROM t WHERE YEAR(d) = 2025", "SELECT name FROM t WHERE UPPER(name) = 'A'" ) - ) { sql => withClue(s"[$sql] ")(criteriaOf(sql).validate() shouldBe Right(())) } + ) { sql => + withClue(s"[$sql] ") { + criteriaOf(sql).validate() shouldBe Right(()) + criteriaOf(sql).typeErrors shouldBe empty + } + } } "a TIME operand" should "be refused against a date-carrying one, and accepted against a TIME" in { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala index 32752b069..551724a50 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala @@ -657,27 +657,34 @@ class NullIfOperandSpec extends AnyFlatSpec with Matchers with TableDrivenProper * `TemporalLiterals` -- the SAME machinery a WHERE clause asks, against the column's own mapping * `format` -- so the engine has ONE date-literal grammar, not two. */ - it should "ask the temporal-literal resolver about a string literal over a date column" in { + it should "read a well-formed date literal as that date, and ask the resolver about the rest" in { + // The ONE comparison rule (the lead's ruling of 2026-10-05): a string literal that spells a + // date or a timestamp, compared with a temporal, IS that temporal -- read here, as the instant + // it denotes, so no per-document `String -> temporal` coercion is needed in any venue. That is + // the half #382 recorded as its own story, and it is accepted now in every venue. forAll( Table( - // πŸ”΄ AMENDED by the #382 review: EVERY row is refused now, and the column records which - // JUDGE refused it, because that is what the wording has to come from. - // - // A temporal operand against a text one cannot be EMITTED -- the pair folds to VARCHAR, - // misses both typed arms and lands on `==`, which is silently `false` for every document. - // So a well-formed literal is refused too, with a message that says the engine cannot do - // it yet rather than pretending the SQL is wrong. A MALFORMED literal is still refused by - // the resolver, which names the literal and the field -- a strictly better message, and - // the reason the resolver stays reachable and goes first. - // - // Making the well-formed half WORK is its own story (a `String -> temporal` coercion per - // subtype, three venues, four ES majors; and `now-1d` is Elasticsearch date math, which - // has no Painless equivalent at all and must stay refused by name). + "expr", + "NULLIF(d, '2025-01-01')", + "NULLIF(d, '2025-01-01 10:00:00')", + "NULLIF(ts, '2025-01-01T10:00:00Z')", + "NULLIF('2025-01-01', d)" + ) + ) { expr => + everyVenueVerdict(expr, "DATE").foreach { case (venue, verdict) => + withClue(s"[$venue] [$expr] -> $verdict ") { + verdict shouldBe Right(()) + } + } + } + forAll( + Table( + // Every other literal beside a date is REFUSED, and the column records which JUDGE + // refused it, because that is what the wording has to come from. Elasticsearch date math + // (`now-1d`) and an epoch number spell no date: the engine refuses them by name. A + // MALFORMED literal is refused by the resolver, which names the literal and the field -- + // a strictly better message, and the reason the resolver stays reachable and goes first. ("expr", "judge"), - ("NULLIF(d, '2025-01-01')", "engine"), - ("NULLIF(d, '2025-01-01 10:00:00')", "engine"), - ("NULLIF(ts, '2025-01-01T10:00:00Z')", "engine"), - ("NULLIF('2025-01-01', d)", "engine"), ("NULLIF(d, 'now-1d')", "engine"), ("NULLIF(d, '1735689600000')", "engine"), ("NULLIF(d, 'x')", "resolver"), diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessResidualsSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessResidualsSpec.scala index ccdb6fb32..396a16425 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessResidualsSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessResidualsSpec.scala @@ -316,8 +316,11 @@ class PainlessResidualsSpec extends AnyFlatSpec with Matchers with TableDrivenPr // The single-chain processor emission must not move: one column, one chain, one parameter. processorOf("CREATE TABLE t (d DATE, c INTEGER SCRIPT AS (YEAR(d)))") shouldBe "def param1 = (ctx.d instanceof String ? " + + "(ctx.d.length() > 10 ? ZonedDateTime.parse((ctx.d).replace(\" \", \"T\"), " + + "DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z'))" + + ".truncatedTo(ChronoUnit.DAYS) : " + "LocalDate.parse((ctx.d).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" + - ".atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(ctx.d).atZone(ZoneId.of('Z')))" + + ".atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(ctx.d).atZone(ZoneId.of('Z')))" + ".get(ChronoField.YEAR); ctx.c = param1" } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessTargetSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessTargetSpec.scala new file mode 100644 index 000000000..61028a7b4 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessTargetSpec.scala @@ -0,0 +1,126 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType, PainlessTarget} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** A comparison between a `date` doc value and another date renders for the Elasticsearch major the + * client module gives the bridge (`PainlessTarget`). + * + * Elasticsearch 6.8 hands a query script a `date` doc value as a `JodaCompatibleZonedDateTime`, + * whose `isBefore` / `isAfter` / `isEqual` against a genuine `ZonedDateTime` fail with + * `Class.cast`; so on 6.8 each doc-reading operand of such a comparison is normalised to UTC + * first. From 7 on the doc value IS a `ZonedDateTime`: the scripts of those majors -- and of a + * rendering no client module gave a major to -- stay byte-for-byte what they were, which the byte + * pins of `ParameterIdentitySpec`, `PredicateTransformSurvivalSpec` and `PainlessResidualsSpec` + * hold for the default target. + */ +class PainlessTargetSpec extends AnyFlatSpec with Matchers { + + private val schema = SchemaTable( + "t", + columns = List( + Column("d", SQLTypes.Date), + Column("d2", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("n", SQLTypes.Int) + ) + ) + + private val normalisation = ".withZoneSameInstant(ZoneId.of('Z'))" + + private def occurrences(s: String): Int = + s.sliding(normalisation.length).count(_ == normalisation) + + private def predicateOf( + sql: String, + target: PainlessTarget, + contextType: PainlessContextType = PainlessContextType.Query + ): String = Parser(sql) match { + case Right(ss: SingleSearch) => + val ctx = PainlessContext(contextType, target) + val body = ss + .update(Some(schema)) + .where + .flatMap(_.criteria) + .map(_.painless(Some(ctx))) + .getOrElse(fail(s"[$sql] no WHERE criteria")) + s"$ctx$body" + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + /** Item 1's comparison shapes: a doc value against a function, the clock, a literal or date + * arithmetic, in both directions -- each with how many of its operands read a document. + */ + private val shapes: Seq[(String, Int)] = Seq( + "SELECT n FROM t WHERE CAST(d AS TIMESTAMP) >= ts" -> 2, + "SELECT n FROM t WHERE LAST_DAY(d) = ts" -> 2, + "SELECT n FROM t WHERE CAST(ts AS DATE) = d" -> 2, + "SELECT n FROM t WHERE DATE_ADD(d, INTERVAL 1 DAY) > ts" -> 2, + "SELECT n FROM t WHERE DATE_ADD(d, INTERVAL 1 DAY) = d2" -> 2, + "SELECT n FROM t WHERE CURRENT_DATE > d" -> 1, + "SELECT n FROM t WHERE COALESCE(d, d2) > CAST('2024-01-01' AS DATE)" -> 1, + "SELECT n FROM t WHERE d2 = d + 1" -> 2, + "SELECT n FROM t WHERE d > d2 - 1" -> 2, + "SELECT n FROM t WHERE d >= CAST(ts AS DATE) - 1" -> 2 + ) + + "a comparison of a date doc value" should + "render as it always did for Elasticsearch 7, 8 and 9 and for no major at all" in { + shapes.foreach { case (sql, _) => + withClue(s"[$sql] ") { + val default = predicateOf(sql, PainlessTarget.Default) + Seq(7, 8, 9).foreach(major => predicateOf(sql, PainlessTarget(major)) shouldBe default) + } + } + } + + it should "normalise every operand that reads a document to UTC on Elasticsearch 6.8, and nothing else" in { + shapes.foreach { case (sql, documentOperands) => + withClue(s"[$sql] ") { + val es6 = predicateOf(sql, PainlessTarget(6)) + val default = predicateOf(sql, PainlessTarget.Default) + // one normalisation per operand that reads a document (date arithmetic, a cast, may + // already carry the same UTC conversion for reasons of their own: the DIFFERENCE is what + // the target adds) -- and nothing else differs from what the other majors render + occurrences(es6) - occurrences(default) shouldBe documentOperands + es6.replace(normalisation, "") shouldBe default.replace(normalisation, "") + } + } + } + + it should "not normalise outside a query: an ingest processor parses its own operands" in { + shapes.foreach { case (sql, _) => + withClue(s"[$sql] ") { + predicateOf(sql, PainlessTarget(6), PainlessContextType.Processor) shouldBe + predicateOf(sql, PainlessTarget.Default, PainlessContextType.Processor) + } + } + } + + "PainlessTarget" should "tell the major that hands Joda-compatible dates" in { + PainlessTarget(6).jodaCompatibleDates shouldBe true + Seq(7, 8, 9).foreach(major => PainlessTarget(major).jodaCompatibleDates shouldBe false) + PainlessTarget.Default.jodaCompatibleDates shouldBe false + PainlessContext().target shouldBe PainlessTarget.Default + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala index e1a4ad85d..e194f2119 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala @@ -467,19 +467,20 @@ class TemporalLiteralsSpec extends AnyFlatSpec with Matchers { |}},"settings":{"index":{"number_of_shards":"1","number_of_replicas":"0"}}}}""".stripMargin val table = Index("events", json).schema - // πŸ”΄ DATE β€” the DECLARED type, preserved. An Elasticsearch `date` field IS a millisecond - // timestamp at runtime, but that is a different fact and it belongs to - // `SQLTypeUtils.runtimeType`, read via `GenericIdentifier.baseType`. Story 21.5 briefly mapped - // it here instead and the spurious `_meta` diff that caused cascaded into an unparseable ES 6.8 - // ALTER and a materialized-view slowdown. `Column.dataType` stays what the user wrote. - table.find("event_ts").map(_.dataType) shouldBe Some(SQLTypes.Date) + // πŸ”΄ TIMESTAMP β€” this mapping carries no elasticsql declaration (no `_meta` data type), and a + // `date` field nobody declared holds instants: the lead's ruling of 2026-10-05, applied at the + // declaration seam (`schema.IndexField`). A DECLARED type is preserved verbatim, a DATE + // included (`ColumnMetaDiffSpec`): story 21.5 mapped every `date` to a timestamp in + // `SQLTypes.apply` instead, which rewrote declarations, and the spurious `_meta` diff that + // caused cascaded into an unparseable ES 6.8 ALTER and a materialized-view slowdown. + table.find("event_ts").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) table.find("fmt_ts").flatMap(_.options.get("format")) shouldBe Some( StringValue("yyyy-MM-dd HH:mm:ss") ) // `date_nanos` is not a SQL type the mapping resolves to: it comes out as ANY, so it is never // a candidate. Pinned on purpose (AC 4: excluded, not covered). table.find("ts_nanos").map(_.dataType) shouldBe Some(SQLTypes.Any) - table.find("items.ts").map(_.dataType) shouldBe Some(SQLTypes.Date) + table.find("items.ts").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) whereSql( resolved("SELECT id FROM events WHERE event_ts >= '2026-06-04 00:00:00'", table) @@ -504,7 +505,7 @@ class TemporalLiteralsSpec extends AnyFlatSpec with Matchers { |"settings":{"index":{"number_of_shards":"1","number_of_replicas":"0"}}}""".stripMargin val single = Index("events_alias", s"""{"events":$events}""") single.name shouldBe "events_alias" - single.schema.find("event_ts").map(_.dataType) shouldBe Some(SQLTypes.Date) + single.schema.find("event_ts").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) whereSql( resolved("SELECT id FROM events_alias WHERE event_ts >= '2026-06-04 00:00:00'", single.schema) ) shouldBe "WHERE event_ts >= '2026-06-04T00:00:00'" @@ -519,7 +520,7 @@ class TemporalLiteralsSpec extends AnyFlatSpec with Matchers { // a concrete index keyed by its own name is untouched by the rule Index("events", s"""{"events":$events}""").schema.find("event_ts").map(_.dataType) shouldBe - Some(SQLTypes.Date) + Some(SQLTypes.Timestamp) // R4-18: a member WITHOUT mappings still counts -- otherwise a two-index alias would resolve // silently to the other member (the silent-wrong-answer mode this story removes) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala new file mode 100644 index 000000000..b9780fa9e --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala @@ -0,0 +1,206 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.prop.TableDrivenPropertyChecks + +/** No TYPE refusal before the column types are known (the lead's ruling of 2026-10-05). + * + * Every type rule `validate()` used to apply when the statement was PARSED -- where every column + * is `Any` -- is asked once the schema is attached (`SingleSearch.validateResolved`), on the real + * types. So: + * - parsing refuses no type, whatever the statement (with no schema, nothing is refused on types + * at all, and Elasticsearch answers); + * - each moved rule still refuses its TRUE mismatches, by name, once the types are known -- one + * row per rule below; + * - and the statements the parse-time rules refused although they are well typed now run: they + * are typed from the real columns. + */ +class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDrivenPropertyChecks { + + // `SchemaTable`, because `TableDrivenPropertyChecks` shadows the name `Table`. + private val schema = SchemaTable( + "t", + columns = List( + Column("id", SQLTypes.Keyword), + Column("g", SQLTypes.Keyword), + Column("s", SQLTypes.Keyword), + Column("n", SQLTypes.Int), + Column("x", SQLTypes.Double), + Column("d", SQLTypes.Date), + Column("d2", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("b", SQLTypes.Boolean) + ) + ) + + private def parsed(sql: String): SingleSearch = Parser(sql) match { + case Right(single: SingleSearch) => single + case other => fail(s"[$sql] parsed as $other: no TYPE is refused at parse") + } + + private def resolved(sql: String): Either[String, Unit] = + parsed(sql).update(Some(schema)).validateResolved() + + "every TYPE rule moved off validate()" should + "refuse nothing at parse, and refuse its true mismatches by name once the types are known" in { + forAll( + Table( + ("rule", "sql", "refusal"), + // a comparison: its operands' match, TIME against a date + ( + "comparison", + "SELECT id FROM t WHERE n + 1 > 'abc'", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ("comparison", "SELECT id FROM t WHERE s = n", "'KEYWORD' is not compatible with 'INT'"), + ("comparison (TIME)", "SELECT id FROM t WHERE CAST(ts AS TIME) > ts", "a TIME value"), + // IN: the operand against the list's element type + ( + "IN", + "SELECT id FROM t WHERE CAST(n AS INT) IN ('a', 'b')", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "IN (HAVING)", + "SELECT g FROM t GROUP BY g HAVING COUNT(n) IN ('a', 'b')", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + // BETWEEN: its bounds, and its operand against them + ( + "BETWEEN bounds", + "SELECT id FROM t WHERE n BETWEEN 1 AND 'a'", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + ( + "BETWEEN operand", + "SELECT id FROM t WHERE CAST(s AS INT) BETWEEN 'a' AND 'b'", + "output 'INT' is not compatible with input 'VARCHAR'" + ), + // NULLIF: #382's rule, and the matching rule over the types the statement fixes + ("NULLIF (#382)", "SELECT NULLIF(s, 1) AS v FROM t", "compares KEYWORD with BIGINT"), + // (a DATE against a TIMESTAMP is comparable, the ONE comparison rule: TIME is not) + ( + "NULLIF (match)", + "SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIME)) AS v FROM t", + "output 'DATE' is not compatible with input 'TIME'" + ), + // CASE: boolean conditions, and a simple CASE's WHEN values against its operand + ( + "CASE WHEN", + "SELECT CASE WHEN n THEN 1 ELSE 0 END AS v FROM t", + "CASE WHEN conditions must be of type BOOLEAN" + ), + ( + "CASE WHEN", + "SELECT CASE n WHEN 'a' THEN 1 ELSE 0 END AS v FROM t", + "CASE WHEN conditions must be of the same type as the expression" + ), + // GREATEST / LEAST: numbers, or dates and timestamps -- one kind + ( + "GREATEST", + "SELECT GREATEST('a', 'b') AS v FROM t", + "GREATEST requires numeric arguments, or date and timestamp arguments, but got " + + "VARCHAR for 'a'" + ), + ( + "LEAST", + "SELECT LEAST(d, n) AS v FROM t", + "LEAST requires numeric arguments, or date and timestamp arguments, but got DATE for d " + + "and INT for n" + ), + // an arithmetic: date arithmetic's rules, and any other's operand match + ( + "date arithmetic", + "SELECT CURRENT_DATE * 2 AS v FROM t", + "operator * cannot be applied to DATE and BIGINT" + ), + ( + "date arithmetic (column)", + "SELECT d + d2 AS v FROM t", + "operator + cannot be applied to DATE and DATE" + ), + ( + "arithmetic", + "SELECT n + s AS v FROM t", + "output 'INT' is not compatible with input 'KEYWORD'" + ), + ("arithmetic (literals)", "SELECT 1 + 'abc' AS v FROM t", "input 'VARCHAR'") + ) + ) { (rule, sql, refusal) => + withClue(s"[$rule] [$sql] ") { + // nothing is refused at parse, and with no schema nothing is refused on types: the + // statement parses, and nothing else asks + parsed(sql) + resolved(sql).swap.getOrElse(fail("accepted once the types are known")) should include( + refusal + ) + } + } + } + + "the statements the parse-time rules refused although they are well typed" should + "be accepted once the column types are known" in { + forAll( + Table( + "sql", + "SELECT id FROM t WHERE CURRENT_DATE - d > 30", + "SELECT id FROM t WHERE d + 7 > CURRENT_DATE", + "SELECT id FROM t WHERE NOW() - ts > 1", + "SELECT id FROM t WHERE CAST(ts AS DATE) - d = 0", + "SELECT id FROM t WHERE d + 1 = CAST('2024-02-01' AS DATE)", + "SELECT id FROM t WHERE d + 1 > '2024-01-01'", + "SELECT g, MAX(d) + 1 AS v FROM t GROUP BY g HAVING v > CAST('2024-01-01' AS DATE)", + // a BOOLEAN column is a boolean condition -- refused at parse, where it was `Any` + "SELECT CASE WHEN b THEN 1 ELSE 0 END AS v FROM t", + // GREATEST / LEAST over dates and timestamps (the lead's ruling of 2026-10-05) + "SELECT GREATEST(d, d2) AS v FROM t", + "SELECT LEAST(d, ts, CURRENT_DATE) AS v FROM t", + "SELECT id FROM t WHERE GREATEST(d, d2) > CAST('2024-02-01' AS DATE)", + "SELECT g, GREATEST(MAX(d), MAX(d2)) AS v FROM t GROUP BY g" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + } + + "a literal compared with a bare column" should + "be left to Elasticsearch, which coerces it against the column's mapping" in { + forAll( + Table( + "sql", + "SELECT id FROM t WHERE n = '5'", + "SELECT id FROM t WHERE d >= '2024-01-01'", + "SELECT id FROM t WHERE s = 5", + "SELECT id FROM t WHERE n IN ('1', '2')", + "SELECT id FROM t WHERE n BETWEEN '1' AND '9'" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + } + + "a string literal compared with a temporal expression" should + "be the date or timestamp it spells, and refused when it spells none" in { + resolved("SELECT id FROM t WHERE d + 1 > '2024-01-01T10:00:00Z'") shouldBe Right(()) + resolved("SELECT id FROM t WHERE DATE_TRUNC(d, MONTH) >= '2024-01-01'") shouldBe Right(()) + resolved("SELECT id FROM t WHERE d + 1 > 'tomorrow'").swap.getOrElse("") should include( + "is not compatible with 'VARCHAR'" + ) + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala index 3d3978125..3de5a7b61 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala @@ -99,14 +99,18 @@ class DdlScriptSchemaSpec extends AnyFlatSpec with Matchers { // answered that question one way, hard-coded and untested, and was wrong for every documented // example. Verified as an ingest pipeline on ES 6.8.23, 7.17.29, 8.18.3 and 9.0.3 against // {"created":"2025-01-10"}, {"created":"2025/01/10"} and {"created":1736467200000}: all three - // store `y: 2025`. + // store `y: 2025`. A string longer than a date is a date-time (Elasticsearch accepts one into a + // DATE column): its UTC day, where the strict date parse threw and stored nothing. createSource( "CREATE TABLE t (created DATE, y INTEGER SCRIPT AS (YEAR(created)))", "y" ) shouldBe "def param1 = (ctx.created instanceof String ? " + + """(ctx.created.length() > 10 ? ZonedDateTime.parse((ctx.created).replace(" ", "T"), """ + + "DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z'))" + + ".truncatedTo(ChronoUnit.DAYS) : " + """LocalDate.parse((ctx.created).replace("/", "-"), """ + - """DateTimeFormatter.ofPattern("yyyy-MM-dd")).atStartOfDay(ZoneId.of('Z')) """ + + """DateTimeFormatter.ofPattern("yyyy-MM-dd")).atStartOfDay(ZoneId.of('Z'))) """ + ": Instant.ofEpochMilli(ctx.created).atZone(ZoneId.of('Z'))).get(ChronoField.YEAR); " + "ctx.y = param1" } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/IngestTemporalSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/IngestTemporalSpec.scala index 1dac75888..71399bd17 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/IngestTemporalSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/IngestTemporalSpec.scala @@ -157,15 +157,20 @@ class IngestTemporalSpec extends AnyFlatSpec with Matchers with TableDrivenPrope // CURRENT_DATE)`: the whole years elapsed from `birthdate` to the start of `CURRENT_DATE` // (UTC), which the processor holds as the current instant and reads as a date. Run as an // ingest pipeline on ES 8.18.3, `{"birthdate":"1990-05-20"}` and `{"birthdate":643161600000}` - // both store that age. + // both store that age. A date-time string in the DATE column (Elasticsearch accepts one) is + // read as the UTC day of the instant it spells -- the arm `DateArithmeticSpec`'s computed + // columns execute -- where the strict date parse threw and stored nothing. source( "CREATE TABLE t (birthdate DATE, age INTEGER SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)))", "age" ) shouldBe "def param1 = ctx.birthdate; " + "def param2 = (param1 instanceof String ? " + + """(param1.length() > 10 ? ZonedDateTime.parse((param1).replace(" ", "T"), """ + + "DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z'))" + + ".truncatedTo(ChronoUnit.DAYS) : " + """LocalDate.parse((param1).replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd"))""" + - ".atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); " + + ".atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); " + "def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), " + "ZoneId.of('Z')); " + "def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); " + diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/MergedSchemaSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/MergedSchemaSpec.scala new file mode 100644 index 000000000..0e20ae572 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/MergedSchemaSpec.scala @@ -0,0 +1,179 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.schema + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.StringValue +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.SingleSearch +import app.softnetwork.elastic.sql.`type`.SQLTypes +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.collection.immutable.ListMap + +/** The schema an index PATTERN or a comma list answers with (`mergedSchema`): the union of the + * matched indices' columns, merged FIELD BY FIELD -- a field the indices map alike keeps its type, + * a field they map differently is left untyped (`Any`), so nothing is refused on it and + * Elasticsearch answers for it, while every other field is typed as over a single index. + */ +class MergedSchemaSpec extends AnyFlatSpec with Matchers { + + private def index(name: String, properties: String): (String, Table) = + name -> Index(name, s"""{"mappings":{"properties":{$properties}},"settings":{}}""").schema + + "indices that agree" should "answer the union of their columns" in { + val (merged, disagreements) = mergedSchema( + "logs-*", + Seq( + index("logs-1", """"ts":{"type":"date"},"g":{"type":"keyword"}"""), + index("logs-2", """"ts":{"type":"date"},"n":{"type":"long"}""") + ) + ) + disagreements shouldBe empty + merged.name shouldBe "logs-*" + merged.columns.map(c => c.name -> c.dataType) should contain theSameElementsAs Seq( + "g" -> SQLTypes.Keyword, + "n" -> SQLTypes.BigInt, + // a `date` nobody declared is a TIMESTAMP at the declaration seam, and stays one merged + "ts" -> SQLTypes.Timestamp + ) + } + + it should "agree on a TIMESTAMP one index declares and another holds undeclared" in { + val declared = Index( + "logs-1", + """{"mappings":{"_meta":{"columns":{"ts":{"data_type":"TIMESTAMP","not_null":"false"}}},""" + + """"properties":{"ts":{"type":"date"}}},"settings":{}}""" + ).schema + val (merged, disagreements) = + mergedSchema("logs-*", Seq("logs-1" -> declared, index("logs-2", """"ts":{"type":"date"}"""))) + disagreements shouldBe empty + merged.find("ts").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) + } + + "indices that disagree on a field" should "leave that field alone untyped, and type every other" in { + val (merged, disagreements) = mergedSchema( + "logs-*", + Seq( + index("logs-1", """"ts":{"type":"date"},"c":{"type":"date"},"g":{"type":"keyword"}"""), + index("logs-2", """"ts":{"type":"date"},"c":{"type":"keyword"},"n":{"type":"long"}""") + ) + ) + disagreements shouldBe Seq("'c' (KEYWORD, TIMESTAMP)") + merged.columns.map(c => c.name -> c.dataType) should contain theSameElementsAs Seq( + "c" -> SQLTypes.Any, + "g" -> SQLTypes.Keyword, + "n" -> SQLTypes.BigInt, + "ts" -> SQLTypes.Timestamp + ) + } + + it should "count a date's format as part of its mapping: a temporal literal is read against it" in { + val withFormat = Table( + "logs-2", + columns = List( + Column( + "ts", + SQLTypes.Timestamp, + options = ListMap("format" -> StringValue("epoch_millis")) + ), + Column("g", SQLTypes.Keyword) + ) + ) + val (merged, disagreements) = mergedSchema( + "logs-*", + Seq( + index("logs-1", """"ts":{"type":"date"},"g":{"type":"keyword"}"""), + "logs-2" -> withFormat + ) + ) + disagreements shouldBe Seq("'ts' (TIMESTAMP)") + merged.find("ts").map(_.dataType) shouldBe Some(SQLTypes.Any) + merged.find("ts").flatMap(_.options.get("format")) shouldBe None + merged.find("g").map(_.dataType) shouldBe Some(SQLTypes.Keyword) + } + + it should "merge sub-fields by the same rule, under their parent" in { + val (merged, disagreements) = mergedSchema( + "logs-*", + Seq( + index( + "logs-1", + """"s":{"type":"text","fields":{"raw":{"type":"keyword"},"n":{"type":"long"}}}""" + ), + index( + "logs-2", + """"s":{"type":"text","fields":{"raw":{"type":"keyword"},"n":{"type":"keyword"},"w":{"type":"keyword"}}}""" + ) + ) + ) + disagreements shouldBe Seq("'s.n' (BIGINT, KEYWORD)") + merged.find("s").map(_.dataType) shouldBe Some(SQLTypes.Text) + merged.find("s.raw").map(_.dataType) shouldBe Some(SQLTypes.Keyword) + merged.find("s.w").map(_.dataType) shouldBe Some(SQLTypes.Keyword) + merged.find("s.n").map(_.dataType) shouldBe Some(SQLTypes.Any) + } + + /** The pattern of `DateArithmeticSpec`: two indices that agree on every field but `c`. */ + private lazy val pattern: Schema = + mergedSchema( + "ev_*", + Seq( + index( + "ev_a", + """"g":{"type":"keyword"},"d":{"type":"date"},"d2":{"type":"date"},"ts":{"type":"date"},"n":{"type":"integer"},"c":{"type":"date"}""" + ), + index("ev_b", """"g":{"type":"keyword"},"d":{"type":"date"},"c":{"type":"keyword"}""") + ) + )._1 + + private def resolved(sql: String): SingleSearch = + Parser(sql) match { + case Right(ss: SingleSearch) => ss.update(Some(pattern)) + case other => fail(s"[$sql] $other") + } + + "a statement over a pattern with one conflicting field" should + "type every field the indices agree on, and refuse nothing on the conflicting one" in { + // the fields they agree on: the type rules apply as over a single index (`d` is a `date` + // elasticsql never declared -- a TIMESTAMP -- so a difference of two is fractional days) + resolved("SELECT g, MAX(d) - MIN(d) AS x FROM ev_* GROUP BY g").select.fields + .find(_.fieldAlias.exists(_.alias == "x")) + .map(_.identifier.reportedType) shouldBe Some(SQLTypes.Double) + resolved("SELECT d * 2 AS x FROM ev_*").validateResolved().swap.getOrElse("") should include( + "operator * cannot be applied to TIMESTAMP and BIGINT" + ) + resolved("SELECT n + g AS x FROM ev_*").validateResolved().isLeft shouldBe true + // the conflicting field: no type, so no type rule refuses it -- Elasticsearch answers + Seq( + "SELECT c * 2 AS x FROM ev_*", + "SELECT c + d AS x FROM ev_*", + "SELECT n FROM ev_* WHERE c = 1", + "SELECT n FROM ev_* WHERE c + 1 > CURRENT_DATE", + "SELECT g, MAX(c) - MIN(c) AS x FROM ev_* GROUP BY g" + ).foreach { sql => + withClue(s"[$sql] ") { + resolved(sql).validateResolved() shouldBe Right(()) + } + } + resolved( + "SELECT c + 1 AS x FROM ev_*" + ).select.fields.head.identifier.reportedType.isUnknown shouldBe + true + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala new file mode 100644 index 000000000..f86c667db --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala @@ -0,0 +1,1615 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.{Sink, Source} +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ + ElasticFailure, + ElasticSuccess, + QueryRows, + QueryStream, + QueryStructured +} +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.persistence.generateUUID +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.time.temporal.ChronoUnit +import java.time.{Instant, LocalDate, LocalDateTime, OffsetDateTime, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.language.implicitConversions +import scala.util.Try + +/** Date arithmetic EXECUTES and answers what SQL engines answer, in every venue a script runs in: a + * row SELECT, a WHERE, a per-group SELECT (`bucket_script`), a HAVING over its alias + * (`bucket_selector`) and a computed column (`SCRIPT AS`, an ingest processor) -- the STORED + * value. + * + * The rules (the lead's ruling of 2026-10-04): DATE - DATE is the calendar days between the two; a + * difference with a TIMESTAMP on either side is the milliseconds between, in days; a DATE moved by + * a whole number of days is a DATE, by a fractional one an instant whose fraction is a time of + * day; a TIMESTAMP moved by any number of days is a TIMESTAMP; a NULL operand gives NULL; and the + * other combinations are refused by name -- once the column types are known, before anything runs + * (the lead's ruling of 2026-10-05: no type is refused when a statement is parsed). + * + * Every expected value is computed HERE, with `java.time`, from the fixture -- never read from the + * engine. The fixture holds a group with two dates, groups with one, and a non-empty group where + * the date columns are absent; DATE and TIMESTAMP columns, the timestamps carrying a time of day; + * negative differences; January 31 to February 1 (1 day) and 2024-02-28 to 2024-03-01 (2 days, + * across a leap day); dates before 1970; dates written as epoch-millisecond NUMBERS; and rows + * where an operand is NULL. The computed-column table also stores a DATE column written as a + * date-TIME string, which Elasticsearch accepts. + * + * Also measured here, each in its own test: an index created by a bulk load with no declaration (a + * `date` keeps its time), an index PATTERN (it answers like its index), the request clock in a + * per-group calculation, casts over arithmetic (NULL-safe, applied), and every type rule that + * waits for the column types (refused by name, never at parse; with no schema, never by core). + */ +trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + private val table = "date_arithmetic" + + private val computedTable = "date_arithmetic_computed" + + private val rawIndex = "date_arithmetic_raw" + + private val patternA = "date_arithmetic_pat_a" + + private val patternB = "date_arithmetic_pat_b" + + private val disagreeA = "date_arithmetic_dis_a" + + private val disagreeB = "date_arithmetic_dis_b" + + /** The documents of [[table]] with every `d` written as a date-TIME string of its UTC day. */ + private val dateTimeTable = "date_arithmetic_dt" + + /** The comparison rule's fixture (`"the ONE comparison rule"`). */ + private val comparisonTable = "date_arithmetic_cmp" + + /** An index created by a bulk load, then altered through elasticsql. */ + private val alteredRawIndex = "date_arithmetic_raw_altered" + + /** A table elasticsql created, to which a bulk load later adds a `date` field. */ + private val grownTable = "date_arithmetic_grown" + + /** The indices of the pattern whose cached schema DDL must refresh. */ + private val refreshedA = "date_arithmetic_rf_a" + private val refreshedB = "date_arithmetic_rf_b" + + private val DayMillis: Long = 86400000L + + /** One document: `None` is a column the document does not carry. + * + * @param epoch + * the dates and timestamps are written as epoch-millisecond NUMBERS + * @param dText + * `d` is written as this text -- a date-time string in a DATE column -- while `d` is the UTC + * day it falls in + */ + private final case class Row( + id: String, + g: String, + d: Option[LocalDate], + d2: Option[LocalDate], + ts: Option[Instant], + ts2: Option[Instant], + n: Option[Int], + x: Option[Double], + epoch: Boolean = false, + dText: Option[String] = None + ) + + private def date(s: String): Option[LocalDate] = Some(LocalDate.parse(s)) + private def instant(s: String): Option[Instant] = Some(Instant.parse(s)) + + private val rows: Seq[Row] = Seq( + Row( + "r1", + "g1", + date("2024-01-31"), + date("2024-02-01"), + instant("2024-01-31T10:30:00Z"), + instant("2024-02-01T22:15:00Z"), + Some(3), + Some(0.25) + ), + Row( + "r2", + "g1", + date("2024-03-01"), + date("2024-02-28"), + instant("2024-03-01T06:00:00Z"), + instant("2024-02-28T18:45:30.500Z"), + Some(-2), + Some(1.5) + ), + Row( + "r3", + "g2", + date("2023-12-25"), + None, + instant("2023-12-25T23:59:59.999Z"), + None, + None, + None + ), + Row("r4", "g3", None, None, None, None, Some(1), Some(2.0)), + Row( + "r5", + "g4", + date("2024-02-29"), + date("2024-03-01"), + instant("2024-02-29T12:00:00Z"), + instant("2024-03-01T00:00:00Z"), + Some(10), + Some(-0.5) + ), + // before 1970: negative epochs, whose days must FLOOR, not truncate toward zero + Row( + "r6", + "g5", + date("1969-12-31"), + date("1960-02-29"), + instant("1969-12-31T23:00:00Z"), + instant("1970-01-01T01:00:00Z"), + Some(-1), + Some(-0.25) + ), + // written as epoch-millisecond NUMBERS, a negative one included + Row( + "r7", + "g6", + date("2024-01-31"), + date("1969-12-31"), + instant("2024-01-31T10:30:00Z"), + instant("1969-12-31T23:00:00Z"), + Some(2), + Some(0.5), + epoch = true + ) + ) + + /** Stored by the computed-column table only: a DATE column holding a date-TIME string, which + * Elasticsearch accepts into a `date` field and which a DATE operand reads as its UTC day. + */ + private val computedOnlyRows: Seq[Row] = Seq( + Row( + "r8", + "g7", + date("2024-01-31"), + date("2024-02-01"), + instant("2024-01-31T22:30:00Z"), + instant("2024-02-01T01:30:00Z"), + Some(1), + Some(1.0), + dText = Some("2024-01-31T22:30:00Z") + ) + ) + + private val columns = + "id KEYWORD, g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, ts2 TIMESTAMP, n INT, x DOUBLE" + + /** The computed columns, each with the type the rules give its expression. */ + private val computed: Seq[(String, String, String)] = Seq( + ("c_dd", "BIGINT", "d - d2"), + ("c_dd2", "BIGINT", "d2 - d"), + ("c_tt", "DOUBLE", "ts - ts2"), + ("c_dt", "DOUBLE", "d - ts"), + ("c_dp1", "DATE", "d + 1"), + ("c_dm1", "DATE", "d - 1"), + ("c_nd", "DATE", "n + d"), + ("c_dp15", "TIMESTAMP", "d + 1.5"), + ("c_dx", "TIMESTAMP", "d + x"), + ("c_tp1", "TIMESTAMP", "ts + 1"), + ("c_tm15", "TIMESTAMP", "ts - 1.5"), + ("c_null", "DATE", "d + NULL"), + // ---- the fix round: operands that are not bare columns, a numeric string, casts ---- + ("c_coal", "BIGINT", "COALESCE(d, d2) - d2"), + ("c_nullif", "BIGINT", "NULLIF(d, d2) - d2"), + ("c_case", "BIGINT", "CASE WHEN n > 0 THEN d ELSE d2 END - d"), + ("c_castd", "BIGINT", "CAST(ts AS DATE) - d"), + ("c_str", "DATE", "d + '1'"), + ("c_tsbig", "BIGINT", "(ts - ts2)::BIGINT"), + ("c_nxbig", "BIGINT", "(n + x)::BIGINT"), + ("c_xint", "INT", "(x * 2)::INT"), + ("c_ddts", "TIMESTAMP", "(d - d2)::TIMESTAMP"), + ("c_tsts", "TIMESTAMP", "(ts - ts2)::TIMESTAMP"), + // ---- fix round 2: GREATEST / LEAST over dates, a number cast to a TIMESTAMP ---- + ("c_gr", "DATE", "GREATEST(d, d2)"), + ("c_le", "TIMESTAMP", "LEAST(d, ts)"), + ("c_grd", "BIGINT", "GREATEST(d, d2) - d"), + ("c_nts", "TIMESTAMP", "CAST(n AS TIMESTAMP)"), + ("c_xts", "TIMESTAMP", "CAST(x AS TIMESTAMP)") + ) + + private def json(r: Row): String = { + def dateValue(v: LocalDate): String = + if (r.epoch) s"${start(v).toEpochMilli}" else s""""$v"""" + def instantValue(v: Instant): String = if (r.epoch) s"${v.toEpochMilli}" else s""""$v"""" + (Seq(s""""id":"${r.id}"""", s""""g":"${r.g}"""") ++ + r.dText.map(t => s""""d":"$t"""").orElse(r.d.map(v => s""""d":${dateValue(v)}""")) ++ + r.d2.map(v => s""""d2":${dateValue(v)}""") ++ + r.ts.map(v => s""""ts":${instantValue(v)}""") ++ + r.ts2.map(v => s""""ts2":${instantValue(v)}""") ++ + r.n.map(v => s""""n":$v""") ++ r.x.map(v => s""""x":$v""")).mkString("{", ",", "}") + } + + private def bulk(index: String, docs: List[String]): Unit = { + implicit val bulkOptions: BulkOptions = BulkOptions(defaultIndex = index, logEvery = 100) + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + client.bulk[String](docs, identity, idKey = Some(Set("id"))) match { + case ElasticSuccess(_) => client.refresh(index) + case ElasticFailure(error) => + fail(s"Bulk indexing into $index failed: ${error.message}") + } + } + + override def beforeAll(): Unit = { + super.beforeAll() + run(s"CREATE TABLE $table ($columns, PRIMARY KEY (id))") + run( + s"CREATE TABLE $computedTable ($columns, " + + computed.map { case (c, t, e) => s"$c $t SCRIPT AS ($e)" }.mkString(", ") + + ", PRIMARY KEY (id))" + ) + bulk(table, rows.map(json).toList) + bulk(computedTable, (rows ++ computedOnlyRows).map(json).toList) + } + + override def afterAll(): Unit = { + Seq( + table, + computedTable, + rawIndex, + patternA, + patternB, + disagreeA, + disagreeB, + dateTimeTable, + comparisonTable, + alteredRawIndex, + grownTable, + refreshedA, + refreshedB + ).foreach(t => Try(Await.result(client.run(s"DROP TABLE IF EXISTS $t"), 60.seconds))) + super.afterAll() + } + + private def run(sql: String): Unit = + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(_) => () + case ElasticFailure(error) => fail(s"[$sql] failed: ${error.message}") + } + + /** The rows of a statement, through `GatewayApi.run` (JDBC / REPL / Flight SQL). */ + private def gatewayRows(sql: String): Either[String, Seq[ListMap[String, Any]]] = + Try { + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) + case ElasticSuccess(QueryStream(stream, _)) => + Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) + case ElasticSuccess(other) => Left(s"unexpected result $other") + case ElasticFailure(error) => Left(error.message) + } + }.toEither.left.map(t => s"${t.getClass.getSimpleName}: ${t.getMessage}").joinRight + + // ------------------------------------------------------------------------------------------- + // The oracle + // ------------------------------------------------------------------------------------------- + + /** An expected value: a number, an instant (a DATE is the instant its day starts at), a boolean, + * a number within a tolerance (a clock read when the statement ran), or NULL. + */ + private sealed trait Expected + private final case class Num(value: Double) extends Expected + private final case class About(value: Double, tolerance: Double) extends Expected + private final case class At(value: Instant) extends Expected + private final case class Bool(value: Boolean) extends Expected + private case object Null extends Expected + + private def start(d: LocalDate): Instant = d.atStartOfDay(ZoneOffset.UTC).toInstant + + private def utcDay(i: Instant): LocalDate = i.atZone(ZoneOffset.UTC).toLocalDate + + private def daysBetween(from: LocalDate, to: LocalDate): Expected = + Num(ChronoUnit.DAYS.between(from, to).toDouble) + + private def fractionalDays(from: Instant, to: Instant): Double = + (to.toEpochMilli - from.toEpochMilli).toDouble / DayMillis + + private def moved(from: Instant, days: Double): Expected = + At(from.plusMillis(Math.round(days * DayMillis))) + + private def expect(value: Option[Expected]): Expected = value.getOrElse(Null) + + /** The UTC day of the statement's clock, read when the test runs. */ + private def today: LocalDate = LocalDate.now(ZoneOffset.UTC) + + /** Elasticsearch wraps every `script_fields` value in a one-element array. */ + private def scalar(v: Any): Any = v match { + case s: Seq[_] => s.headOption.map(scalar).orNull + case c: java.util.Collection[_] => scalar(c.toArray.toSeq) + case other => other + } + + private def asInstant(v: Any): Option[Instant] = + scalar(v) match { + case null => None + case n: java.lang.Number => Some(Instant.ofEpochMilli(Math.round(n.doubleValue()))) + case i: Instant => Some(i) + case z: ZonedDateTime => Some(z.toInstant) + case o: OffsetDateTime => Some(o.toInstant) + case l: LocalDate => Some(start(l)) + case l: LocalDateTime => Some(l.toInstant(ZoneOffset.UTC)) + case d: java.util.Date => Some(d.toInstant) + case s => + val text = s.toString + Try(Instant.parse(text)) + .orElse(Try(ZonedDateTime.parse(text).toInstant)) + .orElse(Try(OffsetDateTime.parse(text).toInstant)) + .orElse(Try(LocalDateTime.parse(text).toInstant(ZoneOffset.UTC))) + .orElse(Try(start(LocalDate.parse(text)))) + .toOption + } + + private def asNumber(v: Any): Option[Double] = + scalar(v) match { + case null => None + case n: java.lang.Number => Some(n.doubleValue()) + case s => Try(s.toString.toDouble).toOption + } + + /** Does a returned value answer the expected one? `None` if it does, else what it answered. */ + private def differs(expected: Expected, actual: Any): Option[String] = + expected match { + case Null => + if (scalar(actual) == null) None else Some(s"$actual instead of NULL") + case Num(want) => + asNumber(actual) match { + case Some(got) if Math.abs(got - want) <= 1e-9 * Math.max(1.0, Math.abs(want)) => None + case _ => Some(s"$actual instead of $want") + } + case About(want, tolerance) => + asNumber(actual) match { + case Some(got) if Math.abs(got - want) <= tolerance => None + case _ => Some(s"$actual instead of about $want") + } + case At(want) => + asInstant(actual) match { + case Some(got) if got == want => None + case _ => Some(s"$actual instead of $want") + } + case Bool(want) => + scalar(actual) match { + case b: java.lang.Boolean if b.booleanValue() == want => None + case s: String if Try(s.toBoolean).toOption.contains(want) => None + case other => Some(s"$other instead of $want") + } + } + + /** A row-level expression and its value for each document. */ + private final case class RowForm(sql: String, value: Row => Expected) + + private def nullIf(a: Option[LocalDate], b: Option[LocalDate]): Option[LocalDate] = + if (a.isDefined && a == b) None else a + + /** GREATEST / LEAST as SQL engines answer them: a NULL argument is skipped, and the answer is + * NULL only when every argument is. + */ + private def latest(values: Option[Instant]*): Option[Instant] = + values.flatten.reduceOption((a, b) => if (a.isAfter(b)) a else b) + + private def earliest(values: Option[Instant]*): Option[Instant] = + values.flatten.reduceOption((a, b) => if (a.isBefore(b)) a else b) + + /** A number cast to a TIMESTAMP is the instant that many milliseconds from the epoch, a + * fractional number truncated as every fractional-to-integral cast is. + */ + private def epochMillis(value: Double): Expected = At(Instant.ofEpochMilli(value.toLong)) + + private val rowForms: Seq[RowForm] = Seq( + RowForm("d - d2", r => expect(for (a <- r.d; b <- r.d2) yield daysBetween(b, a))), + RowForm("d2 - d", r => expect(for (a <- r.d; b <- r.d2) yield daysBetween(a, b))), + RowForm( + "ts - ts2", + r => expect(for (a <- r.ts; b <- r.ts2) yield Num(fractionalDays(b, a))) + ), + RowForm( + "d - ts", + r => expect(for (a <- r.d; b <- r.ts) yield Num(fractionalDays(b, start(a)))) + ), + RowForm( + "ts - d", + r => expect(for (a <- r.ts; b <- r.d) yield Num(fractionalDays(start(b), a))) + ), + RowForm("d + 1", r => expect(r.d.map(a => At(start(a.plusDays(1)))))), + RowForm("d - 1", r => expect(r.d.map(a => At(start(a.minusDays(1)))))), + RowForm("1 + d", r => expect(r.d.map(a => At(start(a.plusDays(1)))))), + RowForm("d + n", r => expect(for (a <- r.d; n <- r.n) yield At(start(a.plusDays(n.toLong))))), + RowForm("n + d", r => expect(for (a <- r.d; n <- r.n) yield At(start(a.plusDays(n.toLong))))), + RowForm("d + 1.5", r => expect(r.d.map(a => moved(start(a), 1.5)))), + RowForm("d - 1.5", r => expect(r.d.map(a => moved(start(a), -1.5)))), + RowForm("d + x", r => expect(for (a <- r.d; x <- r.x) yield moved(start(a), x))), + RowForm("ts + 1", r => expect(r.ts.map(a => moved(a, 1)))), + RowForm("ts - 1.5", r => expect(r.ts.map(a => moved(a, -1.5)))), + RowForm("ts + n", r => expect(for (a <- r.ts; n <- r.n) yield moved(a, n.toDouble))), + RowForm("ts + x", r => expect(for (a <- r.ts; x <- r.x) yield moved(a, x))), + RowForm("d + NULL", _ => Null), + RowForm("NULL - ts", _ => Null), + RowForm( + "CAST('2024-02-15' AS DATE) - d", + r => expect(r.d.map(a => daysBetween(a, LocalDate.parse("2024-02-15")))) + ), + RowForm( + "d + 1 - d2", + r => expect(for (a <- r.d; b <- r.d2) yield daysBetween(b, a.plusDays(1))) + ), + RowForm( + "(ts - ts2) * 24", + r => + expect( + for (a <- r.ts; b <- r.ts2) + yield Num((a.toEpochMilli - b.toEpochMilli) * 24.0 / DayMillis) + ) + ), + // ---- the fix round ---- + // operands that are not bare columns + RowForm( + "COALESCE(d, d2) - d2", + r => expect(for (b <- r.d2; a <- r.d.orElse(r.d2)) yield daysBetween(b, a)) + ), + RowForm( + "NULLIF(d, d2) - d2", + r => expect(for (b <- r.d2; a <- nullIf(r.d, r.d2)) yield daysBetween(b, a)) + ), + RowForm( + "CASE WHEN n > 0 THEN d ELSE d2 END - d", + r => + expect( + for (a <- if (r.n.exists(_ > 0)) r.d else r.d2; b <- r.d) yield daysBetween(b, a) + ) + ), + RowForm( + "CAST(ts AS DATE) - d", + r => expect(for (t <- r.ts; b <- r.d) yield daysBetween(b, utcDay(t))) + ), + // a string literal that is a number counts as that number + RowForm("d + '1'", r => expect(r.d.map(a => At(start(a.plusDays(1)))))), + RowForm("d - '1.5'", r => expect(r.d.map(a => moved(start(a), -1.5)))), + // a typed NULL takes its type, and its value is NULL + RowForm("CAST(NULL AS DATE) - d", _ => Null), + RowForm("d + CAST(NULL AS INT)", _ => Null), + // a cast over arithmetic applies core's cast rule to the RESULT, and keeps NULL a NULL + RowForm( + "(ts - ts2)::BIGINT", + r => expect(for (a <- r.ts; b <- r.ts2) yield Num(fractionalDays(b, a).toLong.toDouble)) + ), + RowForm( + "(n + x)::BIGINT", + r => expect(for (n <- r.n; x <- r.x) yield Num((n + x).toLong.toDouble)) + ), + RowForm("(x * 2)::INT", r => expect(r.x.map(x => Num((x * 2).toInt.toDouble)))), + RowForm( + "(d - d2)::TIMESTAMP", + r => + expect( + for (a <- r.d; b <- r.d2) + yield At(Instant.ofEpochMilli(ChronoUnit.DAYS.between(b, a))) + ) + ), + RowForm( + "(ts - ts2)::TIMESTAMP", + r => + expect( + for (a <- r.ts; b <- r.ts2) yield At(Instant.ofEpochMilli(fractionalDays(b, a).toLong)) + ) + ), + RowForm( + "(d - d2)::DOUBLE", + r => expect(for (a <- r.d; b <- r.d2) yield daysBetween(b, a)) + ), + RowForm("(n + x)::BOOLEAN", r => expect(for (n <- r.n; x <- r.x) yield Bool(n + x != 0))), + // ---- fix round 2 ---- + // GREATEST / LEAST over dates and timestamps: a DATE is its day's start, a NULL is skipped + RowForm("GREATEST(d, d2)", r => expect(latest(r.d.map(start), r.d2.map(start)).map(At))), + RowForm("LEAST(d, ts)", r => expect(earliest(r.d.map(start), r.ts).map(At))), + RowForm("LEAST(ts, ts2)", r => expect(earliest(r.ts, r.ts2).map(At))), + RowForm("GREATEST(d, NULL)", r => expect(r.d.map(a => At(start(a))))), + RowForm( + "GREATEST(d, d2) - d", + r => + expect( + for (a <- r.d; m <- latest(r.d.map(start), r.d2.map(start))) + yield daysBetween(a, utcDay(m)) + ) + ), + // a number cast to a TIMESTAMP: the instant that many milliseconds from the epoch + RowForm("CAST(n AS TIMESTAMP)", r => expect(r.n.map(n => epochMillis(n.toDouble)))), + RowForm("CAST(x AS TIMESTAMP)", r => expect(r.x.map(epochMillis))) + ) + + /** A WHERE predicate and whether each document satisfies it -- `None` (UNKNOWN) keeps none. */ + private final case class WhereForm(sql: String, keeps: Row => Option[Boolean]) + + private def on(s: String): LocalDate = LocalDate.parse(s) + private def at(s: String): Instant = Instant.parse(s) + + private val whereForms: Seq[WhereForm] = Seq( + WhereForm( + "d - d2 = -1", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(b, a) == -1 + ), + WhereForm( + "d2 - d = 1", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(a, b) == 1 + ), + WhereForm( + "d - d2 = 2", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(b, a) == 2 + ), + WhereForm( + "ts - ts2 < -1", + r => for (a <- r.ts; b <- r.ts2) yield fractionalDays(b, a) < -1 + ), + WhereForm( + "d - ts > -0.75", + r => for (a <- r.d; b <- r.ts) yield fractionalDays(b, start(a)) > -0.75 + ), + WhereForm( + "d + 1 - d2 = 0", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(b, a.plusDays(1)) == 0 + ), + WhereForm( + "d - 1 - d2 = 1", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(b, a.minusDays(1)) == 1 + ), + WhereForm( + "n + d - d2 = 2", + r => + for (a <- r.d; b <- r.d2; n <- r.n) + yield ChronoUnit.DAYS.between(b, a.plusDays(n.toLong)) == 2 + ), + WhereForm( + "d + 1.5 - ts > 1", + r => + for (a <- r.d; b <- r.ts) + yield (start(a).toEpochMilli + 129600000L - b.toEpochMilli).toDouble / DayMillis > 1 + ), + WhereForm( + "ts + 1 - ts2 > 0", + r => for (a <- r.ts; b <- r.ts2) yield a.toEpochMilli + DayMillis - b.toEpochMilli > 0 + ), + WhereForm( + "ts - 1.5 - ts2 < -1.5", + r => + for (a <- r.ts; b <- r.ts2) + yield (a.toEpochMilli - 129600000L - b.toEpochMilli).toDouble / DayMillis < -1.5 + ), + WhereForm("d + NULL > CAST('2000-01-01' AS DATE)", _ => None), + // ---- refused at PARSE before the fix round, by a type the column did not have yet ---- + WhereForm( + "CURRENT_DATE - d > 30", + r => r.d.map(a => ChronoUnit.DAYS.between(a, today) > 30) + ), + WhereForm("d + 7 > CURRENT_DATE", r => r.d.map(a => a.plusDays(7).isAfter(today))), + WhereForm( + "NOW() - ts > 1", + r => r.ts.map(t => fractionalDays(t, Instant.now()) > 1) + ), + WhereForm( + "CAST(ts AS DATE) - d = 0", + r => for (t <- r.ts; a <- r.d) yield utcDay(t) == a + ), + WhereForm( + "d + 1 = CAST('2024-02-01' AS DATE)", + r => r.d.map(_.plusDays(1) == on("2024-02-01")) + ), + WhereForm("d + 1 > '2024-01-01'", r => r.d.map(_.plusDays(1).isAfter(on("2024-01-01")))), + // ---- fix round 2 ---- + WhereForm( + "GREATEST(d, d2) > CAST('2024-02-01' AS DATE)", + r => latest(r.d.map(start), r.d2.map(start)).map(_.isAfter(start(on("2024-02-01")))) + ), + WhereForm( + "LEAST(ts, ts2) < CAST('2024-02-01T00:00:00Z' AS TIMESTAMP)", + r => earliest(r.ts, r.ts2).map(_.isBefore(at("2024-02-01T00:00:00Z"))) + ), + WhereForm( + "CAST(n AS TIMESTAMP) > CAST('1970-01-01T00:00:00Z' AS TIMESTAMP)", + r => r.n.map(_ > 0) + ) + ) + + /** The aggregates of one group, `None` where the group has no value. */ + private final case class Group( + g: String, + maxD: Option[LocalDate], + minD: Option[LocalDate], + maxD2: Option[LocalDate], + maxTs: Option[Instant], + minTs: Option[Instant], + maxN: Option[Int], + countD: Int, + distinctD: Int, + avgN: Option[Double], + avgX: Option[Double], + avgD: Option[Double] + ) + + private def mean(values: Seq[Double]): Option[Double] = + if (values.isEmpty) None else Some(values.sum / values.size) + + private lazy val groups: Seq[Group] = rows.groupBy(_.g).toSeq.sortBy(_._1).map { case (g, rs) => + val ds = rs.flatMap(_.d) + val tss = rs.flatMap(_.ts) + Group( + g, + ds.reduceOption((a, b) => if (a.isAfter(b)) a else b), + ds.reduceOption((a, b) => if (a.isBefore(b)) a else b), + rs.flatMap(_.d2).reduceOption((a, b) => if (a.isAfter(b)) a else b), + tss.reduceOption((a, b) => if (a.isAfter(b)) a else b), + tss.reduceOption((a, b) => if (a.isBefore(b)) a else b), + rs.flatMap(_.n).reduceOption(_ max _), + ds.size, + ds.distinct.size, + mean(rs.flatMap(_.n).map(_.toDouble)), + mean(rs.flatMap(_.x)), + mean(ds.map(a => start(a).toEpochMilli.toDouble)) + ) + } + + /** A per-group expression and its value for each group. */ + private final case class GroupForm(sql: String, value: Group => Expected) + + private val groupForms: Seq[GroupForm] = Seq( + GroupForm( + "MAX(d) - MIN(d)", + g => expect(for (a <- g.maxD; b <- g.minD) yield daysBetween(b, a)) + ), + GroupForm( + "MIN(d) - MAX(d)", + g => expect(for (a <- g.maxD; b <- g.minD) yield daysBetween(a, b)) + ), + GroupForm( + "MAX(ts) - MIN(ts)", + g => expect(for (a <- g.maxTs; b <- g.minTs) yield Num(fractionalDays(b, a))) + ), + GroupForm( + "MAX(d) - MIN(ts)", + g => expect(for (a <- g.maxD; b <- g.minTs) yield Num(fractionalDays(b, start(a)))) + ), + GroupForm( + "MAX(ts) - MIN(d)", + g => expect(for (a <- g.maxTs; b <- g.minD) yield Num(fractionalDays(start(b), a))) + ), + GroupForm("MAX(d) + 1", g => expect(g.maxD.map(a => At(start(a.plusDays(1)))))), + GroupForm("MAX(d) - 1", g => expect(g.maxD.map(a => At(start(a.minusDays(1)))))), + GroupForm("1 + MAX(d)", g => expect(g.maxD.map(a => At(start(a.plusDays(1)))))), + GroupForm("MAX(d) + 1.5", g => expect(g.maxD.map(a => moved(start(a), 1.5)))), + GroupForm("MAX(ts) + 1", g => expect(g.maxTs.map(a => moved(a, 1)))), + GroupForm("MAX(ts) - 1.5", g => expect(g.maxTs.map(a => moved(a, -1.5)))), + GroupForm( + "MAX(d) + MAX(n)", + g => expect(for (a <- g.maxD; n <- g.maxN) yield At(start(a.plusDays(n.toLong)))) + ), + GroupForm("MAX(d) - NULL", _ => Null), + GroupForm( + "MAX(d) - CAST('2024-01-01' AS DATE)", + g => expect(g.maxD.map(a => daysBetween(LocalDate.parse("2024-01-01"), a))) + ), + // ---- the fix round ---- + // an aggregate is the metric it produces: COUNT counts, AVG averages + GroupForm("COUNT(d) - 1", g => Num(g.countD - 1.0)), + GroupForm("COUNT(DISTINCT d) * 2", g => Num(g.distinctD * 2.0)), + GroupForm("AVG(d) - 1", g => expect(g.avgD.map(a => Num(a - 1)))), + GroupForm( + "MAX(d) - AVG(n)", + g => expect(for (a <- g.maxD; n <- g.avgN) yield moved(start(a), -n)) + ), + // the request clock in a per-group calculation + GroupForm( + "CURRENT_DATE - MAX(d)", + g => expect(g.maxD.map(a => daysBetween(a, today))) + ), + GroupForm( + "DATEDIFF(CURRENT_DATE, MAX(d))", + g => expect(g.maxD.map(a => daysBetween(a, today))) + ), + GroupForm( + "NOW() - MAX(ts)", + g => expect(g.maxTs.map(t => About(fractionalDays(t, Instant.now()), 1.0 / 24))) + ), + // a cast over a per-group calculation + GroupForm( + "(MAX(n) + AVG(x))::BIGINT", + g => expect(for (n <- g.maxN; x <- g.avgX) yield Num((n + x).toLong.toDouble)) + ), + // ---- fix round 2: GREATEST / LEAST over aggregates, computed per group ---- + // ⚠️ A SELECT's per-group calculation is NULL in a group missing one of its metrics: + // Elasticsearch's bucket_script does not run there (its gap policy skips the group), as for + // every per-group calculation. A HAVING reads every aggregate and skips a NULL one (below). + GroupForm( + "GREATEST(MAX(d), MAX(d2))", + g => expect(for (a <- g.maxD; b <- g.maxD2) yield At(start(if (a.isAfter(b)) a else b))) + ), + GroupForm( + "LEAST(MAX(ts), MAX(d))", + g => + expect(for (t <- g.maxTs; a <- g.maxD; m <- earliest(Some(t), Some(start(a)))) yield At(m)) + ), + GroupForm( + "GREATEST(MAX(d), MAX(d2)) - MIN(d)", + g => + expect( + for (a <- g.maxD; b <- g.maxD2; m <- g.minD) + yield daysBetween(m, if (a.isAfter(b)) a else b) + ) + ), + GroupForm("GREATEST(MAX(n), 2)", g => expect(g.maxN.map(n => Num(Math.max(n, 2).toDouble)))) + ) + + /** A HAVING over a per-group calculation and whether each group is kept. */ + private final case class HavingForm(sql: String, keeps: Group => Option[Boolean]) + + private val havingForms: Seq[HavingForm] = Seq( + HavingForm( + s"SELECT g, MAX(d) - MIN(d) AS x FROM $table GROUP BY g HAVING x > 10", + g => for (a <- g.maxD; b <- g.minD) yield ChronoUnit.DAYS.between(b, a) > 10 + ), + HavingForm( + s"SELECT g, MAX(d) - MIN(d) AS x FROM $table GROUP BY g HAVING x = 0", + g => for (a <- g.maxD; b <- g.minD) yield ChronoUnit.DAYS.between(b, a) == 0 + ), + HavingForm( + s"SELECT g, MAX(ts) - MIN(ts) AS x FROM $table GROUP BY g HAVING x > 0.5", + g => for (a <- g.maxTs; b <- g.minTs) yield fractionalDays(b, a) > 0.5 + ), + HavingForm( + s"SELECT g, MAX(d) - MIN(ts) AS x FROM $table GROUP BY g HAVING x < 0", + g => for (a <- g.maxD; b <- g.minTs) yield fractionalDays(b, start(a)) < 0 + ), + HavingForm( + s"SELECT g, MAX(d) + 1 AS x FROM $table GROUP BY g HAVING x > MAX(d2)", + g => for (a <- g.maxD; b <- g.maxD2) yield a.plusDays(1).isAfter(b) + ), + HavingForm( + s"SELECT g, MAX(d) + 1.5 AS x FROM $table GROUP BY g HAVING x > MAX(ts)", + g => for (a <- g.maxD; b <- g.maxTs) yield start(a).plusMillis(129600000L).isAfter(b) + ), + HavingForm( + s"SELECT g, MAX(ts) - 1 AS x FROM $table GROUP BY g HAVING x < MIN(ts)", + g => for (a <- g.maxTs; b <- g.minTs) yield a.minusMillis(DayMillis).isBefore(b) + ), + HavingForm( + s"SELECT g FROM $table GROUP BY g HAVING MAX(d) > CAST('2024-03-01' AS DATE) - 30", + g => g.maxD.map(_.isAfter(on("2024-01-31"))) + ), + // refused at PARSE before the fix round + HavingForm( + s"SELECT g, MAX(d) + 1 AS x FROM $table GROUP BY g HAVING x > CAST('2024-01-01' AS DATE)", + g => g.maxD.map(_.plusDays(1).isAfter(on("2024-01-01"))) + ), + HavingForm( + s"SELECT g, COUNT(d) - 1 AS x FROM $table GROUP BY g HAVING x > 0", + g => Some(g.countD - 1 > 0) + ), + // ---- fix round 2: GREATEST over aggregates. Inline, the group filter reads every aggregate + // and SKIPS a NULL one (g2 has a `d` and no `d2`); through its SELECT alias it reads the + // per-group calculation, NULL in a group missing one of its metrics ---- + HavingForm( + s"SELECT g FROM $table GROUP BY g HAVING GREATEST(MAX(d), MAX(d2)) > CAST('2023-12-01' AS DATE)", + g => latest(g.maxD.map(start), g.maxD2.map(start)).map(_.isAfter(start(on("2023-12-01")))) + ), + HavingForm( + s"SELECT g, GREATEST(MAX(d), MAX(d2)) AS x FROM $table GROUP BY g HAVING x > CAST('2023-12-01' AS DATE)", + g => + for (a <- g.maxD; b <- g.maxD2) yield (if (a.isAfter(b)) a else b).isAfter(on("2023-12-01")) + ) + ) + + /** Run every statement before asserting anything, so a failure names every one that differs. */ + private def verdict(label: String, outcomes: Seq[(String, Either[String, Seq[String]])]): Unit = { + val wrong = outcomes.collect { + case (sql, Right(diffs)) if diffs.nonEmpty => s"[$sql] ${diffs.mkString("; ")}" + } + val errors = outcomes.collect { case (sql, Left(error)) => s"[$sql] $error" } + info(s"$label: ${outcomes.size} statements; wrong: ${wrong.size}; errors: ${errors.size}") + (wrong ++ errors).foreach(info(_)) + withClue( + s"${wrong.size} wrong, ${errors.size} errors:\n${(wrong ++ errors).take(30).mkString("\n")}\n" + ) { + wrong shouldBe empty + errors shouldBe empty + } + } + + private def byKey(rs: Seq[ListMap[String, Any]], key: String): Map[String, Any] = + rs.map(r => r.getOrElse(key, "?").toString -> r.getOrElse("x", null)).toMap + + // ------------------------------------------------------------------------------------------- + // The venues + // ------------------------------------------------------------------------------------------- + + private def rowVerdicts( + forms: Seq[RowForm], + from: String, + population: Seq[Row] + ): Seq[(String, Either[String, Seq[String]])] = + forms.map { f => + val sql = s"SELECT id, ${f.sql} AS x FROM $from" + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "id") + population.flatMap { r => + differs(f.value(r), got.getOrElse(r.id, null)).map(d => s"${r.id}: $d") + } ++ (if (got.size == population.size) Nil else Seq(s"${got.size} rows")) + } + } + + "date arithmetic in a row SELECT" should "answer the java.time oracle on every document" in { + rowForms.flatMap(f => rows.map(f.value)).distinct.size should be > 10 + verdict("row SELECT", rowVerdicts(rowForms, table, rows)) + } + + private def whereVerdicts( + forms: Seq[WhereForm], + from: String = table + ): Seq[(String, Either[String, Seq[String]])] = + forms.map { f => + val sql = s"SELECT id FROM $from WHERE ${f.sql}" + sql -> gatewayRows(sql).map { rs => + val kept = rs.map(_.getOrElse("id", "?").toString).toSet + val want = rows.filter(r => f.keeps(r).contains(true)).map(_.id).toSet + if (kept == want) Nil + else + Seq( + s"kept ${kept.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}" + ) + } + } + + "date arithmetic in a WHERE" should "keep exactly the documents the java.time oracle keeps" in { + // the predicates keep some documents and drop others + whereForms.exists(f => rows.exists(r => f.keeps(r).contains(true))) shouldBe true + whereForms.exists(f => rows.exists(r => !f.keeps(r).contains(true))) shouldBe true + verdict("WHERE", whereVerdicts(whereForms)) + } + + /** A date column compared with date arithmetic -- the filter `d > CURRENT_DATE - 7` is written + * for -- and the neighbouring comparisons a fix for it must keep working. On Elasticsearch 6.8 a + * `date` doc value is a `JodaCompatibleZonedDateTime`, which `isBefore` / `isAfter` / `isEqual` + * against a genuine `ZonedDateTime` used to fail with `Class.cast`. + * + * ⚠️ A column compared with ANOTHER column (`d2 > d`) is not here: on Elasticsearch 6.8 it is + * sent as a `range` bounded by the other column's NAME, outside this suite's rules. + */ + private val columnForms: Seq[WhereForm] = Seq( + WhereForm("d < CAST('2024-01-07' AS DATE) - 7", r => r.d.map(_.isBefore(on("2023-12-31")))), + WhereForm( + "d >= CAST('2024-03-08' AS DATE) - 7", + r => r.d.map(a => !a.isBefore(on("2024-03-01"))) + ), + WhereForm( + "ts > CAST('2024-03-01T00:00:00Z' AS TIMESTAMP) - 1.5", + r => r.ts.map(_.isAfter(at("2024-02-28T12:00:00Z"))) + ), + WhereForm("d2 = d + 1", r => for (a <- r.d; b <- r.d2) yield b == a.plusDays(1)), + WhereForm("d > d2 - 1", r => for (a <- r.d; b <- r.d2) yield a.isAfter(b.minusDays(1))), + WhereForm( + "DATE_ADD(d, INTERVAL 1 DAY) = d2", + r => for (a <- r.d; b <- r.d2) yield a.plusDays(1) == b + ), + WhereForm("CAST(ts AS DATE) = d", r => for (t <- r.ts; a <- r.d) yield utcDay(t) == a), + WhereForm( + "COALESCE(d, d2) > CAST('2024-01-01' AS DATE)", + r => r.d.orElse(r.d2).map(_.isAfter(on("2024-01-01"))) + ), + WhereForm( + "ts < ts2 + 0.5", + r => for (a <- r.ts; b <- r.ts2) yield a.isBefore(b.plusMillis(DayMillis / 2)) + ), + WhereForm( + "d >= CAST(ts AS DATE) - 1", + r => for (a <- r.d; t <- r.ts) yield !a.isBefore(utcDay(t).minusDays(1)) + ), + WhereForm("d + 1 > d2", r => for (a <- r.d; b <- r.d2) yield a.plusDays(1).isAfter(b)), + WhereForm("d - 1 = d2", r => for (a <- r.d; b <- r.d2) yield a.minusDays(1) == b), + WhereForm("CURRENT_DATE > d", r => r.d.map(today.isAfter(_))), + // ---- fix round 2: GREATEST / LEAST over dates against a date column ---- + WhereForm( + "GREATEST(d, d2) > d", + r => for (a <- r.d; m <- latest(r.d.map(start), r.d2.map(start))) yield m.isAfter(start(a)) + ), + WhereForm( + "LEAST(d, ts) < ts", + r => for (t <- r.ts; m <- earliest(r.d.map(start), r.ts)) yield m.isBefore(t) + ), + // the comparisons that already worked, on every major + WhereForm("d = CAST('2024-01-31' AS DATE)", r => r.d.map(_ == on("2024-01-31"))), + WhereForm("d >= '2024-01-01'", r => r.d.map(a => !a.isBefore(on("2024-01-01")))), + WhereForm( + "CAST(d AS TIMESTAMP) < ts", + r => for (a <- r.d; t <- r.ts) yield start(a).isBefore(t) + ), + WhereForm( + "n + d = CAST('2024-02-03' AS DATE)", + r => for (a <- r.d; n <- r.n) yield a.plusDays(n.toLong) == on("2024-02-03") + ), + WhereForm("d > CAST('2024-01-01' AS DATE)", r => r.d.map(_.isAfter(on("2024-01-01")))), + WhereForm( + "ts > CAST('2024-01-31T00:00:00Z' AS TIMESTAMP)", + r => r.ts.map(_.isAfter(at("2024-01-31T00:00:00Z"))) + ), + WhereForm("d < CURRENT_DATE", r => r.d.map(_.isBefore(today))), + WhereForm( + "CAST(d AS DATE) = CAST('2024-01-31' AS DATE)", + r => r.d.map(_ == on("2024-01-31")) + ), + WhereForm( + "d BETWEEN CAST('2024-01-01' AS DATE) AND CAST('2024-12-31' AS DATE)", + r => r.d.map(a => !a.isBefore(on("2024-01-01")) && !a.isAfter(on("2024-12-31"))) + ), + WhereForm( + "d > CAST('2024-03-08' AS DATE) - INTERVAL 7 DAY", + r => r.d.map(_.isAfter(on("2024-03-01"))) + ) + ) + + "a date column compared with date arithmetic" should "keep exactly the documents the java.time oracle keeps" in { + columnForms.exists(f => rows.exists(r => f.keeps(r).contains(true))) shouldBe true + columnForms.exists(f => rows.exists(r => !f.keeps(r).contains(true))) shouldBe true + verdict("column vs date arithmetic", whereVerdicts(columnForms)) + } + + private def groupVerdicts( + forms: Seq[GroupForm], + from: String = table + ): Seq[(String, Either[String, Seq[String]])] = + forms.map { f => + val sql = s"SELECT g, ${f.sql} AS x FROM $from GROUP BY g" + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "g") + groups.flatMap { g => + differs(f.value(g), got.getOrElse(g.g, null)).map(d => s"${g.g}: $d") + } ++ (if (got.size == groups.size) Nil else Seq(s"${got.size} groups")) + } + } + + "date arithmetic per group" should "answer the java.time oracle in every group, NULL where an operand has no value" in { + // a group with two dates, groups with one, and a non-empty group with none + groups.map(_.g) shouldBe Seq("g1", "g2", "g3", "g4", "g5", "g6") + groups.find(_.g == "g3").flatMap(_.maxD) shouldBe None + verdict("per-group SELECT", groupVerdicts(groupForms)) + } + + "a HAVING over date arithmetic" should "keep exactly the groups the java.time oracle keeps" in { + havingForms.exists(f => groups.exists(g => f.keeps(g).contains(true))) shouldBe true + havingForms.exists(f => groups.exists(g => !f.keeps(g).contains(true))) shouldBe true + verdict( + "HAVING", + havingForms.map { f => + f.sql -> gatewayRows(f.sql).map { rs => + val kept = rs.map(_.getOrElse("g", "?").toString).toSet + val want = groups.filter(g => f.keeps(g).contains(true)).map(_.g).toSet + if (kept == want) Nil + else + Seq( + s"kept ${kept.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}" + ) + } + } + ) + } + + "a DATE aggregate over date-time strings" should + "compare as its UTC day's start per group, in GREATEST / LEAST as in date arithmetic" in { + // Elasticsearch accepts a date-time into a `date` field, and MAX / MIN of it keep that time: + // per group, a DATE operand is still its UTC day -- the oracle is the one of `table`, whose + // `d` is the same days without a time + run(s"CREATE TABLE $dateTimeTable ($columns, PRIMARY KEY (id))") + bulk( + dateTimeTable, + rows.map(r => json(r.copy(dText = r.d.map(day => s"${day}T22:30:00Z")))).toList + ) + val forms = groupForms.filter(f => + Set( + "GREATEST(MAX(d), MAX(d2))", + "LEAST(MAX(ts), MAX(d))", + "GREATEST(MAX(d), MAX(d2)) - MIN(d)", + "MAX(d) + 1", + "MAX(d) - MIN(ts)" + ).contains(f.sql) + ) + forms.size shouldBe 5 + // a group filter reads every aggregate, NULL ones skipped: g1's LEAST is its day's start, + // 2024-03-01T00:00, kept; read with its time, it would be 06:00, and g1 would be dropped + val threshold = Instant.parse("2024-03-01T03:00:00Z") + val having = + s"SELECT g FROM $dateTimeTable GROUP BY g " + + s"HAVING LEAST(MAX(ts), MAX(d)) < CAST('$threshold' AS TIMESTAMP)" + val want = groups + .filter(g => earliest(g.maxTs, g.maxD.map(start)).exists(_.isBefore(threshold))) + .map(_.g) + .toSet + want should contain("g1") + verdict( + s"FROM $dateTimeTable", + groupVerdicts(forms, dateTimeTable) :+ (having -> gatewayRows(having).map { rs => + val kept = rs.map(_.getOrElse("g", "?").toString).toSet + if (kept == want) Nil + else + Seq( + s"kept ${kept.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}" + ) + }) + ) + } + + "a computed column over date arithmetic" should "STORE the java.time oracle's value" in { + val forms: Map[String, RowForm] = rowForms.map(f => f.sql -> f).toMap + val stored = rows ++ computedOnlyRows + val sql = s"SELECT id, ${computed.map(_._1).mkString(", ")} FROM $computedTable" + verdict( + "SCRIPT AS", + Seq(sql -> gatewayRows(sql).map { rs => + val byId = rs.map(r => r.getOrElse("id", "?").toString -> r).toMap + (if (byId.size == stored.size) Nil else Seq(s"${byId.size} rows")) ++ + (for { + (c, _, expression) <- computed + r <- stored + } yield { + val want = + forms.get(expression).map(_.value(r)).getOrElse(fail(s"no oracle for $expression")) + differs(want, byId.get(r.id).flatMap(_.get(c)).orNull).map(d => + s"$c [$expression] ${r.id}: $d" + ) + }).flatten + }) + ) + } + + /** The type DESCRIBE reports for `field` of `index`. */ + private def described(index: String, field: String): Either[String, Option[String]] = + gatewayRows(s"DESCRIBE TABLE $index").map( + _.find(_.get("Field").contains(field)).flatMap(_.get("Type")).map(_.toString) + ) + + private val loaded = Seq( + ("n1", "a", "2024-01-31T10:30:00Z", "2024-02-01T09:15:00Z"), + ("n2", "a", "2024-01-31T23:00:00Z", "2024-02-01T01:00:00Z"), + ("n3", "b", "1969-12-31T23:00:00Z", "1970-01-01T01:00:00Z") + ) + + /** The undeclared dates keep their time: `ts + 1`, `ts2 - ts`, `ts - 1.5` over `index`. */ + private def undeclaredVerdicts(index: String): Seq[(String, Either[String, Seq[String]])] = { + def ts(s: String) = Instant.parse(s) + Seq( + s"SELECT id, ts + 1 AS x FROM $index" -> loaded.map { case (id, _, t, _) => + id -> moved(ts(t), 1) + }, + s"SELECT id, ts2 - ts AS x FROM $index" -> loaded.map { case (id, _, t, t2) => + id -> Num(fractionalDays(ts(t), ts(t2))) + }, + s"SELECT id, ts - 1.5 AS x FROM $index" -> loaded.map { case (id, _, t, _) => + id -> moved(ts(t), -1.5) + } + ).map { case (sql, expected) => + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "id") + expected.flatMap { case (id, want) => + differs(want, got.getOrElse(id, null)).map(d => s"$id: $d") + } + } + } + } + + "an index created by a bulk load, with no declaration" should + "hold TIMESTAMPs: its dates keep their time, and DESCRIBE reports them so" in { + bulk( + rawIndex, + loaded.map { case (id, g, ts, ts2) => + s"""{"id":"$id","g":"$g","ts":"$ts","ts2":"$ts2"}""" + }.toList + ) + // the lead's ruling of 2026-10-05: a `date` field with no elasticsql declaration is a + // TIMESTAMP everywhere, DESCRIBE included + described(rawIndex, "ts") shouldBe Right(Some("TIMESTAMP")) + def ts(s: String) = Instant.parse(s) + verdict( + "undeclared date", + Seq( + s"SELECT id, ts + 1 AS x FROM $rawIndex" -> loaded.map { case (id, _, t, _) => + id -> moved(ts(t), 1) + }, + s"SELECT id, ts2 - ts AS x FROM $rawIndex" -> loaded.map { case (id, _, t, t2) => + id -> Num(fractionalDays(ts(t), ts(t2))) + }, + s"SELECT id, ts - 1.5 AS x FROM $rawIndex" -> loaded.map { case (id, _, t, _) => + id -> moved(ts(t), -1.5) + } + ).map { case (sql, expected) => + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "id") + expected.flatMap { case (id, want) => + differs(want, got.getOrElse(id, null)).map(d => s"$id: $d") + } + } + } :+ { + val sql = + s"SELECT g.keyword AS g, MAX(ts2) - MIN(ts) AS x FROM $rawIndex GROUP BY g.keyword" + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "g") + loaded.groupBy(_._2).toSeq.flatMap { case (g, members) => + val want = Num( + fractionalDays(members.map(m => ts(m._3)).min, members.map(m => ts(m._4)).max) + ) + differs(want, got.getOrElse(g, null)).map(d => s"$g: $d") + } + } + } + ) + } + + it should "stay a TIMESTAMP once an elasticsql ALTER TABLE declares the table, cache refreshed" in { + bulk( + alteredRawIndex, + loaded.map { case (id, g, ts, ts2) => + s"""{"id":"$id","g":"$g","ts":"$ts","ts2":"$ts2"}""" + }.toList + ) + verdict("before the ALTER", undeclaredVerdicts(alteredRawIndex)) + // an unrelated ALTER writes the table's `_meta`, every column of it: the undeclared dates are + // declared as what they are, TIMESTAMPs -- never turned into DATEs + run(s"ALTER TABLE $alteredRawIndex ADD COLUMN note KEYWORD") + client.invalidateAllSchemas() + described(alteredRawIndex, "ts") shouldBe Right(Some("TIMESTAMP")) + described(alteredRawIndex, "note") shouldBe Right(Some("KEYWORD")) + verdict("after the ALTER and a cache refresh", undeclaredVerdicts(alteredRawIndex)) + } + + "a date field a bulk load adds to a table elasticsql created" should + "be a TIMESTAMP, while the declared DATE stays a DATE" in { + run(s"CREATE TABLE $grownTable (id KEYWORD, d DATE, PRIMARY KEY (id))") + bulk( + grownTable, + List("""{"id":"w1","d":"2024-01-31","seen":"2024-01-31T10:30:00Z"}""") + ) + client.invalidateAllSchemas() + described(grownTable, "d") shouldBe Right(Some("DATE")) + described(grownTable, "seen") shouldBe Right(Some("TIMESTAMP")) + verdict( + "a dynamic date field", + Seq(s"SELECT id, seen + 1 AS x, d + 1 AS y FROM $grownTable" -> { + val sql = s"SELECT id, seen + 1 AS x, d + 1 AS y FROM $grownTable" + gatewayRows(sql).map { rs => + val got = rs.headOption.getOrElse(ListMap.empty[String, Any]) + differs(At(Instant.parse("2024-02-01T10:30:00Z")), got.getOrElse("x", null)).toSeq ++ + differs(At(start(on("2024-02-01"))), got.getOrElse("y", null)).toSeq + } + }) + ) + } + + /** The comparison rule's fixture: base's census rows (r1 to r4), and r5, whose `n` = 10 and `s` = + * "12" tell a number read as a NUMBER from one compared as TEXT ("10" < "2"). + */ + private val comparisonDocs = List( + """{"id":"r1","g":"g1","d":"2024-01-31","d2":"2024-02-01","ts":"2024-01-31T10:30:00Z","n":3,"s":"a"}""", + """{"id":"r2","g":"g1","d":"2024-03-01","d2":"2024-02-28","ts":"2024-03-01T06:00:00Z","n":5,"s":"5"}""", + """{"id":"r3","g":"g2","d":"2023-12-25","ts":"2023-12-25T23:59:59Z","n":-2,"s":"3"}""", + """{"id":"r4","g":"g3","s":"b"}""", + """{"id":"r5","g":"g3","d2":"2024-01-15","n":10,"s":"12"}""" + ) + + /** `SELECT id, AS v` over [[comparisonTable]], against the expected value of each + * row -- PostgreSQL 16's answer over the same rows, which for r1 to r4 is also base's. + */ + private def comparisonVerdict( + expression: String, + expected: Map[String, Expected] + ): (String, Either[String, Seq[String]]) = { + val sql = s"SELECT id, $expression AS x FROM $comparisonTable" + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "id") + expected.toSeq.sortBy(_._1).flatMap { case (id, want) => + differs(want, got.getOrElse(id, null)).map(d => s"$id: $d") + } + } + } + + private def kept(sql: String, want: Set[String]): (String, Either[String, Seq[String]]) = + sql -> gatewayRows(sql).map { rs => + val got = rs.map(r => r.getOrElse("id", r.getOrElse("g", "?")).toString).toSet + if (got == want) Nil + else + Seq(s"kept ${got.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}") + } + + private def flags(ones: String*): Map[String, Expected] = + Seq("r1", "r2", "r3", "r4", "r5").map(id => id -> Num(if (ones.contains(id)) 1 else 0)).toMap + + "the ONE comparison rule" should + "read a string literal by what it is compared with, in every clause, as PostgreSQL does" in { + run( + s"CREATE TABLE $comparisonTable (id KEYWORD, g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, " + + "n INT, s KEYWORD, PRIMARY KEY (id))" + ) + bulk(comparisonTable, comparisonDocs) + verdict( + "comparison rule", + Seq( + // the review's statements, accurate on base and refused by round 2: base's values on r1 + // to r4, and r5 read as a number + comparisonVerdict("CASE WHEN COALESCE(n, 0) = '3' THEN 1 ELSE 0 END", flags("r1")), + comparisonVerdict("CASE WHEN n + 1 = '4' THEN 1 ELSE 0 END", flags("r1")), + comparisonVerdict( + "CASE WHEN LENGTH(s) = '1' THEN 1 ELSE 0 END", + flags("r1", "r2", "r3", "r4") + ), + comparisonVerdict( + "CASE WHEN COALESCE(n, 0) > '2' THEN 1 ELSE 0 END", + flags("r1", "r2", "r5") + ), + comparisonVerdict("CASE WHEN NULLIF(n, 0) = '3' THEN 1 ELSE 0 END", flags("r1")), + comparisonVerdict("CASE n WHEN '3' THEN 1 ELSE 0 END", flags("r1")), + kept( + s"SELECT id FROM $comparisonTable WHERE COALESCE(d, d2) BETWEEN '2024-01-01' AND '2024-02-01'", + Set("r1", "r5") + ), + // main refused this one at parse + kept(s"SELECT id FROM $comparisonTable WHERE COALESCE(n, 0) = '3'", Set("r1")), + kept( + s"SELECT id FROM $comparisonTable WHERE COALESCE(d, d2) IN ('2024-01-31', '2024-01-15')", + Set("r1", "r5") + ), + kept( + s"SELECT g, MAX(n) AS m FROM $comparisonTable GROUP BY g HAVING MAX(n) > '4'", + Set("g1", "g3") + ), + // a DATE against a TIMESTAMP is comparable, judged on the declared types + comparisonVerdict( + "CASE d WHEN CAST('2024-01-31' AS DATE) THEN 1 ELSE 0 END", + flags("r1") + ), + comparisonVerdict( + "NULLIF(COALESCE(d, d2), CAST('2024-01-31' AS DATE))", + Map( + "r1" -> Null, + "r2" -> At(start(on("2024-03-01"))), + "r3" -> At(start(on("2023-12-25"))), + "r4" -> Null, + "r5" -> At(start(on("2024-01-15"))) + ) + ), + comparisonVerdict( + "NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP))", + Map( + "r1" -> At(start(on("2024-01-31"))), + "r2" -> At(start(on("2024-03-01"))), + "r3" -> At(start(on("2023-12-25"))), + "r4" -> Null, + "r5" -> Null + ) + ), + // the arithmetic reads it too + comparisonVerdict( + "d - '2024-01-31'", + Map("r1" -> Num(0), "r2" -> Num(30), "r3" -> Num(-37), "r4" -> Null, "r5" -> Null) + ), + comparisonVerdict( + "n + '1'", + Map("r1" -> Num(4), "r2" -> Num(6), "r3" -> Num(-1), "r4" -> Null, "r5" -> Num(11)) + ), + // a literal that spells nothing the operand reads stays refused by name + s"SELECT id, CASE WHEN COALESCE(n, 0) = 'a' THEN 1 ELSE 0 END AS v FROM $comparisonTable" -> + (Try( + Await.result( + client.run( + s"SELECT id, CASE WHEN COALESCE(n, 0) = 'a' THEN 1 ELSE 0 END AS v FROM $comparisonTable" + ), + 60.seconds + ) + ).toEither match { + case Right(ElasticFailure(error)) + if error.message.contains("'BIGINT' is not compatible with 'VARCHAR'") => + Right(Nil) + case other => Right(Seq(s"answered $other instead of the refusal by name")) + }) + ) + ) + } + + "DATE_TRUNC(…, DAY) compared with a date" should "run: a day truncation of a date is the date" in { + verdict( + "DATE_TRUNC", + Seq( + kept( + s"SELECT id FROM $comparisonTable WHERE DATE_TRUNC(ts, DAY) = '2024-01-31'", + Set("r1") + ), + kept( + s"SELECT id FROM $comparisonTable WHERE DATE_TRUNC(ts, DAY) > '2024-01-01'", + Set("r1", "r2") + ), + kept( + s"SELECT id FROM $comparisonTable WHERE DATE_TRUNC(ts, DAY) = CAST('2024-01-31' AS DATE)", + Set("r1") + ), + comparisonVerdict( + "CASE WHEN DATE_TRUNC(ts, DAY) = '2024-01-31' THEN 1 ELSE 0 END", + flags("r1") + ) + ) + ) + } + + "a pattern's cached schema" should "be refreshed by an elasticsql DDL statement on an index it matches" in { + val from = "date_arithmetic_rf_*" + def typed(sql: String, refusal: String): Boolean = + Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) => error.message.contains(refusal) + case _ => false + } + val dTimesTwo = s"SELECT id, d * 2 AS x FROM $from" + val refusal = "operator * cannot be applied to DATE and BIGINT" + run(s"CREATE TABLE $refreshedA (id KEYWORD, d DATE, PRIMARY KEY (id))") + bulk(refreshedA, List("""{"id":"a1","d":"2024-01-31"}""")) + // the pattern's merged schema is read, and cached: `d` is a DATE + typed(dTimesTwo, refusal) shouldBe true + // CREATE: a second index maps `d` as a KEYWORD -- the indices disagree, `d` has no type + run(s"CREATE TABLE $refreshedB (id KEYWORD, d KEYWORD, PRIMARY KEY (id))") + typed(dTimesTwo, refusal) shouldBe false + // DROP: `d` is a DATE again + run(s"DROP TABLE $refreshedB") + typed(dTimesTwo, refusal) shouldBe true + // ALTER: a column added to the one index is known to the pattern at once + run(s"ALTER TABLE $refreshedA ADD COLUMN m DATE") + typed(s"SELECT id, m * 2 AS x FROM $from", refusal) shouldBe true + } + + "an index pattern or a comma list" should "answer like the single index it matches" in { + run(s"CREATE TABLE $patternA ($columns, PRIMARY KEY (id))") + run(s"CREATE TABLE $patternB ($columns, PRIMARY KEY (id))") + bulk(patternA, rows.map(json).toList) + client.refresh(patternB) + val samples = + rowForms.filter(f => Set("d - d2", "d + 1", "ts - ts2", "d + 1.5").contains(f.sql)) + val groupSamples = + groupForms.filter(f => + Set("MAX(d) - MIN(d)", "MAX(ts) - MIN(ts)", "MAX(d) + 1").contains(f.sql) + ) + Seq("date_arithmetic_pat_*", s"$patternA,$patternB").foreach { from => + verdict( + s"FROM $from", + rowVerdicts(samples, from, rows) ++ groupVerdicts(groupSamples, from) ++ Seq { + // a type rule reads the merged schema: refused by name, as over the single index + val sql = s"SELECT id, d * 2 AS x FROM $from" + sql -> (Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) + if error.message.contains("operator * cannot be applied to DATE and BIGINT") => + Right(Nil) + case other => Right(Seq(s"answered $other instead of the refusal by name")) + }) + } :+ { + // a subquery over the pattern is scoped -- and typed -- by its merged mapping, as one + // over the single index is + val sql = + s"SELECT id FROM $table WHERE g IN (SELECT g FROM $from WHERE d > CAST('2024-01-01' AS DATE))" + sql -> gatewayRows(sql).map { rs => + val kept = rs.map(_.getOrElse("id", "?").toString).toSet + val groups = rows.filter(_.d.exists(_.isAfter(on("2024-01-01")))).map(_.g).toSet + val want = rows.filter(r => groups.contains(r.g)).map(_.id).toSet + if (kept == want) Nil + else + Seq( + s"kept ${kept.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}" + ) + } + } + ) + } + } + + "an index pattern whose indices disagree on a field" should + "type every field they agree on, and refuse nothing on that one: Elasticsearch answers" in { + run(s"CREATE TABLE $disagreeA ($columns, PRIMARY KEY (id))") + run(s"CREATE TABLE $disagreeB (id KEYWORD, g KEYWORD, d KEYWORD, PRIMARY KEY (id))") + bulk(disagreeA, rows.map(json).toList) + client.refresh(disagreeB) + val from = "date_arithmetic_dis_*" + // the indices agree on every field but `d` (DATE and KEYWORD): every other field is typed as + // over the single index -- the date-arithmetic rules apply, and a type rule refuses by name + verdict( + s"FROM $from", + rowVerdicts( + rowForms.filter(f => + Set("ts - ts2", "ts + 1", "ts - 1.5", "LEAST(ts, ts2)").contains(f.sql) + ), + from, + rows + ) ++ groupVerdicts( + groupForms.filter(f => Set("MAX(ts) - MIN(ts)", "MAX(ts) + 1").contains(f.sql)), + from + ) :+ { + val sql = s"SELECT id, ts * 2 AS x FROM $from" + sql -> (Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) + if error.message.contains("operator * cannot be applied to TIMESTAMP and BIGINT") => + Right(Nil) + case other => Right(Seq(s"answered $other instead of the refusal by name")) + }) + } + ) + // the field they disagree on has no type: nothing is refused on it, Elasticsearch answers + // (MEASURED, reported) + Seq( + s"SELECT g, MAX(d) - MIN(d) AS x FROM $from GROUP BY g", + s"SELECT id, d + 1 AS x FROM $from", + s"SELECT id, d * 2 AS x FROM $from" + ).foreach { sql => + val answer = Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) => + // never core's type refusal: with no type nothing is refused on types + error.message should not include "cannot be applied to" + error.message should not include "Type mismatch" + s"FAILURE ${error.statusCode.getOrElse("-")} ${error.message.take(200)}" + case Right(ElasticSuccess(_)) => + s"SUCCESS ${gatewayRows(sql).map(_.take(3)).fold(identity, _.toString).take(300)}" + case Left(t) => s"THROW ${t.getMessage.take(200)}" + } + info(s"[$sql] -> $answer") + } + } + + "a combination the type rules refuse" should "be refused by name once the column types are known, before anything runs" in { + val remedy = "subtract two dates for the days between them" + val refused = Seq( + s"SELECT id, d * 2 AS x FROM $table" -> Seq( + "operator * cannot be applied to DATE and BIGINT", + remedy + ), + s"SELECT id, d + d2 AS x FROM $table" -> Seq( + "operator + cannot be applied to DATE and DATE", + remedy + ), + s"SELECT id, 2 - d AS x FROM $table" -> Seq( + "operator - cannot be applied to BIGINT and DATE", + remedy + ), + s"SELECT id, ts / ts2 AS x FROM $table" -> Seq( + "operator / cannot be applied to TIMESTAMP and TIMESTAMP", + remedy + ), + s"SELECT g, MAX(d) / 1000 AS x FROM $table GROUP BY g" -> Seq( + "operator / cannot be applied to DATE and BIGINT", + remedy + ), + s"SELECT g, MAX(d) % MIN(d) AS x FROM $table GROUP BY g" -> Seq( + "operator % cannot be applied to DATE and DATE", + remedy + ), + s"SELECT id FROM $table WHERE d * 2 > 1" -> Seq( + "operator * cannot be applied to DATE and BIGINT", + remedy + ), + s"SELECT id, d + 'x' AS x FROM $table" -> Seq( + "operator + cannot be applied to DATE and VARCHAR", + remedy + ), + s"CREATE TABLE date_arithmetic_refused ($columns, c BIGINT SCRIPT AS (d * 2), PRIMARY KEY (id))" -> + Seq("operator * cannot be applied to DATE and BIGINT", remedy), + // every other TYPE rule that waits for the column types (the lead's ruling of 2026-10-05) + s"SELECT id FROM $table WHERE n + 1 > 'abc'" -> Seq( + "'BIGINT' is not compatible with 'VARCHAR'" + ), + s"SELECT id, n + id AS x FROM $table" -> Seq( + "output 'INT' is not compatible with input 'KEYWORD'" + ), + s"SELECT id, CASE n WHEN 'a' THEN 1 ELSE 0 END AS x FROM $table" -> Seq( + "CASE WHEN conditions must be of the same type as the expression" + ), + s"SELECT id, GREATEST(d, n) AS x FROM $table" -> Seq( + "GREATEST requires numeric arguments, or date and timestamp arguments, but got DATE for d " + + "and INT for n" + ), + // (`NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP))` compares a DATE with a TIMESTAMP, which + // the ONE comparison rule accepts: it RUNS, in "the ONE comparison rule" above) + s"SELECT id, NULLIF(n, 'a') AS x FROM $table" -> Seq("compares INT with VARCHAR"), + // one true mismatch per rule that waits for the column types + s"SELECT id FROM $table WHERE id = n" -> Seq("'KEYWORD' is not compatible with 'INT'"), + s"SELECT id FROM $table WHERE CAST(ts AS TIME) > ts" -> Seq( + "a TIME value cannot be compared" + ), + s"SELECT id FROM $table WHERE CAST(n AS INT) IN ('a', 'b')" -> Seq( + "'INT' is not compatible with 'VARCHAR'" + ), + s"SELECT g FROM $table GROUP BY g HAVING COUNT(n) IN ('a', 'b')" -> Seq( + "'BIGINT' is not compatible with 'VARCHAR'" + ), + s"SELECT id FROM $table WHERE n BETWEEN 1 AND 'a'" -> Seq( + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + s"SELECT id FROM $table WHERE CAST(id AS INT) BETWEEN 'a' AND 'b'" -> Seq( + "output 'INT' is not compatible with input 'VARCHAR'" + ), + s"SELECT id, NULLIF(id, 1) AS x FROM $table" -> Seq("compares KEYWORD with BIGINT"), + s"SELECT id, CASE WHEN n THEN 1 ELSE 0 END AS x FROM $table" -> Seq( + "CASE WHEN conditions must be of type BOOLEAN" + ), + s"SELECT id, LEAST('a', d) AS x FROM $table" -> Seq( + "LEAST requires numeric arguments, or date and timestamp arguments, but got VARCHAR for 'a'" + ), + s"SELECT id, 1 + 'abc' AS x FROM $table" -> Seq("input 'VARCHAR'"), + s"SELECT id FROM $table UNION ALL SELECT n FROM $table" -> Seq( + "compatible types at column 1" + ) + ) + val outcomes = refused.map { case (sql, refusal) => + sql -> (Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) if refusal.forall(error.message.contains) => + Right(Nil) + case Right(ElasticFailure(error)) => Right(Seq(s"refused as [${error.message}]")) + case Right(other) => Right(Seq(s"answered $other instead of refusing")) + case Left(t) => Left(t.toString) + }) + } + Try(Await.result(client.run("DROP TABLE IF EXISTS date_arithmetic_refused"), 60.seconds)) + verdict("refusals", outcomes) + } + + "a CREATE OR REPLACE TABLE ... AS SELECT the rules refuse" should + "be refused before the existing table is touched" in { + val target = "date_arithmetic_ctas" + try { + run(s"CREATE OR REPLACE TABLE $target AS SELECT id, d - d2 AS x FROM $table") + client.refresh(target) + gatewayRows(s"SELECT id, x FROM $target").map(_.size) shouldBe Right(rows.size) + Seq( + s"SELECT g, MAX(d) / 1000 AS x FROM $table GROUP BY g" -> + "operator / cannot be applied to DATE and BIGINT", + // a type rule that no longer refuses at parse, and must still refuse before the replace + s"SELECT id, n + id AS x FROM $table" -> + "output 'INT' is not compatible with input 'KEYWORD'" + ).foreach { case (select, refusal) => + Try( + Await.result(client.run(s"CREATE OR REPLACE TABLE $target AS $select"), 60.seconds) + ).toEither match { + case Right(ElasticFailure(error)) => error.message should include(refusal) + case other => fail(s"[$select] answered $other instead of refusing") + } + // the table it would have replaced is still there, with its rows + gatewayRows(s"SELECT id, x FROM $target").map(_.size) shouldBe Right(rows.size) + } + } finally Try(Await.result(client.run(s"DROP TABLE IF EXISTS $target"), 60.seconds)) + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index 7ae2f2f30..99990c5ed 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -1267,15 +1267,16 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { } // πŸ”΄ The temporal-literal edge. `NULLIF(created, '2024-01-15')` and - // `NULLIF(created, 'yesterday')` are BOTH a TIMESTAMP against a VARCHAR, and BOTH are refused - // -- but by DIFFERENT judges, which is the whole point of the row. + // `NULLIF(created, 'yesterday')` are both a date against a string literal, and they are judged + // apart: the well-formed literal IS the date it spells -- the ONE comparison rule (the lead's + // ruling of 2026-10-05), read once as the instant it denotes, so the ingest script compares two + // instants and needs no per-document `String -> temporal` coercion -- while the malformed one + // is refused by the resolver, which names the literal and the field. // - // AMENDED by the #382 review. The well-formed literal used to be ACCEPTED here, and the script - // it produced compared a `ZonedDateTime` with a `String` using Painless `==`: always `false`, - // never failing, so the NULLIF returned `created` for every document and the CREATE answered - // 200. Making that comparison WORK needs a `String -> temporal` coercion the engine does not - // have (and Elasticsearch date math has no Painless equivalent at all), so it is refused until - // its own story lands, and it says so rather than blaming the SQL. + // (The #382 review had refused the well-formed one too: the script it produced compared a + // `ZonedDateTime` with a `String` using Painless `==`, always `false`, so the NULLIF returned + // `created` for every document. It is the comparison that now works, and the STORED values + // below are what says so.) val wellFormed = client .run("""CREATE TABLE IF NOT EXISTS nullif_date ( | id INT, @@ -1284,16 +1285,26 @@ trait GatewayApiIntegrationSpec extends GatewayIntegrationTestKit { |);""".stripMargin) .futureValue withClue(s"$wellFormed ") { - wellFormed.isFailure shouldBe true - val message = String.valueOf(wellFormed) - // the ENGINE's message: it names the two types and says what is missing - message should include("not supported yet") - message should include("TIMESTAMP") - message should include("VARCHAR") - // ...and NOT the resolver's, which would mean the literal was judged malformed - message should not include "as a date/time value for date field" + wellFormed.isSuccess shouldBe true } - client.indexExists("nullif_date", pattern = false) shouldBe ElasticSuccess(false) + assertDml( + System.nanoTime(), + client + .run("INSERT INTO nullif_date (id, created) VALUES (1, '2024-01-15'), (2, '2024-01-16');") + .futureValue, + Some(DmlResult(inserted = 2)) + ) + val stored = collectRows( + System.nanoTime(), + client.run("SELECT id, c FROM nullif_date ORDER BY id").futureValue + ) + withClue(s"$stored ") { + stored.map(_.get("c").map(String.valueOf).filter(_ != "null").map(_.take(10))) shouldBe Seq( + None, + Some("2024-01-16") + ) + } + client.run("DROP TABLE IF EXISTS nullif_date").futureValue val bad = client .run("""CREATE TABLE IF NOT EXISTS nullif_date_bad ( From b559ad4799b6f422e28a9ad020f84f58ffe6e490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Oct 2026 23:46:56 +0200 Subject: [PATCH 2/5] perf(sql): read each comparison literal once; a quoted number beside an integer must be whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comparison rule re-read every string literal on each type question and during request generation, and tried each one as a date through java.time parsers, whose refusal is an exception (about 2 Β΅s each). Each literal is now read once per node. A cheap shape check runs before any date parse, so '1' or 'abc' never reaches the parser. Request generation is back to its earlier cost (WHERE-heavy 60.7 β†’ 45.8 Β΅s, against 41.5 before the comparison rule). The requests are byte-identical to the previous commit. A quoted number beside an integer type must be a whole number that the type can hold, as in PostgreSQL. COALESCE(n, 0) = '1.5', n + '1.5', NULLIF(n, '1.0') and CASE n WHEN '1.5' ... are refused by name. Beside DOUBLE, REAL or DECIMAL, any number is accepted. The docs no longer attribute d + '1' = the next day to PostgreSQL. It is elasticsql's own rule: PostgreSQL 16 refuses d + '1' and d - '1', and reads ts + '1' as one second. Co-Authored-By: Claude Opus 5.5 (1M context) --- documentation/sql/functions_date_time.md | 3 +- documentation/sql/operators.md | 11 +- .../elastic/sql/function/cond/package.scala | 111 +++++++++--- .../operator/math/ArithmeticExpression.scala | 74 ++++++-- .../app/softnetwork/elastic/sql/package.scala | 128 ++++++++++---- .../softnetwork/elastic/sql/query/Where.scala | 129 ++++++++++++-- .../sql/query/DateArithmeticTypeSpec.scala | 11 +- .../sql/query/TypeCheckResolutionSpec.scala | 159 ++++++++++++++++++ .../elastic/client/DateArithmeticSpec.scala | 118 ++++++++++++- 9 files changed, 660 insertions(+), 84 deletions(-) diff --git a/documentation/sql/functions_date_time.md b/documentation/sql/functions_date_time.md index fb4e2a234..528928b75 100644 --- a/documentation/sql/functions_date_time.md +++ b/documentation/sql/functions_date_time.md @@ -110,7 +110,8 @@ SELECT CURTIME() AS current_time; > answers for its date with a time of day); `DATE + n` is the date `n` days later (`DATE`, as in > PostgreSQL, DuckDB, Oracle and BigQuery) β€” a fractional `n` gives a `TIMESTAMP` whose fraction is > a time of day, as in Oracle; `TIMESTAMP + n` is `n` days later; a number written as a string -> (`'30'`) counts as that number, as PostgreSQL reads an untyped literal. `*`, `/` and `%` on a +> (`'30'`) counts as that number, by elasticsql's own rule (PostgreSQL refuses `DATE + '1'` and +> reads `TIMESTAMP + '1'` as one second). `*`, `/` and `%` on a > date, two dates added and a number minus a date are refused, as PostgreSQL, DuckDB, Oracle, Trino, > Snowflake and SQL Server refuse them. See > [Date arithmetic](operators.md#date-arithmetic--and---with-a-date-timestamp-or-datetime). diff --git a/documentation/sql/operators.md b/documentation/sql/operators.md index b8d07f288..f2e1dfb92 100644 --- a/documentation/sql/operators.md +++ b/documentation/sql/operators.md @@ -482,7 +482,7 @@ a `HAVING` over its alias, a computed column (`SCRIPT AS`) and a materialized vi | `DATE + n`, `DATE - n`, `n` a fractional number (`DOUBLE`, `REAL`, `DECIMAL`) | `n` days later / earlier, the fraction as a time of day | `TIMESTAMP` | | `TIMESTAMP + n`, `TIMESTAMP - n`, any number `n` | `n` days later / earlier | `TIMESTAMP` | | `n + temporal` | the same as `temporal + n` | as above | -| a temporal and a string literal that is a number (`'30'`, `'1.5'`) | the same as with that number | as above | +| a temporal and a string literal that is a number (`'30'`, `'1.5'`) | the same as with that number (elasticsql's own rule, not PostgreSQL's: see below) | as above | | a temporal and `NULL` | `NULL` | the type above, for a typed `NULL` (`CAST(NULL AS DATE) - d` is a `BIGINT`) | | two temporals added, a temporal under `*`, `/` or `%`, a number minus a temporal, a temporal combined with any other string or a boolean | a type error, refused before anything runs | β€” | @@ -504,6 +504,10 @@ and timestamps too: see [GREATEST](functions_conditional.md#greatest)). - PostgreSQL, DuckDB, Oracle and BigQuery add a number of days to a date; Oracle turns the fraction of `DATE + 1.5` into a time of day. - PostgreSQL, DuckDB, Oracle, Trino, Snowflake and SQL Server refuse the other combinations. +- A number written as a string beside a date or a timestamp (`order_date + '30'`) is elasticsql's + own rule, not PostgreSQL's: PostgreSQL refuses `order_date + '1'` + (`operator is not unique: date + unknown`), reads the `'1'` of `order_date - '1'` as a date and + refuses it (`invalid input syntax for type date`), and reads `created_at + '1'` as one second. **Examples:** ```sql @@ -557,7 +561,10 @@ SELECT order_date * 2 FROM orders; it spells (the date of a date-time literal), beside a `TIMESTAMP` the instant it spells (`'2024-01-31'` is its midnight), and beside a `BOOLEAN` the boolean it spells. A `DATE` and a `TIMESTAMP` compare, the `DATE` being the instant its day starts at. Any other literal is text, - and comparing it with a number or a date is refused by name. + and comparing it with a number or a date is refused by name. A number written as a string beside + a date or a timestamp in `+` or `-` (`order_date + '30'`) is not read as PostgreSQL reads it: + elasticsql counts it as that number of days, which PostgreSQL does not (see "The engines this + follows" above). - An index pattern or a comma list (`FROM logs-*`, `FROM orders_2024, orders_2025`) answers like a single index: every field its indices map alike has its type, read once per pattern from the merged mapping. ⚠️ A field two of them map differently (a `date` in one index, a `keyword` in diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 27d88a835..aa92ddafc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -359,12 +359,31 @@ package object cond { * rule (`ComparisonRule.readingOf`, the lead's ruling of 2026-10-05): `NULLIF(n, '3')` * compares `n` with the number 3, `NULLIF(COALESCE(d, d2), '2024-01-31')` with a date. `None` * for anything else, and for a literal that spells no value of the other's kind. + * + * Computed ONCE per `NULLIF` and memoised (its type, its rendering and the comparison it sits + * in all ask it; `ComparisonRule` says why with no lock); attaching the schema copies the + * `NULLIF` (`update`), so a resolved statement's reading is read against the resolved types. */ - private[cond] def secondReading: Option[ComparisonRule.Reading] = - ComparisonRule.readingOf(expr2, ComparisonRule.declaredTypeOf(expr1)) + private[cond] def secondReading: Option[ComparisonRule.Reading] = { + var reading = _secondReading + if (reading eq null) { + reading = ComparisonRule.readingOf(expr2, ComparisonRule.declaredTypeOf(expr1)) + _secondReading = reading + } + reading + } + + private[cond] def firstReading: Option[ComparisonRule.Reading] = { + var reading = _firstReading + if (reading eq null) { + reading = ComparisonRule.readingOf(expr1, ComparisonRule.declaredTypeOf(expr2)) + _firstReading = reading + } + reading + } - private[cond] def firstReading: Option[ComparisonRule.Reading] = - ComparisonRule.readingOf(expr1, ComparisonRule.declaredTypeOf(expr2)) + private[this] var _secondReading: Option[ComparisonRule.Reading] = null + private[this] var _firstReading: Option[ComparisonRule.Reading] = null /** A literal argument counts as what it is read as: `NULLIF(n, '3')` compares two numbers. */ override def argTypes: List[SQLType] = @@ -763,9 +782,15 @@ package object cond { */ override def typeError: Option[String] = typeMismatchError.orElse { - ComparisonRule.mismatch(expr1, expr2).map { case (left, right) => - s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" - } + // judged on the readings this NULLIF already holds (`firstReading`, `secondReading`) + ComparisonRule + .mismatch( + ComparisonRule.comparedType(firstReading, expr1), + ComparisonRule.comparedType(secondReading, expr2) + ) + .map { case (left, right) => + s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" + } } override def update(request: query.SingleSearch): NullIf = { @@ -840,9 +865,23 @@ package object cond { Some("CASE WHEN conditions must be of type BOOLEAN") // Each value is compared with the expression by the ONE comparison rule (`ComparisonRule`): // on their DECLARED types, a DATE comparable with a TIMESTAMP, a string literal read by - // what it is compared with -- `CASE n WHEN '3'`, `CASE d WHEN CAST('2024-01-31' AS DATE)`. - case Some(e) if conditions.exists { case (cond, _) => - ComparisonRule.mismatch(e, cond).isDefined + // what it is compared with -- `CASE n WHEN '3'`, `CASE d WHEN CAST('2024-01-31' AS DATE)` + // -- judged on the readings this CASE already holds (`whenReadings`). + case Some(e) if { + val (declared, readings) = whenReadings(e) + val literalExpression = ComparisonRule.stringLiteral(e).isDefined + conditions.zip(readings).exists { case ((cond, _), reading) => + val left = + if (!literalExpression) declared + else + ComparisonRule + .readingOf(e, ComparisonRule.declaredTypeOf(cond)) + .map(_.sqlType) + .getOrElse(declared) + ComparisonRule + .mismatch(left, ComparisonRule.comparedType(reading, cond)) + .isDefined + } } => Some("CASE WHEN conditions must be of the same type as the expression") case _ => None @@ -944,9 +983,7 @@ package object cond { ): WhenComparison = { def dateCarrying(t: SQLType): Boolean = t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp - val declared = ComparisonRule.declaredTypeOf(expr) - val readings = - conditions.map { case (cond, _) => ComparisonRule.readingOf(cond, declared) } + val (declared, readings) = whenReadings(expr) val processor = context.exists(_.isProcessor) if (receiverIsTime) WhenAsResult else if (dateCarrying(declared)) { @@ -966,6 +1003,41 @@ package object cond { } else WhenAsResult } + /** The CASE expression's DECLARED type and what each `WHEN` value is read as beside it + * (`ComparisonRule.readingOf`), computed ONCE per CASE and memoised (`ComparisonRule` says why + * with no lock): its type rule, the comparison it is rendered as and every `WHEN` arm ask it, + * and a reading is a parse. Attaching the schema copies the CASE (`update`), so a resolved + * statement's readings are read against the resolved types. + */ + private[this] def expressionReadings: (SQLType, List[Option[ComparisonRule.Reading]]) = { + var readings = _expressionReadings + if (readings eq null) { + readings = expression match { + case Some(e) => readingsBeside(e) + case None => (SQLTypes.Any, conditions.map(_ => None)) + } + _expressionReadings = readings + } + readings + } + + private[this] var _expressionReadings: (SQLType, List[Option[ComparisonRule.Reading]]) = null + + private[this] def readingsBeside( + expr: PainlessScript + ): (SQLType, List[Option[ComparisonRule.Reading]]) = { + val declared = ComparisonRule.declaredTypeOf(expr) + (declared, conditions.map { case (cond, _) => ComparisonRule.readingOf(cond, declared) }) + } + + /** [[expressionReadings]] when `expr` IS the CASE expression -- the only one the renderings + * pass -- and computed for `expr` otherwise. + */ + private[this] def whenReadings( + expr: PainlessScript + ): (SQLType, List[Option[ComparisonRule.Reading]]) = + if (expression.exists(_ eq expr)) expressionReadings else readingsBeside(expr) + /** The JAVA type an operand renders (#384's `Identifier.renderedType`): a DATE column's doc * value is a `ZonedDateTime`, a cast to DATE a `LocalDate`. */ @@ -1008,7 +1080,7 @@ package object cond { ctx: PainlessContext, context: Option[PainlessContext] ): String = { - val declared = ComparisonRule.declaredTypeOf(expr) + val (_, readings) = whenReadings(expr) def bind(rendered: String): String = ctx.addParam(LiteralParam(rendered)).getOrElse(rendered) def instant(operand: PainlessScript, ref: String): String = bind( @@ -1017,9 +1089,10 @@ package object cond { val e = bind(expr.painless(context)) val key = if (kind == WhenInstants) instant(expr, e) else e conditions - .map { case (cond, res) => + .zip(readings) + .map { case ((cond, res), reading) => val r = whenResult(cond, res, context) - val test = ComparisonRule.readingOf(cond, declared) match { + val test = reading match { case Some(t: ComparisonRule.TemporalReading) => s"$key == ${t.epochMillis}L" case Some(scalar) => s"$key == ${ComparisonRule.scalarPainless(scalar).getOrElse(cond.painless(context))}" @@ -1046,7 +1119,8 @@ package object cond { LiteralParam(e) ) conditions - .map { case (cond, res) => + .zip(whenReadings(expr)._2) + .map { case ((cond, res), whenReading) => val name = cond match { case e: Expression => @@ -1062,8 +1136,7 @@ package object cond { // two are compared in like any other value. `CASE b WHEN 'true'` used to parse // `"true"` as a long (`Long.parseLong`) and fail the shard. val c = - ComparisonRule - .readingOf(cond, ComparisonRule.declaredTypeOf(expr)) + whenReading .flatMap(reading => ComparisonRule .scalarPainless(reading) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 692e39b73..66e213339 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -102,14 +102,67 @@ case class ArithmeticExpression( /** A string LITERAL beside a NUMBER is the number it spells -- the ONE comparison rule's reading * (`ComparisonRule.readingOf`), as PostgreSQL reads `n + '1'`: 4 for n = 3, where the - * concatenation `"31"` was answered. Beside anything else it stays what it is (a temporal beside - * it is date arithmetic, which reads it itself). + * concatenation `"31"` was answered. Beside a whole-number operand only a whole number is one: + * PostgreSQL refuses `n + '1.5'` over an integer `n` (`invalid input syntax for type integer`), + * and so does the operand match here, the literal staying text. Beside anything else it stays + * what it is (a temporal beside it is date arithmetic, which reads it itself). + * + * Each operand's reading is computed ONCE per expression and memoised (`ComparisonRule` says why + * with no lock): every `argTypes` (so every `out`) and every rendering asks it, and a reading is + * a parse. */ private def literalNumber(operand: PainlessScript): Option[ComparisonRule.NumberReading] = + if (operand eq left) { + var number = _leftNumber + if (number eq null) { + number = numberBeside(left, right) + _leftNumber = number + } + number + } else if (operand eq right) { + var number = _rightNumber + if (number eq null) { + number = numberBeside(right, left) + _rightNumber = number + } + number + } else numberBeside(operand, left) + + private[this] var _leftNumber: Option[ComparisonRule.NumberReading] = null + private[this] var _rightNumber: Option[ComparisonRule.NumberReading] = null + + private def numberBeside( + operand: PainlessScript, + other: PainlessScript + ): Option[ComparisonRule.NumberReading] = ComparisonRule - .readingOf(operand, ComparisonRule.declaredTypeOf(if (operand eq left) right else left)) + .readingOf(operand, ComparisonRule.declaredTypeOf(other)) .collect { case number: ComparisonRule.NumberReading => number } + /** The temporal a string literal operand is read as beside the other one + * (`ArithmeticExpression.temporalString`: `d - '2024-01-31'`), computed ONCE per expression and + * memoised -- the date-arithmetic rendering asks it per operand and per guard. + */ + private def literalTemporal(operand: PainlessScript): Option[ComparisonRule.TemporalReading] = + if (operand eq left) { + var temporal = _leftTemporal + if (temporal eq null) { + temporal = ArithmeticExpression.temporalString(left, right) + _leftTemporal = temporal + } + temporal + } else if (operand eq right) { + var temporal = _rightTemporal + if (temporal eq null) { + temporal = ArithmeticExpression.temporalString(right, left) + _rightTemporal = temporal + } + temporal + } else ArithmeticExpression.temporalString(operand, left) + + private[this] var _leftTemporal: Option[ComparisonRule.TemporalReading] = null + private[this] var _rightTemporal: Option[ComparisonRule.TemporalReading] = null + override def argTypes: List[SQLType] = args.map(argTypeOf) /** πŸ”΄ `/` decides its OWN result type: a division of numbers is FLOATING-POINT whatever its @@ -666,11 +719,9 @@ case class ArithmeticExpression( // A string literal read as the temporal it spells beside the other operand (`d - // '2024-01-31'`): the instant it denotes, known here, so it is a constant -- never NULL, never - // parsed per document. + // parsed per document (and read once per expression, `literalTemporal`). def literalInstant(operand: PainlessScript): Option[Long] = - ArithmeticExpression - .temporalString(operand, if (operand eq left) right else left) - .map(_.epochMillis) + literalTemporal(operand).map(_.epochMillis) // a nullable operand the guard reads is evaluated once, under a name def named(operand: PainlessScript, fallback: String): String = @@ -943,9 +994,12 @@ object ArithmeticExpression { * is fractional (the lead's ruling of 2026-10-05) -- the ONE reader the comparison rule uses too * (`ComparisonRule.numberOf`). * - * PostgreSQL reads an untyped literal by what it is added to: `d + '1'` is the next day and `d - - * '1.5'` a day and a half earlier. A whole number is a `long` literal (`'99999999999'` does not - * fit an `int`), a fractional one the `double` it denotes. + * Beside a DATE or a TIMESTAMP it is a number of days -- elasticsql's OWN rule, not + * PostgreSQL's: `d + '1'` is the next day and `d - '1.5'` a day and a half earlier, where + * PostgreSQL 16 refuses `d + '1'` (`operator is not unique: date + unknown`), reads the `'1'` of + * `d - '1'` as a date (`invalid input syntax for type date`) and reads `ts + '1'` as one second. + * A whole number is a `long` literal (`'99999999999'` does not fit an `int`), a fractional one + * the `double` it denotes. */ private[math] def numericString(operand: PainlessScript): Option[(String, Boolean)] = ComparisonRule diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index e2256373a..88e76f840 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1606,10 +1606,17 @@ package object sql { * `renderedTypeOf` answers. A DATE and a TIMESTAMP are comparable * (`SQLTypeUtils.comparableTemporals`): the DATE is the instant its UTC day starts at. * - An untyped string LITERAL is read by what it is compared with, as PostgreSQL reads one - * ([[ComparisonRule.readingOf]]): beside a number, the number it spells; beside a temporal, - * the date or the timestamp it spells -- beside a DATE, a date-time literal is the date it - * names, as PostgreSQL's date input reads it; beside a BOOLEAN, the boolean it spells. Any - * other literal stays text, and the comparison is judged as such. + * ([[ComparisonRule.readingOf]]): beside a number, the number it spells when that number is + * a value of the other's type -- beside a whole-number type, a whole number only; beside a + * temporal, the date or the timestamp it spells -- beside a DATE, a date-time literal is the + * date it names, as PostgreSQL's date input reads it; beside a BOOLEAN, the boolean it + * spells. Any other literal stays text, and the comparison is judged as such. + * + * A reading costs a parse, so each site computes a literal's reading ONCE per node, never per + * type question or per rendering: memoised in a field of the node, with no lock -- a reading is + * pure and immutable, so a race can only compute it twice, where a `lazy val`'s lock costs more + * than the reading on every fresh node. And a text that cannot spell a date never reaches a + * `java.time` parser and the exception it throws. */ object ComparisonRule { @@ -1674,6 +1681,9 @@ package object sql { .ofPattern("uuuu-MM-dd") .withResolverStyle(java.time.format.ResolverStyle.STRICT) + private val IsoDateTimeUtc = + java.time.format.DateTimeFormatter.ISO_DATE_TIME.withZone(java.time.ZoneOffset.UTC) + private val DayMillis = 86400000L /** The number a text spells, if it spells one. */ @@ -1691,20 +1701,75 @@ package object sql { case _ => None } + /** The number a text spells beside an operand of the numeric type `against`, if it is a value + * of that type, as PostgreSQL's input for the type reads it (the lead's ruling of 2026-10-05): + * - beside a whole-number type (TINYINT, SMALLINT, INT, BIGINT), a whole number BIGINT can + * hold -- `'1.5'`, `'1.0'` and `'1e3'` are no integer (PostgreSQL: `invalid input syntax + * for type integer`), and a whole number past BIGINT's range is a value of no integer type + * (`out of range`); + * - beside DOUBLE, REAL or NUMERIC, any number. + * + * A literal that is no value of the type stays text, and the comparison or the arithmetic that + * reads it is refused as text against that type -- as PostgreSQL refuses `n = '1.5'` and `n + + * '1.5'` over an integer `n`. + */ + def numberOf(text: String, against: SQLNumeric): Option[NumberReading] = + numberOf(text) match { + case Some(number) if number.fractional && wholeNumberType(against) => None + case reading => reading + } + + private def wholeNumberType(t: SQLType): Boolean = t match { + case _: SQLTinyInt | _: SQLSmallInt | _: SQLInt | _: SQLBigInt => true + case _ => false + } + + /** Can the trimmed `text` spell a `uuuu-MM-dd` date (a `/` for the `-`)? A NECESSARY condition + * of the parser below, never a sufficient one: that parser reads a year of at least four + * digits (strict parsing), a sign at most, `-`, two digits, `-`, two digits -- ten characters + * or more of digits, `-`, `/` and `+`, and nothing else. A text that fails it never reaches + * the parser, whose refusal is an EXCEPTION: `'1'` beside a date cost two of them, 2 Β΅s. + */ + private def maySpellDate(text: String): Boolean = + text.length >= 10 && { + var i = 0 + var ok = true + while (ok && i < text.length) { + val c = text.charAt(i) + ok = (c >= '0' && c <= '9') || c == '-' || c == '/' || c == '+' + i += 1 + } + ok + } + + /** Can the trimmed `text` spell an ISO date-time (a space for the `T`)? A NECESSARY condition + * of the parser below: a year that starts with a digit or a sign, a date of ten characters or + * more, a `T` (any case, or the space read as one), and an `HH:mm` time at least -- sixteen + * characters, a `:`. + */ + private def maySpellDateTime(text: String): Boolean = + text.length >= 16 && { + val c = text.charAt(0) + (c >= '0' && c <= '9') || c == '-' || c == '+' + } && text.indexOf(':') > 0 && + (text.indexOf('T') > 0 || text.indexOf('t') > 0 || text.indexOf(' ') > 0) + /** The date a text spells -- `yyyy-MM-dd`, a `/` accepted for the `-` -- if it spells one. */ - private def dateOf(text: String): Option[java.time.LocalDate] = - Try(java.time.LocalDate.parse(text.trim.replace("/", "-"), IsoDate)).toOption + private def dateOf(text: String): Option[java.time.LocalDate] = { + val trimmed = text.trim + if (!maySpellDate(trimmed)) None + else Try(java.time.LocalDate.parse(trimmed.replace("/", "-"), IsoDate)).toOption + } /** The date-time an ISO text spells (a space accepted for the `T`), in UTC unless it names a * zone, if it spells one. */ - private def dateTimeOf(text: String): Option[java.time.ZonedDateTime] = - Try( - java.time.ZonedDateTime.parse( - text.trim.replace(" ", "T"), - java.time.format.DateTimeFormatter.ISO_DATE_TIME.withZone(java.time.ZoneOffset.UTC) - ) - ).toOption + private def dateTimeOf(text: String): Option[java.time.ZonedDateTime] = { + val trimmed = text.trim + if (!maySpellDateTime(trimmed)) None + else + Try(java.time.ZonedDateTime.parse(trimmed.replace(" ", "T"), IsoDateTimeUtc)).toOption + } private def dateReading(date: java.time.LocalDate): TemporalReading = TemporalReading(SQLTypes.Date, date.toString, date.toEpochDay * DayMillis) @@ -1746,13 +1811,14 @@ package object sql { } /** What `literal` is read as beside an operand DECLARED `against`, if it is an untyped string - * literal that spells a value of that kind; `None` otherwise -- it stays text. An operand of - * UNKNOWN type (no schema attached) reads no literal: nothing decides what it should be. + * literal that spells a value of that type (`numberOf(text, against)` for a number); `None` + * otherwise -- it stays text. An operand of UNKNOWN type (no schema attached) reads no + * literal: nothing decides what it should be. */ def readingOf(literal: Token, against: => SQLType): Option[Reading] = stringLiteral(literal).flatMap { text => against match { - case _: SQLNumeric => numberOf(text) + case n: SQLNumeric => numberOf(text, n) case t: SQLTemporal if t != SQLTypes.Time && t != SQLTypes.Temporal => temporalOf(text, t) case SQLTypes.Temporal => temporalOf(text, SQLTypes.Timestamp) @@ -1761,24 +1827,26 @@ package object sql { } } - /** The type `operand` is compared AS beside `other`: what a literal reads as, the declared type - * of anything else. - */ - def comparedType(operand: Token, other: Token): SQLType = - readingOf(operand, declaredTypeOf(other)) - .map(_.sqlType) - .getOrElse(declaredTypeOf(operand)) - /** May two compared types be compared? */ def comparable(left: SQLType, right: SQLType): Boolean = SQLTypeUtils.matches(left, right) || SQLTypeUtils.comparableTemporals(left, right) - /** The two compared types, when they cannot be compared. */ - def mismatch(left: Token, right: Token): Option[(SQLType, SQLType)] = { - val l = comparedType(left, right) - val r = comparedType(right, left) - if (comparable(l, r)) None else Some((l, r)) - } + /** The type an operand is compared AS: what it is read as beside the other operand + * ([[readingOf]], its `reading`), else the type it is declared with ([[declaredTypeOf]]), + * asked only then. + * + * A site passes the reading it holds -- computed ONCE per node, for its type rule and its + * rendering alike -- so a literal is never read a second time to be judged. + */ + def comparedType(reading: Option[Reading], operand: Token): SQLType = + reading match { + case Some(r) => r.sqlType + case None => declaredTypeOf(operand) + } + + /** The two compared types ([[comparedType]]), when they cannot be compared. */ + def mismatch(left: SQLType, right: SQLType): Option[(SQLType, SQLType)] = + if (comparable(left, right)) None else Some((left, right)) /** The Painless a NUMBER or a BOOLEAN reading renders: the literal itself. */ def scalarPainless(reading: Reading): Option[String] = reading match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index cd0a2a218..7463e4d7b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -990,15 +990,45 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { * number or the instant it spells: nothing else can be compared with a metric. `HAVING MAX(n) > * '4'` therefore renders the same comparison before and after resolution; the type rule judges * it once the type is known. + * + * Computed ONCE per comparison and memoised (`ComparisonRule` says why with no lock): the type + * rule and every type question of the rendering ([[valueType]], [[comparisonTargetType]], + * `check`'s dispatch) ask it, and a reading is a parse. Attaching the schema COPIES the + * comparison (`update`), so the reading a resolved statement holds is read against the resolved + * types. + */ + protected def literalReading: Option[ComparisonRule.Reading] = { + var reading = _literalReading + if (reading eq null) { + reading = maybeValue.flatMap(readingFor) + _literalReading = reading + } + reading + } + + private[this] var _literalReading: Option[ComparisonRule.Reading] = null + + /** The type the left operand is DECLARED with (`ComparisonRule.declaredTypeOf`), which every + * literal of this comparison is read against and the type rule judges: computed ONCE, and + * memoised like [[literalReading]]. */ - protected def literalReading: Option[ComparisonRule.Reading] = maybeValue.flatMap(readingFor) + protected def operandDeclaredType: SQLType = { + var declared = _operandDeclaredType + if (declared eq null) { + declared = ComparisonRule.declaredTypeOf(identifier) + _operandDeclaredType = declared + } + declared + } + + private[this] var _operandDeclaredType: SQLType = null /** What `value` -- the right-hand side, a `BETWEEN` bound, an `IN` element -- is read as against * this comparison's left operand ([[literalReading]]). */ protected def readingFor(value: Token): Option[ComparisonRule.Reading] = ComparisonRule.stringLiteral(value).flatMap { text => - val declared = ComparisonRule.declaredTypeOf(identifier) + val declared = operandDeclaredType ComparisonRule.readingOf(value, declared).orElse { if (identifier.hasAggregation && declared.isUnknown) ComparisonRule.numberOf(text).orElse(ComparisonRule.temporalOf(text, SQLTypes.Timestamp)) @@ -1988,10 +2018,31 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { maybeValue.flatMap { value => if (!scripted && Expression.literalAgainstBareColumn(identifier, value)) None else - ComparisonRule.mismatch(identifier, value).map { case (left, right) => + mismatchWith(value, literalReading).map { case (left, right) => s"Type mismatch: '${left.typeId}' is not compatible with '${right.typeId}' in expression: $this" } } + + /** The ONE comparison rule (`ComparisonRule.mismatch`) between the left operand and `value` -- + * the right-hand side, a `BETWEEN` bound, an `IN` element -- judged with what this comparison + * already holds: `value`'s reading (`reading`, computed once) and the left operand's declared + * type ([[operandDeclaredType]]), so a literal is never read a second time to be judged. + */ + protected def mismatchWith( + value: Token, + reading: Option[ComparisonRule.Reading] + ): Option[(SQLType, SQLType)] = { + // the left operand is itself a string literal only in `'2024-01-01' < d`: then it is read + // beside `value`, as `value` is beside it + val left = + if (ComparisonRule.stringLiteral(identifier).isEmpty) operandDeclaredType + else + ComparisonRule + .readingOf(identifier, ComparisonRule.declaredTypeOf(value)) + .map(_.sqlType) + .getOrElse(operandDeclaredType) + ComparisonRule.mismatch(left, ComparisonRule.comparedType(reading, value)) + } } object Expression { @@ -2309,21 +2360,44 @@ case class InExpr[R, +T <: Value[R]]( if (!scripted && Expression.bareColumn(identifier)) None else values.values - .flatMap(element => ComparisonRule.mismatch(identifier, element)) + .zip(readings) + .flatMap { case (element, reading) => mismatchWith(element, reading) } .headOption .map { case (left, right) => s"Type mismatch: '${left.typeId}' is not compatible with '${right.typeId}' in expression: $this" } + /** What each element is READ as (`readingFor`), computed ONCE per IN and memoised (like + * `literalReading`): the type rule, the type questions, both renderings and the group filter all + * ask it. + */ + private def readings: Seq[Option[ComparisonRule.Reading]] = { + var computed = _readings + if (computed eq null) { + computed = values.values.map(readingFor) + _readings = computed + } + computed + } + + private[this] var _readings: Seq[Option[ComparisonRule.Reading]] = null + /** What every element is READ as (`readingFor`), when each one is a string literal the operand * reads -- the list is then the numbers, the dates, the timestamps or the booleans they spell. */ - private def elementReadings: Option[Seq[ComparisonRule.Reading]] = - if (values.values.isEmpty) None - else { - val readings = values.values.map(readingFor) - if (readings.forall(_.isDefined)) Some(readings.flatten) else None + private def elementReadings: Option[Seq[ComparisonRule.Reading]] = { + var computed = _elementReadings + if (computed eq null) { + computed = + if (values.values.isEmpty) None + else if (readings.forall(_.isDefined)) Some(readings.flatten) + else None + _elementReadings = computed } + computed + } + + private[this] var _elementReadings: Option[Seq[ComparisonRule.Reading]] = null /** The type the list is compared IN when its elements are read ([[elementReadings]]), as the * right-hand side of a single comparison is; the list's own type otherwise, as it always was. @@ -2406,8 +2480,9 @@ case class InExpr[R, +T <: Value[R]]( // the epoch milliseconds, it is read as -- a metric is a number (`readingFor`). override protected def bucketPipelineCheck(param: String): String = values.values - .map { v => - val element = readingFor(v) match { + .zip(readings) + .map { case (v, reading) => + val element = reading match { case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" case Some(scalar) => ComparisonRule.scalarPainless(scalar).getOrElse(v.painless(None)) @@ -2816,17 +2891,39 @@ case class BetweenExpr( if (!scripted && Expression.bareColumn(identifier)) fromTo.typeError else Seq(fromTo.from, fromTo.to) - .flatMap(bound => ComparisonRule.mismatch(identifier, bound)) + .flatMap(bound => mismatchWith(bound, boundReading(bound))) .headOption .map { case (left, right) => s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" } + /** What each bound is READ as (`readingFor`), computed ONCE per BETWEEN and memoised (like + * `literalReading`): the type rule, the type questions, both renderings and the group filter all + * ask it. + */ + private def boundReadings: (Option[ComparisonRule.Reading], Option[ComparisonRule.Reading]) = { + var computed = _boundReadings + if (computed eq null) { + computed = (readingFor(fromTo.from), readingFor(fromTo.to)) + _boundReadings = computed + } + computed + } + + private[this] var _boundReadings + : (Option[ComparisonRule.Reading], Option[ComparisonRule.Reading]) = + null + + private def boundReading(bound: Token): Option[ComparisonRule.Reading] = + if (bound eq fromTo.from) boundReadings._1 + else if (bound eq fromTo.to) boundReadings._2 + else readingFor(bound) + /** The SQL type of a bound as it is compared: its reading, or its own type -- the question * `valueType` answers for the right-hand side of a single comparison. */ private def boundType(bound: Token): SQLType = - readingFor(bound) + boundReading(bound) .map(_.sqlType) .getOrElse(bound match { case id: Identifier => id.chainType @@ -2835,7 +2932,7 @@ case class BetweenExpr( /** The JAVA type a bound renders -- the question `valueRenderedType` answers for a single one. */ private def boundRenderedType(bound: Token): SQLType = - readingFor(bound) + boundReading(bound) .map(_.sqlType) .getOrElse(bound match { case id: Identifier => id.renderedType @@ -2879,7 +2976,7 @@ case class BetweenExpr( op: Operator ): String = { def bound(b: Token): String = { - val rendered = readingPainless(readingFor(b), b, context) + val rendered = readingPainless(boundReading(b), b, context) val from = boundRenderedType(b) val to = comparisonTargetType if ( @@ -2900,7 +2997,7 @@ case class BetweenExpr( // `bucketPipelinePainless` already wraps this call in it. A string bound is the number, or the // epoch milliseconds, it is read as -- a metric is a number (`readingFor`). override protected def bucketPipelineCheck(param: String): String = { - def bound(b: Token): String = readingFor(b) match { + def bound(b: Token): String = boundReading(b) match { case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" case Some(scalar) => ComparisonRule.scalarPainless(scalar).getOrElse(b.toString) case None => b.toString diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala index 0315ee595..ac1559c00 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala @@ -51,10 +51,13 @@ import org.scalatest.matchers.should.Matchers * or arguments, `NULLIF` its first argument -- `GREATEST` / `LEAST` over dates included, a * DATE when every argument is one, a TIMESTAMP when they mix; * - a typed NULL is its type (`CAST(NULL AS DATE)` is a DATE), a bare `NULL` has none; - * - a string literal that is a number is that number (`'1'`, `'1.5'`); one that spells a date or - * a timestamp is, beside a temporal, that temporal read as the other operand's type (`d - - * '2024-01-31'` is DATE minus DATE, as PostgreSQL reads it) and a string beside anything else; - * any other is a string; + * - a string literal that is a number is that number (`'1'`, `'1.5'`), so a number of days + * beside a temporal -- elasticsql's OWN rule, not PostgreSQL's: PostgreSQL 16 refuses `d + + * '1'` (`operator is not unique: date + unknown`), reads the `'1'` of `d - '1'` as a date + * (`invalid input syntax for type date`) and reads `ts + '1'` as one second; one that spells a + * date or a timestamp is, beside a temporal, that temporal read as the other operand's type + * (`d - '2024-01-31'` is DATE minus DATE, as PostgreSQL reads it) and a string beside anything + * else; any other is a string; * - a `date` field elasticsql never declared (`u`, an index created by a bulk load) holds * instants: a TIMESTAMP, typed so at the declaration seam -- DESCRIBE shows it so too. */ diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala index b9780fa9e..776158c51 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} import app.softnetwork.elastic.sql.`type`.SQLTypes @@ -45,7 +46,9 @@ class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDriven Column("g", SQLTypes.Keyword), Column("s", SQLTypes.Keyword), Column("n", SQLTypes.Int), + Column("big", SQLTypes.BigInt), Column("x", SQLTypes.Double), + Column("r", SQLTypes.Real), Column("d", SQLTypes.Date), Column("d2", SQLTypes.Date), Column("ts", SQLTypes.Timestamp), @@ -203,4 +206,160 @@ class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDriven "is not compatible with 'VARCHAR'" ) } + + /** A quoted number is read as a VALUE of the other operand's type, as PostgreSQL's input for that + * type reads it (the lead's ruling of 2026-10-05). Beside a whole-number type, `'1.5'` and + * `'1.0'` are no integer -- PostgreSQL 16: `invalid input syntax for type integer`, for a + * comparison and for `n + '1.5'` alike -- so the literal stays text and is refused by name at + * every site that reads one; beside DOUBLE, REAL or DECIMAL (a DOUBLE here) any number is one. + */ + "a quoted number beside a whole-number operand" should + "be a whole number, refused by name otherwise, and any number beside a fractional one" in { + forAll( + Table( + ("site", "sql", "refusal"), + ( + "comparison (INT)", + "SELECT id FROM t WHERE NULLIF(n, 0) = '1.5'", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "comparison (INT)", + "SELECT id FROM t WHERE NULLIF(n, 0) = '1.0'", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "comparison (BIGINT)", + "SELECT id FROM t WHERE COALESCE(big, 0) = '1.5'", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "comparison (BIGINT)", + "SELECT id FROM t WHERE COALESCE(big, 0) = '1.0'", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "CASE condition", + "SELECT id, CASE WHEN COALESCE(n, 0) = '1.5' THEN 1 ELSE 0 END AS v FROM t", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "an exponent", + "SELECT id FROM t WHERE NULLIF(n, 0) = '1e3'", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "past BIGINT", + "SELECT id FROM t WHERE COALESCE(big, 0) = '99999999999999999999'", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "IN", + "SELECT id FROM t WHERE NULLIF(n, 0) IN ('1', '1.5')", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "IN (HAVING)", + "SELECT g FROM t GROUP BY g HAVING MAX(big) IN ('1', '1.0')", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "BETWEEN", + "SELECT id FROM t WHERE COALESCE(big, 0) BETWEEN '1' AND '1.0'", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + ("NULLIF (INT)", "SELECT NULLIF(n, '1.5') AS v FROM t", "compares INT with VARCHAR"), + ( + "NULLIF (BIGINT)", + "SELECT NULLIF(big, '1.0') AS v FROM t", + "compares BIGINT with VARCHAR" + ), + ( + "CASE WHEN (INT)", + "SELECT CASE n WHEN '1.5' THEN 1 ELSE 0 END AS v FROM t", + "CASE WHEN conditions must be of the same type as the expression" + ), + ( + "CASE WHEN (BIGINT)", + "SELECT CASE big WHEN '1.0' THEN 1 ELSE 0 END AS v FROM t", + "CASE WHEN conditions must be of the same type as the expression" + ), + // a group filter compares the metric, which is no text + ( + "HAVING", + "SELECT g FROM t GROUP BY g HAVING MAX(n) > '4.5'", + "HAVING cannot be applied to MAX(n) > '4.5'" + ), + ( + "arithmetic (INT)", + "SELECT n + '1.5' AS v FROM t", + "output 'INT' is not compatible with input 'VARCHAR'" + ), + ( + "arithmetic (INT, literal first)", + "SELECT '1.0' * n AS v FROM t", + "output 'VARCHAR' is not compatible with input 'INT'" + ), + ( + "arithmetic (BIGINT)", + "SELECT big - '1.0' AS v FROM t", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + ( + "arithmetic (per group)", + "SELECT g, SUM(n) + '1.5' AS v FROM t GROUP BY g", + "output 'INT' is not compatible with input 'VARCHAR'" + ) + ) + ) { (site, sql, refusal) => + withClue(s"[$site] [$sql] ") { + parsed(sql) + resolved(sql).swap.getOrElse(fail("accepted once the types are known")) should include( + refusal + ) + } + } + forAll( + Table( + "sql", + // a whole number is an integer, however it is written + "SELECT id FROM t WHERE NULLIF(n, 0) = '1'", + "SELECT id FROM t WHERE NULLIF(n, 0) = ' 1 '", + "SELECT id FROM t WHERE NULLIF(n, 0) = '+1'", + "SELECT id FROM t WHERE COALESCE(big, 0) = '10000000000'", + "SELECT CASE big WHEN '2' THEN 1 ELSE 0 END AS v FROM t", + "SELECT n + '1' AS v FROM t", + // beside DOUBLE, REAL or DECIMAL, any number + "SELECT id FROM t WHERE COALESCE(x, 0) = '1.5'", + "SELECT id FROM t WHERE COALESCE(x, 0) = '1e0'", + "SELECT id FROM t WHERE COALESCE(r, 0) = '1.0'", + "SELECT id FROM t WHERE CAST(s AS DECIMAL(10,2)) = '1.5'", + "SELECT id FROM t WHERE COALESCE(x, 0) IN ('1', '1.5')", + "SELECT id FROM t WHERE COALESCE(x, 0) BETWEEN '1.0' AND '1.5'", + "SELECT NULLIF(x, '1.5') AS v FROM t", + "SELECT CASE x WHEN '1.5' THEN 1 ELSE 0 END AS v FROM t", + "SELECT x + '1.5' AS v FROM t", + "SELECT g, AVG(n) + '1.5' AS v FROM t GROUP BY g", + "SELECT g FROM t GROUP BY g HAVING AVG(n) > '4.5'", + // beside a temporal a number is a number of days, whatever its fraction + "SELECT d + '1.5' AS v FROM t" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + } + + /** The literal reading a comparison renders is the one its type rule judged: a DOUBLE operand + * compares the number a quoted `'1.5'` spells, and an integer one the whole number `'1'` spells. + */ + "a quoted number read beside a number" should "render as that number" in { + def criteriaPainless(sql: String): String = { + val search = parsed(sql).update(Some(schema)) + search.validateResolved() shouldBe Right(()) + search.where + .flatMap(_.criteria) + .map(_.painless(Some(PainlessContext(context = PainlessContextType.Query)))) + .getOrElse(fail(s"[$sql] no WHERE")) + } + criteriaPainless("SELECT id FROM t WHERE COALESCE(x, 0) = '1.5'") should include("== 1.5") + criteriaPainless("SELECT id FROM t WHERE COALESCE(big, 0) = '1'") should include("== 1L") + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala index f86c667db..b32e5f2c5 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala @@ -95,6 +95,9 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with /** The comparison rule's fixture (`"the ONE comparison rule"`). */ private val comparisonTable = "date_arithmetic_cmp" + /** A quoted number beside an INT, a BIGINT and a DOUBLE (`"a quoted number"`). */ + private val numberTable = "date_arithmetic_num" + /** An index created by a bulk load, then altered through elasticsql. */ private val alteredRawIndex = "date_arithmetic_raw_altered" @@ -297,6 +300,7 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with disagreeB, dateTimeTable, comparisonTable, + numberTable, alteredRawIndex, grownTable, refreshedA, @@ -1236,9 +1240,10 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with */ private def comparisonVerdict( expression: String, - expected: Map[String, Expected] + expected: Map[String, Expected], + from: String = comparisonTable ): (String, Either[String, Seq[String]]) = { - val sql = s"SELECT id, $expression AS x FROM $comparisonTable" + val sql = s"SELECT id, $expression AS x FROM $from" sql -> gatewayRows(sql).map { rs => val got = byKey(rs, "id") expected.toSeq.sortBy(_._1).flatMap { case (id, want) => @@ -1349,6 +1354,115 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with ) } + /** The documents of [[numberTable]]: an INT, a BIGINT past INT's range, a DOUBLE with a fraction, + * and a row holding none of them. + */ + private val numberDocs = List( + """{"id":"r1","n":1,"big":1,"x":1.5}""", + """{"id":"r2","n":2,"big":10000000000,"x":2.0}""", + """{"id":"r3","n":-3,"big":-3,"x":-1.25}""", + """{"id":"r4"}""" + ) + + "a quoted number" should + "be a value of the other operand's type, as PostgreSQL reads it: any number beside a DOUBLE, a whole one beside an integer" in { + run(s"CREATE TABLE $numberTable (id KEYWORD, n INT, big BIGINT, x DOUBLE, PRIMARY KEY (id))") + bulk(numberTable, numberDocs) + def values(r1: Expected, r2: Expected, r3: Expected, r4: Expected): Map[String, Expected] = + Map("r1" -> r1, "r2" -> r2, "r3" -> r3, "r4" -> r4) + def refused(sql: String, refusal: String): (String, Either[String, Seq[String]]) = + sql -> (Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) if error.message.contains(refusal) => Right(Nil) + case Right(ElasticFailure(error)) => Right(Seq(s"refused as [${error.message}]")) + case Right(other) => Right(Seq(s"answered $other instead of the refusal by name")) + case Left(t) => Left(t.toString) + }) + verdict( + "quoted numbers", + Seq( + // beside a DOUBLE, any number: x = 1.5, 2.0, -1.25 and NULL + comparisonVerdict( + "CASE WHEN COALESCE(x, 0) = '1.5' THEN 1 ELSE 0 END", + values(Num(1), Num(0), Num(0), Num(0)), + numberTable + ), + comparisonVerdict( + "CASE WHEN COALESCE(x, 0) = '2' THEN 1 ELSE 0 END", + values(Num(0), Num(1), Num(0), Num(0)), + numberTable + ), + comparisonVerdict( + "CASE x WHEN '1.5' THEN 1 ELSE 0 END", + values(Num(1), Num(0), Num(0), Num(0)), + numberTable + ), + comparisonVerdict( + "NULLIF(x, '1.5')", + values(Null, Num(2.0), Num(-1.25), Null), + numberTable + ), + comparisonVerdict("x + '1.5'", values(Num(3.0), Num(3.5), Num(0.25), Null), numberTable), + comparisonVerdict( + "x * '2.5'", + values(Num(3.75), Num(5.0), Num(-3.125), Null), + numberTable + ), + kept(s"SELECT id FROM $numberTable WHERE COALESCE(x, 0) IN ('1.5', '3')", Set("r1")), + kept( + s"SELECT id FROM $numberTable WHERE COALESCE(x, 0) BETWEEN '1.0' AND '1.5'", + Set("r1") + ), + kept(s"SELECT id FROM $numberTable WHERE COALESCE(x, 0) > '1.75'", Set("r2")), + // beside an integer, a whole number is one -- past INT's range too, beside a BIGINT + comparisonVerdict( + "CASE WHEN COALESCE(big, 0) = '10000000000' THEN 1 ELSE 0 END", + values(Num(0), Num(1), Num(0), Num(0)), + numberTable + ), + comparisonVerdict("n + '1'", values(Num(2), Num(3), Num(-2), Null), numberTable), + // beside an integer, '1.5' and '1.0' are no integer: refused by name, as PostgreSQL 16 + // refuses them (`invalid input syntax for type integer`) + refused( + s"SELECT id, CASE WHEN COALESCE(n, 0) = '1.5' THEN 1 ELSE 0 END AS x FROM $numberTable", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id, CASE WHEN NULLIF(n, 0) = '1.0' THEN 1 ELSE 0 END AS x FROM $numberTable", + "'INT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id, CASE WHEN COALESCE(big, 0) = '1.5' THEN 1 ELSE 0 END AS x FROM $numberTable", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id, CASE WHEN COALESCE(big, 0) = '1.0' THEN 1 ELSE 0 END AS x FROM $numberTable", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id FROM $numberTable WHERE COALESCE(n, 0) IN ('1', '1.5')", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id FROM $numberTable WHERE COALESCE(big, 0) BETWEEN '1' AND '1.0'", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + refused(s"SELECT id, NULLIF(n, '1.5') AS x FROM $numberTable", "compares INT with VARCHAR"), + refused( + s"SELECT id, CASE big WHEN '1.0' THEN 1 ELSE 0 END AS x FROM $numberTable", + "CASE WHEN conditions must be of the same type as the expression" + ), + refused( + s"SELECT id, n + '1.5' AS x FROM $numberTable", + "output 'INT' is not compatible with input 'VARCHAR'" + ), + refused( + s"SELECT id, big - '1.0' AS x FROM $numberTable", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ) + ) + ) + } + "DATE_TRUNC(…, DAY) compared with a date" should "run: a day truncation of a date is the date" in { verdict( "DATE_TRUNC", From aa703ce886443b079126881d1ad15194ba2cdd77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Oct 2026 01:00:10 +0200 Subject: [PATCH 3/5] fix(sql): NULLIF reads a temporal literal once; boolean CASE renders constants and NULL-safe conditions - NULLIF with a date literal of the other shape (a DATE operand with a date-time literal, or a TIMESTAMP operand with a date-only literal, in either order) no longer binds a per-document parse of the literal. It renders the literal's reading, in queries and in computed columns. It failed in queries and stored NULL in computed columns; main refused it by name. Its values now equal PostgreSQL's. - CASE WHEN '1' / 'true' / TRUE renders the constant (1L / 0L) instead of (true ? 1L : 0L), which Elasticsearch 6.8 refuses to compile. This also fixes CASE b WHEN TRUE on 6.8, which is broken on main. - CASE WHEN takes the ELSE branch on a document with no value for the column, in queries and in computed columns. - Docs: the whole-number rule beside integer types in operators.md; the BETWEEN wording; an accurate test comment. Co-Authored-By: Claude Opus 5.5 (1M context) --- documentation/sql/operators.md | 17 +- .../elastic/sql/function/cond/package.scala | 82 ++++++- .../elastic/sql/function/package.scala | 20 +- .../app/softnetwork/elastic/sql/package.scala | 18 ++ .../elastic/sql/type/SQLTypeUtils.scala | 46 +++- .../query/MixedTemporalComparisonSpec.scala | 3 +- .../sql/query/TypeCheckResolutionSpec.scala | 98 ++++++++ .../elastic/client/DateArithmeticSpec.scala | 232 +++++++++++++++++- 8 files changed, 483 insertions(+), 33 deletions(-) diff --git a/documentation/sql/operators.md b/documentation/sql/operators.md index f2e1dfb92..c5684e5e2 100644 --- a/documentation/sql/operators.md +++ b/documentation/sql/operators.md @@ -559,12 +559,17 @@ SELECT order_date * 2 FROM orders; a comparison β€” `=`, `<>`, `<`, `<=`, `>`, `>=`, `BETWEEN`, `IN`, `CASE x WHEN v`, `NULLIF`, a `CASE` condition β€” a literal beside a number is the number it spells, beside a `DATE` the date it spells (the date of a date-time literal), beside a `TIMESTAMP` the instant it spells - (`'2024-01-31'` is its midnight), and beside a `BOOLEAN` the boolean it spells. A `DATE` and a - `TIMESTAMP` compare, the `DATE` being the instant its day starts at. Any other literal is text, - and comparing it with a number or a date is refused by name. A number written as a string beside - a date or a timestamp in `+` or `-` (`order_date + '30'`) is not read as PostgreSQL reads it: - elasticsql counts it as that number of days, which PostgreSQL does not (see "The engines this - follows" above). + (`'2024-01-31'` is its midnight), and beside a `BOOLEAN` the boolean it spells. Beside a + whole-number type (`TINYINT`, `SMALLINT`, `INT`, `BIGINT`) only a whole number is read: `'1.5'` + and `'1.0'` are no integer, so `COALESCE(qty, 0) = '1.5'` or `qty + '1.0'` over an integer `qty` + is refused by name, as PostgreSQL refuses it (`invalid input syntax for type integer`); beside + `DOUBLE`, `REAL` or `DECIMAL`, any number is read. (A `WHERE` comparison with a bare integer + column, `WHERE qty = '1.5'`, is the exception: it runs as a query on the column's mapping, and + Elasticsearch reads the literal.) A `DATE` and a `TIMESTAMP` compare, the `DATE` being the + instant its day starts at. Any other literal is text, and comparing it with a number or a date + is refused by name. A number written as a string beside a date or a timestamp in `+` or `-` + (`order_date + '30'`) is not read as PostgreSQL reads it: elasticsql counts it as that number of + days, whatever its fraction, which PostgreSQL does not (see "The engines this follows" above). - An index pattern or a comma list (`FROM logs-*`, `FROM orders_2024, orders_2025`) answers like a single index: every field its indices map alike has its type, read once per pattern from the merged mapping. ⚠️ A field two of them map differently (a `date` in one index, a `keyword` in diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index aa92ddafc..610816caa 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -19,6 +19,7 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ query, renderedTypeOf, + BooleanValue, ComparisonRule, Expr, GenericIdentifier, @@ -385,6 +386,23 @@ package object cond { private[this] var _secondReading: Option[ComparisonRule.Reading] = null private[this] var _firstReading: Option[ComparisonRule.Reading] = null + /** A string literal argument READ as a date or a timestamp is rendered from that reading + * (`ComparisonRule.temporalPainless`), never through the per-document `String -> temporal` + * coercion the generic rendering binds for a literal: that coercion parses the literal's OWN + * text by the operand's type, so a date-time literal beside a DATE and a date-only one beside + * a TIMESTAMP -- both of which the reading reads, as PostgreSQL does -- threw on every + * document, and a computed column stored NULL (`ignore_failure`). + */ + override protected def ownArgumentPainless( + index: Int, + context: Option[PainlessContext] + ): Option[String] = + (index match { + case 0 => firstReading + case 1 => secondReading + case _ => None + }).collect { case t: ComparisonRule.TemporalReading => ComparisonRule.temporalPainless(t) } + /** A literal argument counts as what it is read as: `NULLIF(n, '3')` compares two numbers. */ override def argTypes: List[SQLType] = List( @@ -670,7 +688,12 @@ package object cond { def scalar(reading: Option[ComparisonRule.Reading], rendered: String): String = reading.flatMap(ComparisonRule.scalarPainless).getOrElse(rendered) val arg0 = operand(scalar(firstReading, rendered0), context) - val arg1 = operand(scalar(secondReading, rendered1), context) + // A second argument READ as a temporal is compared as the instant it denotes + // (`comparedMillis`) and is never the answer, so its value is not bound: it would be + // evaluated on every document for nothing. + val arg1 = + if (secondReading.exists(_.isInstanceOf[ComparisonRule.TemporalReading])) rendered1 + else operand(scalar(secondReading, rendered1), context) // πŸ”΄ The SECOND argument is guarded too (issue #373, found by review) -- the same hole // `checkCase` had, one method up in this file. `NULLIF(CAST(d AS TIME), CAST(ts AS // TIME))` called the comparison with a null `arg1` for any document missing `ts`: @@ -1038,6 +1061,28 @@ package object cond { ): (SQLType, List[Option[ComparisonRule.Reading]]) = if (expression.exists(_ eq expr)) expressionReadings else readingsBeside(expr) + /** The boolean CONSTANT a `WHEN` value is, if it is one: a string literal read as a boolean + * (`CASE b WHEN '1'`, `ComparisonRule.BooleanReading`), or the literal `TRUE` / `FALSE`. + */ + private[this] def booleanConstantOf( + value: PainlessScript, + reading: Option[ComparisonRule.Reading] + ): Option[Boolean] = + reading match { + case Some(ComparisonRule.BooleanReading(b)) => Some(b) + case Some(_) => None + case None => + value match { + case b: BooleanValue => Some(b.value) + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (b: BooleanValue) :: Nil => Some(b.value) + case _ => None + } + case _ => None + } + } + /** The JAVA type an operand renders (#384's `Identifier.renderedType`): a DATE column's doc * value is a `ZonedDateTime`, a cast to DATE a `LocalDate`. */ @@ -1135,17 +1180,26 @@ package object cond { // the boolean it spells (`ComparisonRule.readingOf`), brought to the type the // two are compared in like any other value. `CASE b WHEN 'true'` used to parse // `"true"` as a long (`Long.parseLong`) and fail the shard. + // + // A boolean CONSTANT -- such a reading, or `TRUE` / `FALSE` -- is that number + // itself (`1L` / `0L`): converted like a value, `(true ? 1L : 0L)` is a + // conditional over a constant, which Elasticsearch 6.8's Painless refuses to + // compile ("Extraneous conditional statement"). val c = - whenReading - .flatMap(reading => - ComparisonRule - .scalarPainless(reading) - .map( - SQLTypeUtils - .coerce(_, reading.sqlType, out, nullable = false, context) + booleanConstantOf(cond, whenReading) + .flatMap(SQLTypeUtils.booleanConstant(_, out)) + .getOrElse( + whenReading + .flatMap(reading => + ComparisonRule + .scalarPainless(reading) + .map( + SQLTypeUtils + .coerce(_, reading.sqlType, out, nullable = false, context) + ) ) + .getOrElse(SQLTypeUtils.coerce(cond, out, context)) ) - .getOrElse(SQLTypeUtils.coerce(cond, out, context)) val r = boxWhenNoDefault(res match { case i: Identifier if i.name == name && name.nonEmpty => @@ -1211,8 +1265,14 @@ package object cond { }) if (!cond.isInstanceOf[CriteriaWithConditionalFunction[_]] && cond.nullable) { ctx.addParam(LiteralParam(c)) match { - case Some(c) => s"$c ? $r" - case _ => s"$c ? $r" + // A VALUE condition -- a boolean column, a function of one -- is NULL + // where the document has no value, and a NULL condition is not true: the + // branch does not apply (SQL), where `param1 ? 1 : 0` threw on that + // document (a query failed; a computed column stored NULL). `==` is + // null-safe in Painless. A predicate already renders NULL as false. + case Some(c) if cond.isInstanceOf[Identifier] => s"$c == true ? $r" + case Some(c) => s"$c ? $r" + case _ => s"$c ? $r" } } else { s"$c ? $r" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index d86d35295..cdc37d110 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -324,6 +324,20 @@ package object function { def checkIfNullable: Boolean = args.exists(_.nullable) || args.exists(argRendersNullOnFailure) + /** The Painless of the argument at `index` when this function renders it ITSELF, in place of + * the conversion [[painless]] gives an argument with no parameter of its own (a literal) -- + * which, with a context, BINDS that conversion to `argTypes(index)` as a parameter of the + * script, evaluated on every document. `None`, the default, leaves the argument to it. + * + * `NULLIF` renders this way a string literal it reads as a date or a timestamp: from its + * READING, never through a per-document parse of the literal's text (see `NullIf`). + */ + protected def ownArgumentPainless( + index: Int, + context: Option[PainlessContext] + ): Option[String] = + None + override def painless(context: Option[PainlessContext]): String = { context match { case Some(ctx) => @@ -412,8 +426,10 @@ package object function { }).getOrElse { if (a.nullable) s"arg$i" else - SQLTypeUtils - .coerce(a.painless(context), a.baseType, argTypes(i), nullable = false, context) + ownArgumentPainless(i, context).getOrElse( + SQLTypeUtils + .coerce(a.painless(context), a.baseType, argTypes(i), nullable = false, context) + ) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 88e76f840..8fcc58014 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1854,6 +1854,24 @@ package object sql { case BooleanReading(value) => Some(value.toString) case _ => None } + + /** The VALUE a temporal reading denotes, as Painless that parses nothing: a DATE is the + * `LocalDate` of its day, a TIMESTAMP the UTC `ZonedDateTime` of its instant (core's + * number-to-TIMESTAMP conversion, `SQLTypeUtils.coerce`) -- the java.time types a `CAST('…' AS + * DATE)` / `CAST('…' AS TIMESTAMP)` of the same text renders, built from the reading when the + * statement is rendered rather than parsed from the literal on every document. + */ + def temporalPainless(reading: TemporalReading): String = + if (reading.sqlType == SQLTypes.Date) + s"LocalDate.ofEpochDay(${Math.floorDiv(reading.epochMillis, DayMillis)}L)" + else + SQLTypeUtils.coerce( + s"${reading.epochMillis}L", + SQLTypes.BigInt, + SQLTypes.Timestamp, + nullable = false, + None + ) } /** The type a TYPE RULE reads for an operand once the column types are known: the one the operand diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 63be3e49a..996cc696e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -445,18 +445,9 @@ object SQLTypeUtils { s"$expr.toInstant().toEpochMilli()" // ---- BOOLEAN -> NUMERIC ---- - case (SQLTypes.Boolean, SQLTypes.Numeric | SQLTypes.Int) => - s"($expr ? 1 : 0)" - case (SQLTypes.Boolean, SQLTypes.BigInt) => - s"($expr ? 1L : 0L)" - case (SQLTypes.Boolean, SQLTypes.Double) => - s"($expr ? 1.0 : 0.0)" - case (SQLTypes.Boolean, SQLTypes.Real) => - s"($expr ? 1.0f : 0.0f)" - case (SQLTypes.Boolean, SQLTypes.SmallInt) => - s"(short)($expr ? 1 : 0)" - case (SQLTypes.Boolean, SQLTypes.TinyInt) => - s"(byte)($expr ? 1 : 0)" + case (SQLTypes.Boolean, numeric) if booleanNumerals(numeric).isDefined => + val BooleanNumerals(one, zero, narrowing) = booleanNumerals(numeric).get + s"${narrowing.getOrElse("")}($expr ? $one : $zero)" // ---- -> BOOLEAN ---- // πŸ”΄ There were seven arms FROM boolean and NONE to it, so every `(_, Boolean)` pair fell @@ -808,6 +799,37 @@ object SQLTypeUtils { case _ => false } + /** What a BOOLEAN is as a number of type `to`: TRUE's numeral, FALSE's, and the narrowing cast + * the type needs, if any. + */ + private final case class BooleanNumerals(one: String, zero: String, narrowing: Option[String]) + + /** ONE table for both renderings of a BOOLEAN as a number: the one `coerce` renders, which tests + * the value on every document, and the one [[booleanConstant]] renders, which folds a constant. + * `None` for a type that is not a number. + */ + private def booleanNumerals(to: SQLType): Option[BooleanNumerals] = to match { + case SQLTypes.Numeric | SQLTypes.Int => Some(BooleanNumerals("1", "0", None)) + case SQLTypes.BigInt => Some(BooleanNumerals("1L", "0L", None)) + case SQLTypes.Double => Some(BooleanNumerals("1.0", "0.0", None)) + case SQLTypes.Real => Some(BooleanNumerals("1.0f", "0.0f", None)) + case SQLTypes.SmallInt => Some(BooleanNumerals("1", "0", Some("(short)"))) + case SQLTypes.TinyInt => Some(BooleanNumerals("1", "0", Some("(byte)"))) + case _ => None + } + + /** A boolean CONSTANT as a number of type `to` -- `1L` / `0L` for a BIGINT -- or `None` for a + * type that is not a number. + * + * πŸ”΄ Not `coerce`'s `(true ? 1L : 0L)`: Elasticsearch 6.8's Painless refuses a conditional whose + * condition is a constant ("Extraneous conditional statement"), so a simple `CASE b WHEN '1'` + * (or `WHEN TRUE`) failed to compile there. + */ + private[sql] def booleanConstant(value: Boolean, to: SQLType): Option[String] = + booleanNumerals(to).map { case BooleanNumerals(one, zero, narrowing) => + s"${narrowing.getOrElse("")}${if (value) one else zero}" + } + def canConvert(from: SQLType, to: SQLType): Boolean = { // 1. Identity diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala index f9258cd97..373bb6aa8 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala @@ -383,7 +383,8 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr // comparable wherever a comparison is made, judged on the declared types. BETWEEN brings each // bound to the operand's java.time type, as a single comparison does, and NULLIF compares the // instants the two denote -- neither calls `isBefore` / `isEqual` across a `LocalDate` and a - // `ZonedDateTime`, which is what failed the shard. RUN in `DateArithmeticSpec`. + // `ZonedDateTime`, which is what failed the shard. The NULLIF form is RUN in + // `DateArithmeticSpec`; the BETWEEN form is pinned here at RESOLVE only. resolvedVerdict( "SELECT name FROM t WHERE CAST(d AS DATE) BETWEEN CAST('2025-01-01' AS DATE) " + "AND CAST('2025-12-31T00:00:00Z' AS TIMESTAMP)" diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala index 776158c51..1dd562a92 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala @@ -362,4 +362,102 @@ class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDriven criteriaPainless("SELECT id FROM t WHERE COALESCE(x, 0) = '1.5'") should include("== 1.5") criteriaPainless("SELECT id FROM t WHERE COALESCE(big, 0) = '1'") should include("== 1L") } + + /** The Painless a statement's one script field renders in a query, its declarations included. */ + private def fieldPainless(sql: String): String = { + val search = parsed(sql).update(Some(schema)) + search.validateResolved() shouldBe Right(()) + val field = search.scriptFields + .filterNot(_.isAggregation) + .headOption + .getOrElse(fail(s"[$sql] no script field")) + val ctx = PainlessContext(context = PainlessContextType.Query) + val body = field.painless(Some(ctx)) + s"$ctx$body" + } + + /** The ingest script of every computed column of a CREATE TABLE. */ + private def ingestPainless(ddl: String): String = Parser(ddl) match { + case Right(create: CreateTable) => + create.schema.columns.flatMap(_.script).map(_.source).mkString(" ;; ") + case other => fail(s"[$ddl] parsed as $other") + } + + /** NULLIF reads a string literal beside a temporal as the date or the timestamp it spells, and + * renders it from that READING: the literal's text is never parsed on a document -- by the + * operand's own format it could not be, for a date-time beside a DATE or a date beside a + * TIMESTAMP -- and a literal that is the answer is the value its reading denotes. + */ + "a NULLIF literal read as a date or a timestamp" should + "be rendered from its reading, never parsed on a document, in a query and a computed column" in { + forAll( + Table( + ("expression", "type", "value"), + ("NULLIF(d, '2024-01-31 10:00:00')", "DATE", None), + ("NULLIF(d, '2024-01-31T10:00:00Z')", "DATE", None), + ("NULLIF(ts, '2024-01-31')", "TIMESTAMP", None), + ("NULLIF(COALESCE(d, d2), '2024-01-31T10:00:00Z')", "DATE", None), + ("NULLIF('2024-01-31T10:00:00Z', d)", "DATE", Some("LocalDate.ofEpochDay(19753L)")), + ( + "NULLIF('2024-01-31', ts)", + "TIMESTAMP", + Some("Instant.ofEpochMilli(1706659200000L).atZone(ZoneId.of('Z'))") + ) + ) + ) { (expression, sqlType, value) => + Seq( + "query" -> fieldPainless(s"SELECT $expression AS v FROM t"), + "computed column" -> ingestPainless( + s"CREATE TABLE z (id INT, d DATE, d2 DATE, ts TIMESTAMP, " + + s"v $sqlType SCRIPT AS ($expression), PRIMARY KEY (id))" + ) + ).foreach { case (venue, script) => + withClue(s"[$venue] [$expression] $script ") { + script should not include "\"2024-01-31" + value.foreach(v => script should include(v)) + } + } + } + } + + /** `CASE b WHEN '1'` / `WHEN TRUE` compares with a boolean CONSTANT: rendered as the number it + * is, never as `(true ? 1L : 0L)`, a conditional over a constant, which Elasticsearch 6.8's + * Painless refuses to compile ("Extraneous conditional statement"). + */ + "a simple CASE over a BOOLEAN, compared with a boolean constant" should + "render the constant as a number, never as a conditional over a constant" in { + forAll( + Table( + ("value", "constant"), + ("'1'", "1L"), + ("'true'", "1L"), + ("TRUE", "1L"), + ("'0'", "0L"), + ("'false'", "0L"), + ("FALSE", "0L") + ) + ) { (value, constant) => + val script = fieldPainless(s"SELECT CASE b WHEN $value THEN 1 ELSE 0 END AS v FROM t") + withClue(s"[$value] $script ") { + script should not include "(true ?" + script should not include "(false ?" + script should include(s"= $constant;") + } + } + } + + /** A BOOLEAN column as a CASE condition is NULL on a document that has no value, and a NULL + * condition is not true: rendered `== true`, never as the bare value, which threw on that + * document. + */ + "a BOOLEAN column as a CASE condition" should + "render NULL as not true, in a query and in a computed column" in { + fieldPainless("SELECT CASE WHEN b THEN 1 ELSE 0 END AS v FROM t") should include( + "== true ? 1 : 0" + ) + ingestPainless( + "CREATE TABLE z (id INT, flag BOOLEAN, c INT SCRIPT AS (CASE WHEN flag THEN 1 ELSE 0 END), " + + "PRIMARY KEY (id))" + ) should include("== true ? 1 : 0") + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala index b32e5f2c5..8b2ff6407 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala @@ -108,6 +108,15 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with private val refreshedA = "date_arithmetic_rf_a" private val refreshedB = "date_arithmetic_rf_b" + /** NULLIF with a date or a timestamp literal (`"NULLIF with a date or a timestamp literal"`). */ + private val nullifTable = "date_arithmetic_nullif" + + /** A simple CASE over a BOOLEAN (`"a simple CASE over a BOOLEAN"`). */ + private val booleanTable = "date_arithmetic_bool" + + /** A BOOLEAN column as a CASE condition (`"a CASE WHEN over a BOOLEAN column"`). */ + private val flagTable = "date_arithmetic_flag" + private val DayMillis: Long = 86400000L /** One document: `None` is a column the document does not carry. @@ -304,7 +313,10 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with alteredRawIndex, grownTable, refreshedA, - refreshedB + refreshedB, + nullifTable, + booleanTable, + flagTable ).foreach(t => Try(Await.result(client.run(s"DROP TABLE IF EXISTS $t"), 60.seconds))) super.afterAll() } @@ -342,6 +354,9 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with private final case class Bool(value: Boolean) extends Expected private case object Null extends Expected + /** An instant read from the clock while the statement ran: between `from` and `to`. */ + private final case class Within(from: Instant, to: Instant) extends Expected + private def start(d: LocalDate): Instant = d.atStartOfDay(ZoneOffset.UTC).toInstant private def utcDay(i: Instant): LocalDate = i.atZone(ZoneOffset.UTC).toLocalDate @@ -414,6 +429,11 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with case Some(got) if got == want => None case _ => Some(s"$actual instead of $want") } + case Within(from, to) => + asInstant(actual) match { + case Some(got) if !got.isBefore(from) && !got.isAfter(to) => None + case _ => Some(s"$actual instead of an instant between $from and $to") + } case Bool(want) => scalar(actual) match { case b: java.lang.Boolean if b.booleanValue() == want => None @@ -1487,6 +1507,216 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with ) } + /** The STORED value of each computed column of `from`, for each document. */ + private def storedVerdict( + from: String, + expected: Seq[(String, Map[String, Expected])] + ): (String, Either[String, Seq[String]]) = { + val sql = s"SELECT id, ${expected.map(_._1).mkString(", ")} FROM $from" + val ids = expected.flatMap(_._2.keys).toSet + sql -> gatewayRows(sql).map { rs => + val byId = rs.map(r => r.getOrElse("id", "?").toString -> r).toMap + (if (byId.keySet == ids) Nil else Seq(s"rows ${byId.keySet.toSeq.sorted.mkString(",")}")) ++ + (for { + (column, values) <- expected + (id, want) <- values.toSeq.sortBy(_._1) + diff <- differs(want, byId.get(id).flatMap(_.get(column)).orNull) + } yield s"$column $id: $diff") + } + } + + /** The documents of [[nullifTable]], PostgreSQL 16's fixture for the answers below: a DATE that + * is the literals' day and one that is not, TIMESTAMPs at that day's midnight, later that day + * and the next midnight, a document with no value at all and one with only `d2`. + */ + private val nullifDocs = List( + """{"id":"n1","d":"2024-01-31","d2":"2024-02-01","ts":"2024-01-31T00:00:00Z"}""", + """{"id":"n2","d":"2024-02-01","d2":"2024-01-31","ts":"2024-01-31T10:00:00Z"}""", + """{"id":"n3","d":"2024-01-31","d2":"2024-01-30","ts":"2024-02-01T00:00:00Z"}""", + """{"id":"n4"}""", + """{"id":"n5","d2":"2024-01-31","ts":"2024-01-31T23:59:59.999Z"}""" + ) + + private def nullifAnswers( + n1: Expected, + n2: Expected, + n3: Expected, + n4: Expected, + n5: Expected + ): Map[String, Expected] = + Map("n1" -> n1, "n2" -> n2, "n3" -> n3, "n4" -> n4, "n5" -> n5) + + /** A DATE beside a date-TIME literal, a TIMESTAMP beside a date-only one: PostgreSQL reads the + * literal as the operand's type -- the day it names, the midnight it starts at -- and so does + * the comparison rule. The literal used to be parsed AGAIN on every document, by the operand's + * own format, which failed for these two shapes: the query failed and a computed column stored + * NULL. Every expected value is PostgreSQL 16's answer over the same documents. + */ + "NULLIF with a date or a timestamp literal of the other shape" should + "answer PostgreSQL's values in a query and STORE them in a computed column, in either argument order" in { + val columns = Seq( + ("c1", "DATE", "NULLIF(d, '2024-01-31 10:00:00')"), + ("c2", "TIMESTAMP", "NULLIF(ts, '2024-01-31')"), + ("c3", "DATE", "NULLIF('2024-01-31T10:00:00Z', d)"), + ("c4", "TIMESTAMP", "NULLIF('2024-01-31', ts)"), + ("c5", "DATE", "NULLIF(COALESCE(d, d2), '2024-01-31T10:00:00Z')") + ) + run( + s"CREATE TABLE $nullifTable (id KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, " + + columns.map { case (c, t, e) => s"$c $t SCRIPT AS ($e)" }.mkString(", ") + + ", PRIMARY KEY (id))" + ) + bulk(nullifTable, nullifDocs) + val jan30 = At(start(on("2024-01-30"))) + val jan31 = At(start(on("2024-01-31"))) + val feb1 = At(start(on("2024-02-01"))) + val feb2 = At(start(on("2024-02-02"))) + val ts2 = At(at("2024-01-31T10:00:00Z")) + val ts3 = At(at("2024-02-01T00:00:00Z")) + val ts5 = At(at("2024-01-31T23:59:59.999Z")) + val day = At(start(today)) + val answers: Seq[(String, Map[String, Expected])] = Seq( + // the operand first + "NULLIF(d, '2024-01-31 10:00:00')" -> nullifAnswers(Null, feb1, Null, Null, Null), + "NULLIF(d, '2024-01-31T10:00:00Z')" -> nullifAnswers(Null, feb1, Null, Null, Null), + "NULLIF(ts, '2024-01-31')" -> nullifAnswers(Null, ts2, ts3, Null, ts5), + // the literal first: the answer is the literal, read as the operand's type + "NULLIF('2024-01-31T10:00:00Z', d)" -> nullifAnswers(Null, jan31, Null, jan31, jan31), + "NULLIF('2024-01-31 10:00:00', d)" -> nullifAnswers(Null, jan31, Null, jan31, jan31), + "NULLIF('2024-01-31', ts)" -> nullifAnswers(Null, jan31, jan31, jan31, jan31), + "NULLIF('2024-01-31T10:00:00Z', COALESCE(d, d2))" -> + nullifAnswers(Null, jan31, Null, jan31, Null), + // operands that are not columns + "NULLIF(COALESCE(d, d2), '2024-01-31T10:00:00Z')" -> + nullifAnswers(Null, feb1, Null, Null, Null), + "NULLIF(CAST(ts AS DATE), '2024-01-31T10:00:00Z')" -> + nullifAnswers(Null, Null, feb1, Null, Null), + "NULLIF(CAST(d AS TIMESTAMP), '2024-01-31')" -> nullifAnswers(Null, feb1, Null, Null, Null), + // PostgreSQL: `d + interval '1 day'` and `d + 1` give the same answers here + "NULLIF(DATE_ADD(d, INTERVAL 1 DAY), '2024-02-01')" -> + nullifAnswers(Null, feb2, Null, Null, Null), + "NULLIF(DATE_TRUNC(ts, DAY), '2024-01-31')" -> nullifAnswers(Null, Null, feb1, Null, Null), + "NULLIF(LEAST(d, d2), '2024-01-31T10:00:00Z')" -> + nullifAnswers(Null, Null, jan30, Null, Null), + "NULLIF(CURRENT_DATE, '2024-01-31T10:00:00Z')" -> nullifAnswers(day, day, day, day, day) + ) + // the clock: NOW() is never the literal's midnight, so the answer is the instant it was read + val clockSql = s"SELECT id, NULLIF(NOW(), '2024-01-31') AS x FROM $nullifTable" + val before = Instant.now().truncatedTo(ChronoUnit.MILLIS) + val clockRows = gatewayRows(clockSql) + val after = Instant.now() + val clock = clockSql -> clockRows.map { rs => + val got = byKey(rs, "id") + Seq("n1", "n2", "n3", "n4", "n5").flatMap { id => + differs(Within(before, after), got.getOrElse(id, null)).map(d => s"$id: $d") + } + } + verdict( + "NULLIF with a temporal literal", + answers.map { case (expression, values) => + comparisonVerdict(expression, values, nullifTable) + } ++ Seq( + clock, + storedVerdict( + nullifTable, + columns.map { case (column, _, expression) => + column -> answers.toMap.getOrElse(expression, fail(s"no answer for $expression")) + } + ) + ) + ) + } + + /** `CASE b WHEN '1'` compares the BOOLEAN with the boolean the literal spells, as PostgreSQL + * reads it (`'1'`, `'true'`, `'0'`, `'false'`), and `CASE b WHEN TRUE` with the constant. Each + * used to render the constant through the per-document conversion, `(true ? 1L : 0L)`, which + * Elasticsearch 6.8's Painless refuses to compile ("Extraneous conditional statement"): the + * query failed, and so did the CREATE TABLE of a computed column. A document with no `b` matches + * no `WHEN`. Every expected value is PostgreSQL 16's answer. + */ + "a simple CASE over a BOOLEAN, compared with a boolean constant" should + "answer PostgreSQL's values on every major, in a query and in a computed column" in { + run( + s"CREATE TABLE $booleanTable (id KEYWORD, b BOOLEAN, " + + "w1 INT SCRIPT AS (CASE b WHEN '1' THEN 1 ELSE 0 END), " + + "w2 INT SCRIPT AS (CASE b WHEN TRUE THEN 1 ELSE 0 END), PRIMARY KEY (id))" + ) + bulk( + booleanTable, + List("""{"id":"t1","b":true}""", """{"id":"t2","b":false}""", """{"id":"t3"}""") + ) + def answers(t1: Expected, t2: Expected, t3: Expected): Map[String, Expected] = + Map("t1" -> t1, "t2" -> t2, "t3" -> t3) + val whenTrue = answers(Num(1), Num(0), Num(0)) + val whenFalse = answers(Num(0), Num(1), Num(0)) + verdict( + "simple CASE over a BOOLEAN", + Seq( + comparisonVerdict("CASE b WHEN '1' THEN 1 ELSE 0 END", whenTrue, booleanTable), + comparisonVerdict("CASE b WHEN 'true' THEN 1 ELSE 0 END", whenTrue, booleanTable), + comparisonVerdict("CASE b WHEN 'false' THEN 1 ELSE 0 END", whenFalse, booleanTable), + comparisonVerdict("CASE b WHEN '0' THEN 1 ELSE 0 END", whenFalse, booleanTable), + comparisonVerdict("CASE b WHEN TRUE THEN 1 ELSE 0 END", whenTrue, booleanTable), + comparisonVerdict("CASE b WHEN FALSE THEN 1 ELSE 0 END", whenFalse, booleanTable), + comparisonVerdict( + "CASE b WHEN '1' THEN 1.5 ELSE 0 END", + answers(Num(1.5), Num(0), Num(0)), + booleanTable + ), + storedVerdict(booleanTable, Seq("w1" -> whenTrue, "w2" -> whenTrue)) + ) + ) + } + + /** A BOOLEAN column as a CASE condition is NULL on a document that has no value, and a NULL + * condition is not true: the branch does not apply, and the ELSE does -- or NULL, with no ELSE. + * It used to throw on that document (`param1 ? 1 : 0` over a null): the query failed, and a + * computed column stored NULL where SQL gives the ELSE value. Every expected value is PostgreSQL + * 16's answer. + */ + "a CASE WHEN over a BOOLEAN column" should + "take the ELSE branch on a document with no value, in a query, a WHERE and a computed column" in { + run( + s"CREATE TABLE $flagTable (id KEYWORD, flag BOOLEAN, n INT, " + + "c INT SCRIPT AS (CASE WHEN flag THEN 1 ELSE 0 END), " + + "c2 INT SCRIPT AS (CASE WHEN flag THEN n ELSE 0 END), PRIMARY KEY (id))" + ) + bulk( + flagTable, + List( + """{"id":"f1","flag":true,"n":5}""", + """{"id":"f2","flag":false,"n":6}""", + """{"id":"f3","n":7}""" + ) + ) + def answers(f1: Expected, f2: Expected, f3: Expected): Map[String, Expected] = + Map("f1" -> f1, "f2" -> f2, "f3" -> f3) + verdict( + "CASE WHEN over a BOOLEAN column", + Seq( + comparisonVerdict( + "CASE WHEN flag THEN 1 ELSE 0 END", + answers(Num(1), Num(0), Num(0)), + flagTable + ), + comparisonVerdict( + "CASE WHEN flag THEN n ELSE 0 END", + answers(Num(5), Num(0), Num(0)), + flagTable + ), + comparisonVerdict("CASE WHEN flag THEN 1 END", answers(Num(1), Null, Null), flagTable), + kept( + s"SELECT id FROM $flagTable WHERE CASE WHEN flag THEN 1 ELSE 0 END = 0", + Set("f2", "f3") + ), + storedVerdict( + flagTable, + Seq("c" -> answers(Num(1), Num(0), Num(0)), "c2" -> answers(Num(5), Num(0), Num(0))) + ) + ) + ) + } + "a pattern's cached schema" should "be refreshed by an elasticsql DDL statement on an index it matches" in { val from = "date_arithmetic_rf_*" def typed(sql: String, refusal: String): Boolean = From 53c5a3c2d3401edfdd02e1f447c7c91390d184ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Oct 2026 06:51:33 +0200 Subject: [PATCH 4/5] build: client modules take their Elasticsearch major from the build Each client module (es6 rest / jest, es7 rest, es8 / es9 java) now generates a BuildInfo object (elasticVersion, elasticMajor) from its own elasticSearchVersion. The search-body serializers read ElasticsearchMajor from it, and every painlessTarget uses the serializer's major. No client source writes a major by hand any more; the ES 8 and ES 9 modules hard-coded 8 and 9. The ES 8 and ES 9 serializers stay byte-identical. Co-Authored-By: Claude Opus 5.5 (1M context) --- build.sbt | 23 +++++++++++++++++++ .../jest/JestSearchBodySerializer.scala | 5 +++- ...tHighLevelClientSearchBodySerializer.scala | 5 +++- ...tHighLevelClientSearchBodySerializer.scala | 5 +++- .../elastic/client/java/JavaClientApi.scala | 4 +++- .../java/JavaClientSearchBodySerializer.scala | 5 ++++ .../elastic/client/java/JavaClientApi.scala | 4 +++- .../java/JavaClientSearchBodySerializer.scala | 5 ++++ 8 files changed, 51 insertions(+), 5 deletions(-) diff --git a/build.sbt b/build.sbt index 546ed3193..3e18e8141 100644 --- a/build.sbt +++ b/build.sbt @@ -167,6 +167,19 @@ lazy val log4jClosureSettings: Seq[Setting[_]] = Seq( } ) +/** The Elasticsearch major a CLIENT module is built against, generated from its own + * `elasticSearchVersion` so that no client source writes it by hand: the module's search-body + * serializer and the `PainlessTarget` its scripts are rendered for both read it. + */ +def clientBuildInfoSettings(pkg: String, obj: String): Seq[Setting[_]] = Seq( + buildInfoKeys := Seq[BuildInfoKey]( + BuildInfoKey("elasticVersion" -> elasticSearchVersion.value), + BuildInfoKey("elasticMajor" -> elasticSearchMajorVersion(elasticSearchVersion.value)) + ), + buildInfoPackage := pkg, + buildInfoObject := obj +) + lazy val licensing = project .in(file("licensing")) .configs(IntegrationTest) @@ -454,11 +467,13 @@ lazy val es6testkit = testkitProject(Versions.es6) lazy val es6rest = project .in(file("es6/rest")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.rest", "RestClientBuildInfo"), elasticSearchVersion := Versions.es6 ) .dependsOn( @@ -479,11 +494,13 @@ lazy val es6cli = lazy val es6jest = project .in(file("es6/jest")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.jest", "JestClientBuildInfo"), elasticSearchVersion := Versions.es6 ) .dependsOn( @@ -519,11 +536,13 @@ lazy val es7testkit = testkitProject(Versions.es7) lazy val es7rest = project .in(file("es7/rest")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.rest", "RestClientBuildInfo"), elasticSearchVersion := Versions.es7 ) .dependsOn( @@ -564,11 +583,13 @@ lazy val es8testkit = testkitProject(Versions.es8) lazy val es8java = project .in(file("es8/java")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.java", "JavaClientBuildInfo"), elasticSearchVersion := Versions.es8 ) .dependsOn( @@ -619,11 +640,13 @@ lazy val es9testkit = testkitProject( lazy val es9java = project .in(file("es9/java")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.java", "JavaClientBuildInfo"), scalaVersion := scala213, crossScalaVersions := Seq(scala213), elasticSearchVersion := Versions.es9, diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchBodySerializer.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchBodySerializer.scala index aaae20f83..86e511a0e 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchBodySerializer.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchBodySerializer.scala @@ -33,7 +33,10 @@ import com.sksamuel.elastic4s.searches.SearchRequest */ object JestSearchBodySerializer extends SearchBodySerializer { - val ElasticsearchMajor: Int = 6 + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = JestClientBuildInfo.elasticMajor /** The named error a transform-bearing extended_stats is refused with on this module. */ val TransformExtendedStatsUnsupported: String = diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala index 285ae7f14..5a4c177ee 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala @@ -33,7 +33,10 @@ import com.sksamuel.elastic4s.searches.SearchRequest */ object RestHighLevelClientSearchBodySerializer extends SearchBodySerializer { - val ElasticsearchMajor: Int = 6 + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = RestClientBuildInfo.elasticMajor /** The named error a transform-bearing extended_stats is refused with on this module. */ val TransformExtendedStatsUnsupported: String = diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala index 3bc5439ec..8cb7fd492 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala @@ -49,7 +49,10 @@ import com.sksamuel.elastic4s.requests.searches.aggs.{AbstractAggregation, Aggre */ object RestHighLevelClientSearchBodySerializer extends SearchBodySerializer { - val ElasticsearchMajor: Int = 7 + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = RestClientBuildInfo.elasticMajor /** The request with every [[ScriptedExtendedStatsAggregation]] replaced by the * `ExtendedStatsAggregation` it wraps β€” at the root and at every depth (the windowed bind sits diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 2def9d106..befb890fe 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -1039,7 +1039,9 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { * * A `date` doc value is a `ZonedDateTime` here: scripts render as they always have. */ - implicit def painlessTarget: PainlessTarget = PainlessTarget(8) + implicit def painlessTarget: PainlessTarget = PainlessTarget( + JavaClientSearchBodySerializer.ElasticsearchMajor + ) override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit timestamp: Long, diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala index 6ff2d46f6..25399cd4d 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala @@ -53,6 +53,11 @@ import com.sksamuel.elastic4s.requests.searches.{ */ object JavaClientSearchBodySerializer extends SearchBodySerializer { + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = JavaClientBuildInfo.elasticMajor + private val scriptedExtendedStats: PartialFunction[AbstractAggregation, XContentBuilder] = { case agg: ScriptedExtendedStatsAggregation => extendedStatsWithScript(agg) } diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 6129a13e8..30c96e077 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -1039,7 +1039,9 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { * * A `date` doc value is a `ZonedDateTime` here: scripts render as they always have. */ - implicit def painlessTarget: PainlessTarget = PainlessTarget(9) + implicit def painlessTarget: PainlessTarget = PainlessTarget( + JavaClientSearchBodySerializer.ElasticsearchMajor + ) override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit timestamp: Long, diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala index 6ff2d46f6..25399cd4d 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala @@ -53,6 +53,11 @@ import com.sksamuel.elastic4s.requests.searches.{ */ object JavaClientSearchBodySerializer extends SearchBodySerializer { + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = JavaClientBuildInfo.elasticMajor + private val scriptedExtendedStats: PartialFunction[AbstractAggregation, XContentBuilder] = { case agg: ScriptedExtendedStatsAggregation => extendedStatsWithScript(agg) } From 0687815975ecd4ac922e5b483c8fbecba72be6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Oct 2026 15:10:21 +0200 Subject: [PATCH 5/5] fix(sql): a CASE or NULLIF operand is one value; GREATEST / LEAST, ABS and a simple CASE answer PostgreSQL's types - A CASE or a NULLIF used as an operand is one value. The operator applies to every branch: CASE WHEN n > 0 THEN n ELSE 0 END + 1 added the 1 to the ELSE branch only. A NULLIF that nulls gives NULL instead of failing the query: x / NULLIF(n, 0), NULLIF(n, 0) + 1, a % NULLIF(b, 0), in queries and computed columns (a computed column was silently absent). - GREATEST / LEAST answer their arguments' common type on every row. Whole numbers stay whole: GREATEST(n, 2) answered 3.0. With a DOUBLE argument every row is a double, where an integer argument that won, or survived a NULL, came back as an integer. - ABS is typed by its argument, as abs(integer) is an integer. ABS(n) + 1 runs (it failed to compile), and a quoted fraction beside ABS(n) is refused by name again. - A simple CASE answers its THEN / ELSE values in their own type. It answered them in the operand's type ("0" over a KEYWORD). - NULLIF with a quoted literal first is typed by the literal's reading: NULLIF('2024-01-31', ts) is a TIMESTAMP, and CREATE TABLE ... AS SELECT declares it. - A number cast to DATE is refused by name, as PostgreSQL refuses it. A number cast to TIMESTAMP keeps its epoch-millisecond conversion. - Docs: the NULLIF guard works for / and %; the result types of GREATEST / LEAST, ABS and NULLIF; (a / b)::INTEGER is a truncated quotient, since casts over arithmetic apply. Values, and a refusal by name where PostgreSQL refuses, match PostgreSQL 16 on Elasticsearch 8.18.3 and 6.8.23. Closes #380 Closes #392 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../elastic/sql/SQLQuerySpec.scala | 2 +- documentation/sql/dql_statements.md | 9 +- documentation/sql/functions_conditional.md | 36 +- documentation/sql/functions_math.md | 3 +- documentation/sql/known_limitations.md | 29 +- documentation/sql/operators.md | 61 +- .../elastic/sql/SQLQuerySpec.scala | 2 +- .../elastic/sql/function/cond/package.scala | 320 +++++++- .../sql/function/convert/package.scala | 41 + .../elastic/sql/function/math/package.scala | 48 +- .../operator/math/ArithmeticExpression.scala | 64 +- .../elastic/sql/schema/package.scala | 21 + .../sql/query/ParameterIdentitySpec.scala | 18 +- .../sql/query/TypeCheckResolutionSpec.scala | 244 ++++++ .../elastic/client/DateArithmeticSpec.scala | 757 +++++++++++++++++- 15 files changed, 1523 insertions(+), 132 deletions(-) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index b0f1f2888..446a2421e 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2525,7 +2525,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); def param5 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate().plus(2, ChronoUnit.DAYS)); def param6 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param2 != null && param1.isEqual(param2) ? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6", + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); def param5 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).plus(2, ChronoUnit.DAYS)); def param6 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param1 != null && param2 != null && param1.isEqual(param2) ? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6", | "params": { | "__now__": 1767139200000 | } diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index caa6fa35b..f300e78e2 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -1405,9 +1405,12 @@ LEAST(e1, e2, ...) `GREATEST` returns the largest non-null value among the given expressions; `LEAST` returns the smallest. The arguments are all numeric, or all dates and timestamps (a `DATE` compares as the start of its day, UTC; the result is a `DATE` over dates and a `TIMESTAMP` when -they mix). NULL arguments are ignored (ANSI semantics); the result is NULL only -when every argument is NULL. Both are emitted as Painless ternary chains over -`Math.max` / `Math.min` β€” over dates, over the epoch milliseconds each one denotes. They are +they mix). Over numbers the result is the common type of all the arguments on every row: the +widest whole type when every argument is a whole number, a `DOUBLE` as soon as one is fractional. +NULL arguments are ignored (ANSI semantics); the result is NULL only +when every argument is NULL. Both are emitted as Painless ternary chains β€” over whole numbers the +winner is chosen by a comparison, over fractional numbers by `Math.max` / `Math.min`, over dates +over the epoch milliseconds each one denotes. They are conditional functions, not aggregates β€” `GREATEST(...) OVER (...)` is not supported β€” but over aggregates they are computed per group (`GREATEST(MAX(a), MAX(b))` beside a `GROUP BY`). diff --git a/documentation/sql/functions_conditional.md b/documentation/sql/functions_conditional.md index f570c9b84..f2b01d13c 100644 --- a/documentation/sql/functions_conditional.md +++ b/documentation/sql/functions_conditional.md @@ -241,7 +241,8 @@ NULLIF(expr1, expr2) - `expr2` - second expression **Output:** -- Type of `expr1`, or NULL if equal +- Type of `expr1` β€” a quoted literal first is the value it is read as beside `expr2`, and has its + type: `NULLIF('2024-01-31', ts)` is a `TIMESTAMP`, `NULLIF('3', n)` a number β€” or NULL if equal **Examples:** @@ -264,20 +265,27 @@ FROM products -- Converts 0 prices to NULL ``` -**3. Avoid division by zero β€” no longer needed, and measured not to work:** +**3. Guard a zero divisor β€” needed for `%`, not for `/`:** ```sql --- ⚠️ Since 0.24.0 the engine returns NULL for a zero divisor by itself, so write this: +-- Since 0.24.0 the engine returns NULL for a zero divisor of `/` by itself, so write this: SELECT total_sales / total_orders AS avg_order_value FROM sales_summary -- NULL on the rows where total_orders = 0 + +-- `%` is not guarded by itself: NULLIF is the guard +SELECT + total_sales % NULLIF(total_orders, 0) AS remainder +FROM sales_summary +-- NULL on the rows where total_orders = 0 or is missing ``` -> ⚠️ `total_sales / NULLIF(total_orders, 0)` β€” the idiom this section used to recommend β€” throws a -> `null_pointer_exception` in a search on exactly the rows where `total_orders = 0`, and silently -> drops the computed column in an ingest pipeline. Measured on real Elasticsearch, identically -> before and after `0.24.0`. `NULLIF` remains correct everywhere else; see the -> [`/` operator](operators.md) for the division rule. +> `total_sales / NULLIF(total_orders, 0)` β€” the idiom this section used to recommend β€” answers +> `NULL` on the rows where `total_orders` is `0` or missing, in a search and in a computed column, +> and so does `total_sales % NULLIF(total_orders, 0)`. Before `0.24.0` both threw a +> `null_pointer_exception` in a search on exactly those rows, and silently dropped the computed +> column in an ingest pipeline. See the [`/` and `%` operators](operators.md) for the zero-divisor +> rules. **4. Clean data:** ```sql @@ -463,8 +471,10 @@ GREATEST(expression1, expression2, ...) - One or more expressions of one kind: all numeric, or all `DATE` / `TIMESTAMP` / `DATETIME` **Output:** -- Numeric (widest input type) over numbers; over dates, a `DATE` when every argument is a `DATE` - and a `TIMESTAMP` when they mix; NULL if every argument is NULL +- Over numbers, the common type of all the arguments, as PostgreSQL types it, and every row has + it: the widest whole type when every argument is a whole number, a `DOUBLE` as soon as one is + fractional (a `REAL` when the fractional ones are all `REAL`); over dates, a `DATE` when every + argument is a `DATE` and a `TIMESTAMP` when they mix; NULL if every argument is NULL **Examples:** @@ -522,8 +532,10 @@ LEAST(expression1, expression2, ...) - One or more expressions of one kind: all numeric, or all `DATE` / `TIMESTAMP` / `DATETIME` **Output:** -- Numeric (widest input type) over numbers; over dates, a `DATE` when every argument is a `DATE` - and a `TIMESTAMP` when they mix; NULL if every argument is NULL +- Over numbers, the common type of all the arguments, as PostgreSQL types it, and every row has + it: the widest whole type when every argument is a whole number, a `DOUBLE` as soon as one is + fractional (a `REAL` when the fractional ones are all `REAL`); over dates, a `DATE` when every + argument is a `DATE` and a `TIMESTAMP` when they mix; NULL if every argument is NULL **Examples:** diff --git a/documentation/sql/functions_math.md b/documentation/sql/functions_math.md index 0181f9eb4..4837cf8d0 100644 --- a/documentation/sql/functions_math.md +++ b/documentation/sql/functions_math.md @@ -21,7 +21,8 @@ ABS(x) - `x` - `NUMERIC` **Output:** -- `NUMERIC` +- The type of `x`, as PostgreSQL's `abs`: `ABS` of an `INT` is an `INT` (a whole number), of a + `DOUBLE` a `DOUBLE` **Examples:** ```sql diff --git a/documentation/sql/known_limitations.md b/documentation/sql/known_limitations.md index 38a1b1bce..e9df7a3c9 100644 --- a/documentation/sql/known_limitations.md +++ b/documentation/sql/known_limitations.md @@ -222,37 +222,32 @@ branch. The restriction is not specific to division, and parenthesising does not | `CAST(a / b AS INTEGER)`, `CAST(a + b AS INTEGER)`, `CAST((a / b) AS INTEGER)` | Parse error | | `FLOOR(a / b)`, `ABS(a / b)`, `COALESCE(a / b, 0)` | Parse error | | `CASE WHEN b != 0 THEN a / b ELSE 0 END` | Parse error | -| `a / NULLIF(b, 0)`, `FLOOR(x)`, `CAST(x AS INTEGER)` | Accepted | +| `(a / b)::INTEGER`, `a / NULLIF(b, 0)`, `FLOOR(x)`, `CAST(x AS INTEGER)` | Accepted | The rule is **directional**: arithmetic *over* a function call is fine, a function call *over* arithmetic is not. -⚠️ **One spelling parses and then ignores the cast.** `(a / b)::INTEGER` is accepted, but the -conversion is discarded: it emits exactly what `a / b` emits, so the result is a DOUBLE even though -the expression reports INTEGER. Do not use it as a workaround for the rejections above β€” it is the -one shape in this family that fails *silently* rather than loudly. It is pre-existing and applies to -any operator (`(a + b)::DOUBLE`, `(d + 1)::INTEGER`). - -The practical consequences are that there is no single-expression way to write a truncated quotient, -and no in-expression guard for `%`: +**The `::` spelling works.** Since `0.24.0`, `(a / b)::INTEGER` applies the cast, truncated toward +zero: `(n / m)::INTEGER` is `3` for 7 / 2 and `-3` for -7 / 2, and NULL for a zero divisor, in a +search and in a computed column. Before `0.24.0` the conversion was discarded and the result stayed +a DOUBLE. This holds for any operator (`(a + b)::BIGINT`), so a truncated quotient is one +expression: ```sql --- instead of CAST(n / m AS INTEGER), compute the quotient into a column and cast THAT column -CREATE TABLE t (n INTEGER, m INTEGER, q DOUBLE SCRIPT AS (n / m)); -SELECT CAST(q AS INTEGER) AS whole FROM t; +-- instead of CAST(n / m AS INTEGER) +SELECT (n / m)::INTEGER AS whole FROM t; ``` -## `%` by zero is not guarded +## `%` by zero is not guarded by itself The `0.24.0` rule that makes `a / 0` yield NULL covers `/` only. With integer operands `a % 0` throws β€” HTTP 400 in a search, and in a computed column the ingest processor's `ignore_failure` swallows it so the column is simply absent. With **floating** operands it produces `NaN`, which Elasticsearch refuses to index, so **the whole document is rejected**. -Because of the restriction above there is no in-expression guard, and `a % NULLIF(b, 0)` throws a -`null_pointer_exception` on exactly the rows the guard is for. Keep a zero divisor out of the data -instead β€” filter it in `WHERE`, or compute the remainder from an already-filtered index. See -[operators](operators.md#-mod). +Guard it with `NULLIF`: `a % NULLIF(b, 0)` answers NULL on the rows where `b` is `0` or missing, in +a search and in a computed column (before `0.24.0` it threw a `null_pointer_exception` on exactly +those rows). See [operators](operators.md#-mod). ## `ORDER BY` over arithmetic on a nullable column diff --git a/documentation/sql/operators.md b/documentation/sql/operators.md index c5684e5e2..0b3152d94 100644 --- a/documentation/sql/operators.md +++ b/documentation/sql/operators.md @@ -290,25 +290,21 @@ SELECT CAST(10 AS DOUBLE) / 3 AS result; -- 3.333... > `CREATE TABLE u (n INTEGER, m INTEGER, c DOUBLE SCRIPT AS (n / m))` stored `3.0` for 7/2 before > `0.24.0`; it stores `3.5`. > -> ⚠️ **There is no truncating-division operator, and no single expression that produces one.** -> Other engines spell truncation `DIV` (MySQL) or `//` (DuckDB); this engine has neither β€” and the -> obvious rewrites are **parse errors**, because an arithmetic expression is not accepted as the -> operand of a function, of a `CAST` or of a `CASE` branch: +> ⚠️ **There is no truncating-division operator: write `(a / b)::INTEGER`.** Other engines spell +> truncation `DIV` (MySQL) or `//` (DuckDB); this engine has neither. Since `0.24.0` a `::` cast +> applies to arithmetic and truncates toward zero: `(n / m)::INTEGER` is `3` for 7 / 2 and `-3` for +> -7 / 2, and NULL for a zero divisor, in a search and in a computed column. The other obvious +> rewrites are **parse errors**, because an arithmetic expression is not accepted as the operand of +> a function, of a `CAST` or of a `CASE` branch: > > ```sql > CAST(a / b AS INTEGER) CAST(a + b AS INTEGER) CAST((a / b) AS INTEGER) -- βœ— rejected > FLOOR(a / b) ABS(a / b) COALESCE(a / b, 0) -- βœ— rejected -> FLOOR(x) a / NULLIF(b, 0) -- βœ“ accepted +> (a / b)::INTEGER FLOOR(x) a / NULLIF(b, 0) -- βœ“ accepted > ``` > > The rule is **directional**: `f()` is rejected, ` f(…)` is fine. It is > not specific to division β€” any arithmetic operand is refused β€” and parenthesising does not help. -> To get an integral quotient today, compute it into a column and cast **that** column: -> -> ```sql -> CREATE TABLE t (n INTEGER, m INTEGER, q DOUBLE SCRIPT AS (n / m)); -> SELECT CAST(q AS INTEGER) AS whole FROM t; -- 3 for 7 / 2, truncated toward zero -> ``` > > `%` (MOD) is unaffected by this rule and keeps deriving its type from its operands β€” see the `%` > section below, including what it does with a zero divisor. @@ -333,18 +329,16 @@ division **threw** (a search answered HTTP 400 `arithmetic_exception: / by zero` pipeline silently left the computed column out of the document), while floating division produced `Infinity` β€” which Elasticsearch refuses to index, so **the whole document was rejected**. -> ⚠️ **Two limitations to know about, both of them older than this rule and neither of them fixed -> by it:** +> ⚠️ **One limitation to know about, older than this rule and not fixed by it: `ORDER BY +> ` over nullable columns fails.** A script sort is typed `number`, and Elasticsearch +> rejects a sort script that can return null β€” which any arithmetic over a nullable column can, +> whether from a missing value or from a zero divisor. Measured identically before and after +> `0.24.0`. Sort by a plain column, or by a computed column. > -> - **`ORDER BY ` over nullable columns fails.** A script sort is typed `number`, and -> Elasticsearch rejects a sort script that can return null β€” which any arithmetic over a nullable -> column can, whether from a missing value or from a zero divisor. Measured identically before and -> after `0.24.0`. Sort by a plain column, or by a computed column. -> - **`NULLIF` inside a division fails on exactly the rows it protects.** -> `total / NULLIF(order_count, 0)` throws a `null_pointer_exception` in a search on any row where -> `order_count = 0`, and silently drops the computed column in an ingest pipeline. Measured -> identically before and after `0.24.0`. **Since `0.24.0` you do not need it for division: write -> `total / order_count`.** +> **You do not need `NULLIF` for a division: write `total / order_count`.** It works too: +> `total / NULLIF(order_count, 0)` answers NULL on the rows where `order_count` is `0` or missing, +> in a search and in a computed column. Before `0.24.0` it threw a `null_pointer_exception` in a +> search on exactly those rows, and silently dropped the computed column in an ingest pipeline. ⚠️ **Corrected in `0.24.0`: the other guards this page used to recommend do not parse.** As above, an arithmetic expression is not accepted as the operand of a function, of a `CAST` or of a `CASE` @@ -391,26 +385,23 @@ expr1 % expr2 **Output:** - Integer (remainder of division) -> ⚠️ **A zero divisor is NOT guarded for `%`.** The `0.24.0` rule that turns `a / 0` into NULL -> covers `/` only, and `%` has two distinct failures: +> ⚠️ **A zero divisor is NOT guarded for `%` by itself.** The `0.24.0` rule that turns `a / 0` into +> NULL covers `/` only, and a bare `%` has two distinct failures: > > - **integer operands** β€” `a % 0` throws. A search fails with HTTP 400 > `arithmetic_exception: / by zero`; in a computed column the ingest processor's > `ignore_failure` swallows it and the column is simply **absent** from the indexed document. > - **floating operands** β€” `a % 0` is `NaN`, and Elasticsearch refuses to index a non-finite > number, so **the whole document is rejected** (`document_parsing_exception: [double] supports -> only finite values`). This is the same data-loss failure the `/` guard was shipped to fix, and -> it is still open for `%`. -> -> ⚠️ **There is no in-expression way to guard it.** `CASE WHEN b != 0 THEN a % b END`, -> `COALESCE(a % b, 0)` and `FLOOR(a % b)` are all **parse errors** β€” an arithmetic expression is -> not accepted as the operand of a function, of a `CAST` or of a `CASE` branch (see the note under -> `/` above; the restriction is not specific to division). And `a % NULLIF(b, 0)` parses but then -> throws `null_pointer_exception` on exactly the rows the guard is for, exactly as it does for -> division. +> only finite values`). This is the same data-loss failure the `/` guard was shipped to fix. > -> Until `%` is covered, keep a zero divisor out of the data β€” filter it in `WHERE` -> (`WHERE b <> 0`), or compute the remainder into its own column from an already-filtered index. +> **Guard it with `NULLIF`: write `a % NULLIF(b, 0)`.** It answers NULL on the rows where `b` is +> `0` or missing and the remainder everywhere else, for integer and floating operands alike, in a +> search and in a computed column. Before `0.24.0` it threw a `null_pointer_exception` on exactly +> the rows the guard is for. The other guards one might write are **parse errors**: +> `CASE WHEN b != 0 THEN a % b END`, `COALESCE(a % b, 0)` and `FLOOR(a % b)` β€” an arithmetic +> expression is not accepted as the operand of a function, of a `CAST` or of a `CASE` branch (see +> the note under `/` above; the restriction is not specific to division). **Examples:** diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 6f31f8661..136252270 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2509,7 +2509,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); def param5 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate().plus(2, ChronoUnit.DAYS)); def param6 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param2 != null && param1.isEqual(param2) ? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6", + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); def param5 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).plus(2, ChronoUnit.DAYS)); def param6 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param1 != null && param2 != null && param1.isEqual(param2) ? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6", | "params": { | "__now__": 1767139200000 | } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 610816caa..24dec2536 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -25,6 +25,7 @@ import app.softnetwork.elastic.sql.{ GenericIdentifier, Identifier, LiteralParam, + NumericValue, PainlessContext, PainlessOperandForm, PainlessScript, @@ -34,9 +35,13 @@ import app.softnetwork.elastic.sql.{ } import app.softnetwork.elastic.sql.`type`.{ SQLAny, + SQLBigInt, SQLBool, + SQLInt, SQLNumeric, + SQLSmallInt, SQLTemporal, + SQLTinyInt, SQLType, SQLTypeUtils, SQLTypes, @@ -286,6 +291,9 @@ package object cond { private val simpleOperand = """(?:[A-Za-z_][A-Za-z0-9_]*|-?\d+(?:\.\d+)?[lLfFdD]?|"(?:[^"\\]|\\.)*")""".r + /** A whole-number Painless literal (`GREATEST` / `LEAST` compare two of them when rendering). */ + private val WholeLiteral = """-?\d+[lL]?""".r + /** How a simple `CASE WHEN ` compares (`Case.whenComparison`). */ private[cond] sealed trait WhenComparison private[cond] case object WhenAsResult extends WhenComparison @@ -431,9 +439,14 @@ package object cond { * Two questions, two derivations: `out` says how to compare, `reportedType` says what comes * out. A pair that cannot be compared at all is refused by [[typeMismatchError]] before either * matters. + * + * A first argument that is a string literal READ beside the second (`firstReading`) is the + * value it is read as, and is rendered as one: `NULLIF('2024-01-31', ts)` answers that + * midnight, a TIMESTAMP, as PostgreSQL types it -- not the VARCHAR the literal is written as, + * which a `CREATE TABLE … AS SELECT` declared, storing text. */ override def reportedType: SQLType = - reportedArgTypes.headOption.getOrElse(baseType) + firstReading.map(_.sqlType).getOrElse(reportedArgTypes.headOption.getOrElse(baseType)) /** The rejection this `NULLIF` earns when its two arguments RENDER types SQL does not compare * (issue #382, the LEAD RULING of 2026-09-23). @@ -852,7 +865,25 @@ package object cond { s"$exprPart $whenThen$elsePart $END" } - override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(argTypes) + /** The type of the CASE's VALUE: the least common supertype of its `THEN` results and its + * `ELSE` -- never the `CASE ` operand, which is only compared ([[comparisonType]]), as + * [[reportedType]] already reads it. With the operand in, `CASE s WHEN 'a' THEN 1 ELSE 0 END` + * over a KEYWORD `s` answered the strings `"1"` / `"0"` and `CASE x WHEN 1.5 THEN 1 ELSE 0 + * END` over a DOUBLE `x` the doubles `1.0` / `0.0` while reporting BIGINT; PostgreSQL answers + * the integers 1 and 0. + */ + override def baseType: SQLType = + SQLTypeUtils.leastCommonSuperType( + conditions.map { case (_, res) => res.out } ++ default.map(_.out) + ) + + /** The type a simple CASE compares its operand and each `WHEN` value in: the common supertype + * of the operand and the results -- what [[baseType]] answered while it included the operand + * -- or the type a cast over the CASE sets, so every comparison renders exactly as it did. + * Only the VALUE's type moved ([[baseType]]). + */ + private[this] def comparisonType: SQLType = + if (out != baseType) out else SQLTypeUtils.leastCommonSuperType(argTypes) override def validate(): Either[String, Unit] = { // Story 22.2 (AD-6) β€” a CASE-WHEN condition is parsed by `case_condition`, which shares @@ -944,12 +975,13 @@ package object cond { e: String, c: String, v: String, - candidateNullable: Boolean + candidateNullable: Boolean, + compared: SQLType ): String = { val guard = if (candidateNullable) s"$e != null && $c != null" else s"$e != null" // πŸ”΄ A TIME is decided by what the RECEIVER RENDERS, not by `out` (issue #373, found by - // review). `out` is the CASE's RESULT type -- the least common supertype of the expression, - // the conditions and the results -- and it reports TIMESTAMP for + // review). `out` -- then the least common supertype of the expression and the results, now + // [[comparisonType]] -- reports TIMESTAMP for // `CASE CAST(d AS TIME) WHEN CAST(ts AS TIME) …` whose operands are both `LocalTime`, so an // arm keyed on it was DEAD and that shape still emitted `isEqual`. MEASURED on ES 8.18: // `dynamic method [java.time.LocalTime, isEqual/1] not found`, the very failure this PR's @@ -957,7 +989,7 @@ package object cond { // describes the rendering -- the same key `NULLIF` needed. if (receiverIsTime) s"$guard && $e.compareTo($c) == 0 ? $v" else - out match { + compared match { case SQLTypes.Varchar => s"$guard && $e.compareTo($c) == 0 ? $v" case _: SQLTemporal => @@ -987,8 +1019,10 @@ package object cond { /** How `CASE WHEN …` compares its expression with each value -- the ONE comparison * rule (`ComparisonRule`) decides what is compared; this decides how it renders. * - * The historical rendering converts BOTH sides to `out`, the CASE's RESULT type, and is kept - * wherever that compares what SQL compares ([[WhenAsResult]]). It does not when: + * The historical rendering converts BOTH sides to [[comparisonType]] -- the common type of the + * expression and the results, what `out` was until the CASE's value was typed by its results + * alone -- and is kept wherever that compares what SQL compares ([[WhenAsResult]]). It does + * not when: * - the expression is a DATE or a TIMESTAMP and a value is a string literal read as one, the * two render different java.time types, or the result is not a temporal: a `LocalDate` * value against a column's `ZonedDateTime` failed the shard, and a numeric result compared @@ -1008,6 +1042,8 @@ package object cond { t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp val (declared, readings) = whenReadings(expr) val processor = context.exists(_.isProcessor) + // the type the two sides are converted to ([[WhenAsResult]]): the old `out` + val compared = comparisonType if (receiverIsTime) WhenAsResult else if (dateCarrying(declared)) { val temporalReading = @@ -1015,13 +1051,13 @@ package object cond { // a KNOWN mismatch only: with no schema attached a type is unknown, and the statement // keeps the rendering it always had val mismatch = - (!out.isInstanceOf[SQLTemporal] && !out.isUnknown) || + (!compared.isInstanceOf[SQLTemporal] && !compared.isUnknown) || conditions.exists { case (cond, _) => val rendered = javaTypeOf(cond) !rendered.isUnknown && rendered != javaTypeOf(expr) } if (temporalReading || (!processor && mismatch)) WhenInstants else WhenAsResult - } else if (declared.isNumber && !out.isNumber && !out.isUnknown) { + } else if (declared.isNumber && !compared.isNumber && !compared.isUnknown) { if (!processor || readings.exists(_.isDefined)) WhenNumbers else WhenAsResult } else WhenAsResult } @@ -1159,7 +1195,10 @@ package object cond { case Some(expr) if whenComparison(expr, context) != WhenAsResult => comparedWhens(expr, whenComparison(expr, context), ctx, context) case Some(expr) => // case with expression to evaluate - val e = SQLTypeUtils.coerce(expr, out, context) + // the operand and each value are converted to the type they are COMPARED in; the + // results, below, to the CASE's own (`out`) + val compared = comparisonType + val e = SQLTypeUtils.coerce(expr, compared, context) val expParam = ctx.addParam( LiteralParam(e) ) @@ -1187,7 +1226,7 @@ package object cond { // compile ("Extraneous conditional statement"). val c = booleanConstantOf(cond, whenReading) - .flatMap(SQLTypeUtils.booleanConstant(_, out)) + .flatMap(SQLTypeUtils.booleanConstant(_, compared)) .getOrElse( whenReading .flatMap(reading => @@ -1195,10 +1234,10 @@ package object cond { .scalarPainless(reading) .map( SQLTypeUtils - .coerce(_, reading.sqlType, out, nullable = false, context) + .coerce(_, reading.sqlType, compared, nullable = false, context) ) ) - .getOrElse(SQLTypeUtils.coerce(cond, out, context)) + .getOrElse(SQLTypeUtils.coerce(cond, compared, context)) ) val r = boxWhenNoDefault(res match { @@ -1229,11 +1268,12 @@ package object cond { // (`addParam` returns the existing one), and `c != null` on a candidate // that cannot be null is redundant bytes, never a different answer. ctx.addParam(LiteralParam(c)) match { - case Some(bound) => checkCase(e, bound, r, candidateNullable = true) - case _ => checkCase(e, c, r, candidateNullable = false) + case Some(bound) => + checkCase(e, bound, r, candidateNullable = true, compared) + case _ => checkCase(e, c, r, candidateNullable = false, compared) } case _ => - checkCase(e, c, r, candidateNullable = false) + checkCase(e, c, r, candidateNullable = false, compared) } } .mkString(" : ") @@ -1344,15 +1384,17 @@ package object cond { * yields NULL only when every argument is NULL: pairwise(x, y) = (x == null ? y : (y == null ? x * : Math.{max|min}(x, y))) * - * Over numbers the base type narrows to the widest input numeric. Over dates and timestamps (the - * lead's ruling of 2026-10-05: every SQL engine compares them, and they failed in Elasticsearch - * before) the same fold runs over the epoch milliseconds each argument denotes -- a DATE its UTC - * day's start, through the conversion date arithmetic uses - * (`ArithmeticExpression.instantMillis`) -- so dates compare as instants whatever each one - * renders as (a doc value, a `LocalDate`, the clock), and a NULL is skipped exactly as it is for - * numbers. The result is the instant that wins, a UTC `ZonedDateTime` (TIMESTAMP is what it - * renders); per group, where a date is the number its metric is, it is that number. Its reported - * type is its arguments' common one: DATE for dates, TIMESTAMP when they mix. + * Over numbers the type is the common type of ALL the arguments ([[numericType]]), as PostgreSQL + * types it, and every row answers it: per document the value is converted to it on every path + * ([[typedResult]]). Over dates and timestamps (the lead's ruling of 2026-10-05: every SQL + * engine compares them, and they failed in Elasticsearch before) the same fold runs over the + * epoch milliseconds each argument denotes -- a DATE its UTC day's start, through the conversion + * date arithmetic uses (`ArithmeticExpression.instantMillis`) -- so dates compare as instants + * whatever each one renders as (a doc value, a `LocalDate`, the clock), and a NULL is skipped + * exactly as it is for numbers. The result is the instant that wins, a UTC `ZonedDateTime` + * (TIMESTAMP is what it renders); per group, where a date is the number its metric is, it is + * that number. Its reported type is its arguments' common one: DATE for dates, TIMESTAMP when + * they mix. */ sealed trait NumericReducer extends TransformFunction[SQLAny, SQLNumeric] @@ -1360,6 +1402,10 @@ package object cond { def values: List[PainlessScript] def operator: ConditionalOp protected def mathFn: String // "Math.max" | "Math.min" + protected def greatest: Boolean // GREATEST (the larger wins) or LEAST + + /** The comparison under which the left argument wins. */ + private def keepsLeft: String = if (greatest) ">=" else "<=" override def fun: Option[ConditionalOp] = Some(operator) @@ -1377,13 +1423,46 @@ package object cond { */ lazy val overDates: Boolean = values.exists(v => NumericReducer.dateCarrying(renderedTypeOf(v))) + /** The numeric type of `GREATEST` / `LEAST`: the common type of ALL its arguments, as + * PostgreSQL resolves it (its CASE / UNION rule), which every row then has. + * - whole numbers only: the widest of them -- an INT beside a BIGINT is a BIGINT; + * - any other number -- a DOUBLE, a REAL, a NUMERIC of no known kind (`SQRT`, `ROUND`) -- + * makes it fractional: a DOUBLE, or a REAL when every fractional argument is a REAL. + * + * Each argument is read by the type it is DECLARED with (`ComparisonRule.declaredTypeOf`: its + * reported type, the metric for an aggregate), never by `out` ([[argTypes]]): a column + * resolved against its schema answers its COLUMN's type there whatever function it wears, so + * `CAST(n AS DOUBLE)` over an INT `n` read INT, the arguments looked whole, and the winner was + * chosen as a whole number -- the literal `1` itself where `n` is below 1 or NULL. A bare NULL + * has no type and is left out. + * + * `None` when an argument is not a number (dates take [[overDates]]), or when one's type is + * still unknown (no schema attached) and no fractional argument decides the type: an unknown + * column may hold fractions, so it is never read as a whole number. The reducer then renders + * as it always did. + */ + private[cond] lazy val numericType: Option[SQLNumeric] = { + val declared = values.map(ComparisonRule.declaredTypeOf).filterNot(_ == SQLTypes.Null) + val fractional = declared.filter(t => t.isNumber && !NumericReducer.wholeNumber(t)) + if (declared.isEmpty || declared.exists(t => !t.isNumber && !t.isUnknown)) None + else if (declared.forall(NumericReducer.wholeNumber)) + SQLTypeUtils.leastCommonSuperType(declared) match { + case whole: SQLNumeric => Some(whole) + case _ => None + } + else if (fractional.isEmpty) None + else if (fractional.forall(_ == SQLTypes.Real) && !declared.exists(_.isUnknown)) + Some(SQLTypes.Real) + else Some(SQLTypes.Double) + } + override def baseType: SQLType = if (overDates) SQLTypes.Timestamp else - SQLTypeUtils.leastCommonSuperType(argTypes) match { + numericType.getOrElse(SQLTypeUtils.leastCommonSuperType(argTypes) match { case n: SQLNumeric => n case _ => outputType - } + }) override def sql: String = s"$operator(${values.map(_.sql).mkString(", ")})" @@ -1432,8 +1511,12 @@ package object cond { ) } - /** The least common supertype of the arguments' REPORTED types, as `COALESCE` reports. */ - override def reportedType: SQLType = SQLTypeUtils.leastCommonSuperType(reportedArgTypes) + /** Over numbers, [[numericType]] -- the type every row answers, so DESCRIBE, a `CREATE TABLE … + * AS SELECT` and JDBC metadata declare what is stored; otherwise the least common supertype of + * the arguments' REPORTED types, as `COALESCE` reports. + */ + override def reportedType: SQLType = + numericType.getOrElse(SQLTypeUtils.leastCommonSuperType(reportedArgTypes)) override def nullable: Boolean = values.forall(_.nullable) @@ -1464,37 +1547,144 @@ package object cond { // Right-fold pairwise: pairwise(a, pairwise(b, pairwise(c, …))). // The combined sub-tree is itself nullable only when BOTH sides can be // null, which lets us drop the guard on parent levels too. - private def fold(args: List[(String, Boolean)]): (String, Boolean) = + private def fold(args: List[(String, Boolean)], integral: Boolean): (String, Boolean) = args match { case Nil => throw new IllegalArgumentException(s"$operator requires at least one argument") case (s, n) :: Nil => (s.trim, n) case (s, xNullable) :: rest => val x = s.trim - val (y, yNullable) = fold(rest) + val (y, yNullable) = fold(rest, integral) + val best = if (integral) winner(x, y) else s"$mathFn($x, $y)" val expr = (xNullable, yNullable) match { - case (true, true) => s"($x == null ? $y : ($y == null ? $x : $mathFn($x, $y)))" - case (true, false) => s"($x == null ? $y : $mathFn($x, $y))" - case (false, true) => s"($y == null ? $x : $mathFn($x, $y))" - case (false, false) => s"$mathFn($x, $y)" + case (true, true) => s"($x == null ? $y : ($y == null ? $x : $best))" + case (true, false) => s"($x == null ? $y : $best)" + case (false, true) => s"($y == null ? $x : $best)" + case (false, false) => best } // result is null only if every branch can be null (expr, xNullable && yNullable) } + /** The argument that wins, chosen by a comparison: Painless's `Math.max` / `Math.min` take + * `double`s only, so over whole numbers they answered `3.0` for `GREATEST(n, 2)` over an + * INTEGER `n` -- the right number in the wrong type (issue #380) -- and lost the precision of + * a BIGINT past 2^53. The winner is then converted with every other path ([[typedResult]]). + * Two whole-number literals are compared now, so the winner is the literal itself -- never a + * conditional over constants, which Elasticsearch 6.8 refuses as a whole script. + */ + private def winner(x: String, y: String): String = { + def literal(v: String): Option[BigInt] = + if (WholeLiteral.pattern.matcher(v).matches()) + scala.util.Try(BigInt(v.stripSuffix("L").stripSuffix("l"))).toOption + else None + (literal(x), literal(y)) match { + case (Some(a), Some(b)) => if (if (greatest) a >= b else a <= b) x else y + case _ => s"($x $keepsLeft $y ? $x : $y)" + } + } + + /** Are the arguments whole numbers -- [[numericType]] is one -- each rendered as a name, a + * literal or one parenthesised group, compared per document? Then the winner is chosen by + * [[winner]]. Per group -- no context, or a view's per-group calculation -- every metric is + * the double Elasticsearch computed, and the reducer keeps `Math.max` / `Math.min`; so does an + * argument rendered as a bare conditional (a function's NULL guard), which a comparison would + * read wrongly, and a bare `NULL`, which has no type and no comparison. + */ + private def overWholeNumbers( + callArgs: List[String], + context: Option[PainlessContext] + ): Boolean = + context.exists(!_.isTransform) && numericType.exists(NumericReducer.wholeNumber) && + !values.exists(ArithmeticExpression.bareNull) && + callArgs.forall(a => NumericReducer.oneValue(a.trim)) + + /** The value converted to [[numericType]], per document: ONE conversion every path goes through + * -- the winner, each NULL-skipping branch, a nested `GREATEST` / `LEAST`, a literal argument. + * + * The fold hands back the argument that survives as it is (`x == null ? y : …` answers `y` + * itself), and the winner as it is, so a column answered two classes: `GREATEST(n, x)` gave + * the INT `n` where the DOUBLE `x` is NULL (`Integer` -2 beside `Double` values), and + * `GREATEST(GREATEST(n, 0), x)` the `Integer` 0. MEASURED on Elasticsearch 8.18.3 and 6.8.23. + * `Math.max` hid it while it answered a double, except in the NULL-skipping branches. + * + * Per group -- no context, or a view's per-group calculation -- every metric is the double + * Elasticsearch computed, and the rendering is the one it always was. With a number literal + * among the arguments the value is never NULL and is converted as it stands; otherwise it is + * read once, under a name, behind a NULL guard whose NULL branch is that name -- a `def`, as + * the converted value is, where a `null` literal would make the conditional unusable as the + * argument of an outer `Math.max` (`Cannot cast null to a primitive type [double]`). + */ + private def typedResult(value: String, context: Option[PainlessContext]): String = + perDocumentType(context) match { + case Some((ctx, t)) => + val cast = SQLTypeUtils.painlessType(t) + val v = value.trim + if (values.exists(NumericReducer.numberLiteral)) + s"(($cast) ${if (NumericReducer.oneValue(v)) v else s"($v)"})" + else { + val isName = + ArithmeticExpression.isBareName(v) && (v.head.isLetter || v.head == '_') + val name = if (isName) v else ctx.bindLocal(v, "lv") + s"($name == null ? $name : (def) (($cast) $name))" + } + case _ => value + } + + /** The context a value is converted in, and the type it is converted to: per document only. */ + private def perDocumentType( + context: Option[PainlessContext] + ): Option[(PainlessContext, SQLNumeric)] = + for { + ctx <- context.filterNot(_.isTransform) + t <- numericType + } yield (ctx, t) + + /** A literal argument as a literal of [[numericType]] where its own spelling is not one: a + * whole number past the range of an `int` is a `long` literal (`3000000000L`) beside a BIGINT, + * which Painless refuses to compile without its suffix (`Invalid int constant`). + */ + private def typedArgument( + rendered: String, + argument: PainlessScript, + context: Option[PainlessContext] + ): String = + perDocumentType(context) match { + case Some((_, SQLTypes.BigInt)) if NumericReducer.numberLiteral(argument) => + val r = rendered.trim + if ( + WholeLiteral.pattern.matcher(r).matches() && !r.endsWith("L") && !r.endsWith("l") && + !BigInt(r).isValidInt + ) s"${r}L" + else rendered + case _ => rendered + } + override def toPainlessCall( callArgs: List[String], context: Option[PainlessContext] ): String = if (overDates) overDatesPainless(callArgs, context) - else - callArgs match { - case Nil => - throw new IllegalArgumentException(s"$operator requires at least one argument") - case x :: Nil => x - case _ => fold(callArgs.zip(values.map(nullableArgument(_, context))))._1 - } + else { + val typedArgs = + callArgs.zip(values).map { case (rendered, argument) => + typedArgument(rendered, argument, context) + } + typedResult( + typedArgs match { + case Nil => + throw new IllegalArgumentException(s"$operator requires at least one argument") + case x :: Nil => x + case _ => + fold( + typedArgs.zip(values.map(nullableArgument(_, context))), + overWholeNumbers(typedArgs, context) + )._1 + }, + context + ) + } /** The same fold over the epoch milliseconds each date denotes, then the instant that wins. * @@ -1542,7 +1732,7 @@ package object cond { } } if (millis.isEmpty) return if (perDocument.isEmpty) "((def) null)" else "null" - val (winner, nullable) = fold(millis) + val (winner, nullable) = fold(millis, integral = false) perDocument match { case Some(ctx) => val name = if (nullable) ctx.bindLocal(winner, "lv") else s"($winner)" @@ -1556,6 +1746,7 @@ package object cond { case class Greatest(values: List[PainlessScript]) extends NumericReducer { override def operator: ConditionalOp = Greatest override protected def mathFn: String = "Math.max" + override protected def greatest: Boolean = true override def update(request: query.SingleSearch): Greatest = this.copy(values = values.map { @@ -1567,6 +1758,7 @@ package object cond { case class Least(values: List[PainlessScript]) extends NumericReducer { override def operator: ConditionalOp = Least override protected def mathFn: String = "Math.min" + override protected def greatest: Boolean = false override def update(request: query.SingleSearch): Least = this.copy(values = values.map { @@ -1584,6 +1776,46 @@ package object cond { t == SQLTypes.Date || t == SQLTypes.Timestamp || t == SQLTypes.DateTime || t == SQLTypes.Temporal + /** A whole-number type: TINYINT, SMALLINT, INT or BIGINT. */ + private[cond] def wholeNumber(t: SQLType): Boolean = t match { + case _: SQLTinyInt | _: SQLSmallInt | _: SQLInt | _: SQLBigInt => true + case _ => false + } + + /** A number literal -- bare, or the one function of the nameless identifier the parser wraps an + * operand in -- which is never NULL. + */ + private[cond] def numberLiteral(v: PainlessScript): Boolean = v match { + case _: NumericValue[_] => true + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (_: NumericValue[_]) :: Nil => true + case _ => false + } + case _ => false + } + + /** A rendering that is ONE value wherever it is placed: a name, a literal, or a single + * parenthesised group. + */ + private[cond] def oneValue(rendered: String): Boolean = + simpleOperand.pattern.matcher(rendered).matches() || (rendered.startsWith("(") && { + var depth = 0 + var i = 0 + var closedEarly = false + while (i < rendered.length && !closedEarly) { + rendered.charAt(i) match { + case '(' => depth += 1 + case ')' => + depth -= 1 + if (depth == 0 && i < rendered.length - 1) closedEarly = true + case _ => + } + i += 1 + } + !closedEarly && depth == 0 + }) + /** Is this operand -- or the nameless identifier carrying it -- `GREATEST` / `LEAST` over * dates? Per group its value is the epoch milliseconds of the date that wins * (`ArithmeticExpression.epochMillis`). diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index 9cf99f93e..e46139854 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ query, + renderedTypeOf, Alias, DateMathRounding, Expr, @@ -28,6 +29,7 @@ import app.softnetwork.elastic.sql.{ Updateable } import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.function.cond.functionsOf import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression package object convert { @@ -195,8 +197,47 @@ package object convert { override def roundingScript: Option[String] = DateMathRounding(targetType) override def dateMathScript: Boolean = isTemporal + + /** A NUMBER converted to a DATE is refused by name, as PostgreSQL refuses it (`cannot cast type + * integer to date`) -- `(d - d2)::DATE`, `CAST(n AS DATE)`, `CAST(YEAR(d) AS DATE)`. A number + * is no date, and no conversion makes it one: the number used to pass through unconverted + * under a DATE label (`(d - d2)::DATE` answered -1, 2 and 0). + * + * Decided on what the operand RENDERS ([[convertedType]]): `YEAR(d)` is a number although its + * column is a date. Asked once the column types are known, like every type rule; an operand + * whose type is still unknown is accepted. A number cast to a TIMESTAMP keeps core's + * epoch-millisecond conversion. + */ + override def typeError: Option[String] = + if (targetType != SQLTypes.Date) None + else { + val source = convertedType + if (!source.isNumber) None + else + Some( + s"Type mismatch: cannot cast ${source.typeId} to DATE in expression: $sql; a number " + + "is not a date: add a number of days to a date instead" + ) + } + + /** The type of the value converted, as it renders: its outermost function's own type -- never + * that function's `out`, which this conversion's constructor sets to the TARGET type + * (`value.cast(targetType)`), so that `CAST(1 AS DATE)` and `CAST(YEAR(d) AS DATE)` would read + * as dates -- an aggregate's metric, or the column's type. + */ + private def convertedType: SQLType = value match { + case i: Identifier if !i.isAggregation && i.functions.nonEmpty => i.functions.head.baseType + case other => renderedTypeOf(other) + } } + /** Every conversion type error in an expression ([[Conversion.typeError]]), wherever a conversion + * sits in it -- the walk a computed column's resolved table asks, as it asks the date-arithmetic + * rules (`ArithmeticExpression.typeErrorsOf`). + */ + def typeErrorsOf(chain: FunctionChain): Seq[String] = + functionsOf(chain).collect { case c: Conversion => c }.flatMap(_.typeError).distinct + case object Cast extends Expr("CAST") with TokenRegex case object TryCast extends Expr("TRY_CAST") with TokenRegex { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 14567aa12..713a8b3d3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ query, + renderedTypeOf, Expr, IntValue, PainlessContext, @@ -26,7 +27,15 @@ import app.softnetwork.elastic.sql.{ TokenRegex, Updateable } -import app.softnetwork.elastic.sql.`type`.{SQLNumeric, SQLType, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{ + SQLBigInt, + SQLInt, + SQLNumeric, + SQLSmallInt, + SQLTinyInt, + SQLType, + SQLTypes +} package object math { @@ -99,6 +108,43 @@ package object math { arg: PainlessScript ) extends MathematicalFunction { override def args: List[PainlessScript] = List(arg) + + /** `ABS` is typed by its argument, as PostgreSQL's `abs(integer) -> integer`: `ABS(n)` over an + * INT column is an INT, so a literal beside it is read as that type's value -- `ABS(n) = + * '1.5'` is refused by name like `n = '1.5'`, where NUMERIC read `'1.5'` as a number -- and a + * `CREATE TABLE … AS SELECT` declares an INT, where NUMERIC could not create the index. An + * argument whose type is not a known number keeps NUMERIC. + * + * The type REPORTED, not the one Painless is handed (`out`, still NUMERIC): what a consumer + * converts this value to is unchanged. + */ + override def reportedType: SQLType = mathOp match { + case Abs => + arg.reportedType match { + case n: SQLNumeric => n + case _ => super.reportedType + } + case _ => super.reportedType + } + + /** `ABS` of a whole number answers a whole number of its argument's type per document, as its + * type says -- Painless's `Math.abs` takes a `double` only, so it answered `3.0` for an INT + * `-3`. The argument is a parameter or a literal, read at most twice. Per group -- no context, + * or a view's per-group calculation -- every metric is the double Elasticsearch computed, and + * the rendering is the one it always was. + */ + override def toPainlessCall( + callArgs: List[String], + context: Option[PainlessContext] + ): String = + (mathOp, renderedTypeOf(arg), callArgs) match { + case (Abs, _: SQLTinyInt | _: SQLSmallInt | _: SQLInt | _: SQLBigInt, List(a)) + if context.exists(!_.isTransform) => + val x = s"(${a.trim})" + s"($x < 0 ? -$x : $x)" + case _ => super.toPainlessCall(callArgs, context) + } + override def update(request: query.SingleSearch): MathematicalFunctionWithOp = { arg match { case updatable: Updateable => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 66e213339..7a9d55a35 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -20,7 +20,7 @@ import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.`type`._ import app.softnetwork.elastic.sql.function.{BinaryFunction, FunctionChain, TransformFunction} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction -import app.softnetwork.elastic.sql.function.cond.{functionsOf, NumericReducer} +import app.softnetwork.elastic.sql.function.cond.{functionsOf, Case, NullIf, NumericReducer} import app.softnetwork.elastic.sql.function.time.DateDiff import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.query.NestedElement @@ -415,7 +415,45 @@ case class ArithmeticExpression( * the operand is known non-null. */ private def coercionSkipped(operand: PainlessScript, target: SQLType): Boolean = - operand.nullable && renderedType(operand).isNumber && target.isNumber + mayBeNull(operand) && renderedType(operand).isNumber && target.isNumber + + /** A `CASE` or a `NULLIF` written as an operand -- the nameless identifier the parser wraps it in + * -- is ONE value, and its rendering is a CONDITIONAL (`c ? a : b`, `x == 1 ? null : x`) over + * which `+`, `-`, `*`, `/` and `%` bind tighter: spliced as it stands, `CASE WHEN n > 0 THEN n + * ELSE 0 END + 1` was `param2 ? n : 0 + 1`, the ELSE branch plus one -- 3, 5, 1, 1, 10 where + * PostgreSQL answers 4, 6, 1, 1, 11 (issue #380), SQL parentheses or not. + */ + private def conditionalOperand(operand: PainlessScript): Boolean = operand match { + case i: Identifier if i.name.trim.isEmpty && !i.isAggregation => + i.functions.headOption.exists { + case _: Case | _: NullIf => true + case _ => false + } + case _ => false + } + + /** Can this operand be NULL when it runs? Its own nullability, or -- for a `CASE` or a `NULLIF` + * operand, whose nameless identifier answers `false` whatever it holds -- the function's: a + * `NULLIF` exists to produce NULL, and a `CASE` is NULL when a branch can be or it has no + * `ELSE`. Unguarded, `NULLIF(n, 1) + 1` threw on the very row it nulls (issue #380). + */ + private def mayBeNull(operand: PainlessScript): Boolean = + operand.nullable || (operand match { + case i: Identifier if conditionalOperand(i) => + i.functions.head match { + case c: Case => c.nullable + case _ => true + } + case _ => false + }) + + /** A conditional operand's rendering, parenthesised to be the ONE value it is wherever an + * operator or a conversion is applied to it; anything else, and a bare name, as it stands. + */ + private def asOneValue(operand: PainlessScript, rendered: String): String = + if (conditionalOperand(operand) && !ArithmeticExpression.isBareName(rendered.trim)) + s"($rendered)" + else rendered private def isFloating(t: SQLType): Boolean = t == SQLTypes.Double || t == SQLTypes.Real @@ -491,7 +529,7 @@ case class ArithmeticExpression( /** A NULL literal beside a temporal makes the expression NULL on every row, whatever the other * operand holds, so a consumer must guard it. */ - override def nullable: Boolean = left.nullable || right.nullable || nullValued + override def nullable: Boolean = mayBeNull(left) || mayBeNull(right) || nullValued override def toPainless(base: String, idx: Int, context: Option[PainlessContext]): String = { context match { @@ -523,7 +561,7 @@ case class ArithmeticExpression( SQLTypeUtils.coerce(rendered, renderedType(operand), target, nullable = false, context) def render(operand: PainlessScript): String = - coerced(operandPainless(operand, idx, context), operand) + coerced(asOneValue(operand, operandPainless(operand, idx, context)), operand) /** A rendering that IS a Painless name needs no local of its own. */ def isBareName(e: String): Boolean = @@ -586,7 +624,7 @@ case class ArithmeticExpression( } val leftParam = - if (left.nullable && !isBareName(l)) bind(l, s"lv$idx") else l + if (mayBeNull(left) && !isBareName(l)) bind(l, s"lv$idx") else l /** πŸ”΄ The divisor is bound when it is not a bare NAME, even if it is not nullable -- because * the zero guard reads it a SECOND time. `$rightParam` appears in `($rightParam == 0)` and @@ -599,7 +637,7 @@ case class ArithmeticExpression( * A non-nullable, non-name LEFT operand needs no such binding: it is read once. */ val rightParam = - if (!isBareName(r) && (right.nullable || (floatingDivision(out) && divisorMayBeZero))) + if (!isBareName(r) && (mayBeNull(right) || (floatingDivision(out) && divisorMayBeZero))) bind(r, s"rv$idx") else r @@ -616,8 +654,8 @@ case class ArithmeticExpression( */ val lhs = if (needsDoubleCast(target)) s"((double) $leftParam)" else leftParam val guards = - (if (left.nullable) List(s"$leftParam == null") else Nil) ::: - (if (right.nullable) List(s"$rightParam == null") else Nil) ::: + (if (mayBeNull(left)) List(s"$leftParam == null") else Nil) ::: + (if (mayBeNull(right)) List(s"$rightParam == null") else Nil) ::: (if (floatingDivision(target) && divisorMayBeZero && !divisorIsLiteralZero) List(s"$rightParam == 0") else Nil) @@ -800,6 +838,14 @@ case class ArithmeticExpression( case literal if literalNumber(literal).isDefined => val number = literalNumber(literal).get SQLTypeUtils.coerce(number.painless, number.sqlType, target, nullable = false, context) + case conditional if conditionalOperand(conditional) => + SQLTypeUtils.coerce( + asOneValue(conditional, conditional.painless(context)), + conditional.baseType, + target, + nullable = false, + context + ) case _ => SQLTypeUtils.coerce(operand, target, context) } val l = coerced(left) @@ -984,7 +1030,7 @@ object ArithmeticExpression { * DATE, `CAST(NULL AS INT)` a whole number -- and a typed NULL is then judged by its type, * whatever its value. */ - private def bareNull(operand: PainlessScript): Boolean = operand match { + private[sql] def bareNull(operand: PainlessScript): Boolean = operand match { case Null => true case i: Identifier => i.name.trim.isEmpty && i.functions == List(Null) case _ => false diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 3b2ec46a1..17cc09a09 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -24,6 +24,7 @@ import app.softnetwork.elastic.sql.function.{ FunctionN, FunctionWithIdentifier } +import app.softnetwork.elastic.sql.function.convert import app.softnetwork.elastic.sql.function.cond.{Case, NullIf} import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Parser @@ -968,6 +969,26 @@ package object schema { case errors => return Left(errors.mkString("; ")) } + // A number converted to a DATE -- `(d - d2)::DATE`, `CAST(n AS DATE)` -- refused by name for + // the same reasons, as a query refuses it (`Conversion.typeError`): the script would store the + // number itself in a DATE column. + def conversionErrors(column: Column): Seq[String] = + column.script + .filterNot(_.materialized) + .toSeq + .flatMap(_.validationExpr) + .flatMap { + case chain: FunctionChain => convert.typeErrorsOf(chain) + case _ => Nil + } + .map(reason => s"Column '${column.path}': $reason") ++ + column.multiFields.flatMap(conversionErrors) + + schema.columns.flatMap(conversionErrors).distinct match { + case Nil => + case errors => return Left(errors.mkString("; ")) + } + if (!schema.isRegular) return Right(()) def columnErrors(column: Column): Seq[String] = diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/ParameterIdentitySpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ParameterIdentitySpec.scala index 86e4d7530..7c3658e82 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/ParameterIdentitySpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ParameterIdentitySpec.scala @@ -291,7 +291,8 @@ class ParameterIdentitySpec extends AnyFlatSpec with Matchers { // `CAST(lastUpdated AS date) - INTERVAL 3 DAY` must key apart from the bare `lastUpdated` in // THEN, else THEN returns `lastUpdated - 3 days`. The interval lands on the CAST's parameter. // Executed on ES 8.18 (NOW = 2025-01-08): a -> createdAt 2025-01-01, d -> lastSeen + 2 = - // 2025-01-03, e -> lastUpdated = 2025-01-04 (the old pin answered 2025-01-01 for e). + // 2025-01-03, e -> lastUpdated = 2025-01-04 (the old pin answered 2025-01-01 for e) -- with + // the THEN values then converted to the operand's DATE; they are now the values themselves. // ⚠️ That is the NO-schema rendering, the one the bridge fixtures pin. The SCHEMA rendering of // this same statement fails the shard on `main` AND here (`Class.cast` at // `param2.isEqual(param5)`: `CURRENT_DATE - 7` is coerced to a `ZonedDateTime` against a @@ -301,20 +302,25 @@ class ParameterIdentitySpec extends AnyFlatSpec with Matchers { "SELECT CASE CURRENT_DATE - INTERVAL 7 DAY WHEN CAST(lastUpdated AS date) - INTERVAL 3 DAY THEN lastUpdated " + "WHEN lastSeen THEN lastSeen + INTERVAL 2 DAY ELSE createdAt END AS c, identifier FROM Table" ) + // The THEN value is the bare column, as it is: a simple CASE's value is typed by its THEN / + // ELSE values, never converted to its operand's DATE. emitted should include( "def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); " + - "def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); " + "def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); " ) // The candidate is guarded too (issue #373): `lastSeen` is a column and a document may not // have it, which was an NPE before. emitted should endWith( "? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6" ) - // Rule 2, normalisation FIRST: the branch's DATE narrowing was appended to this object before - // the chain rendered; the UTC normalisation must still come first (`LocalDate` has no - // `toInstant()`). + // Rule 2, normalisation FIRST: the UTC normalisation comes before the interval on the + // branch's own object (`LocalDate` has no `toInstant()`), and the WHEN candidate's DATE + // narrowing stays on a parameter of its own. emitted should include( - "doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate().plus(2, ChronoUnit.DAYS)" + "doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).plus(2, ChronoUnit.DAYS)" + ) + emitted should include( + "def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); " ) } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala index 1dd562a92..b20bf339a 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala @@ -460,4 +460,248 @@ class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDriven "PRIMARY KEY (id))" ) should include("== true ? 1 : 0") } + + /** The type every SELECT item REPORTS -- what a `CREATE TABLE … AS SELECT` declares -- by alias. + */ + private def reported(sql: String): Map[String, String] = { + val search = parsed(sql).update(Some(schema)) + search.validateResolved() shouldBe Right(()) + search.select.fields.map { f => + f.fieldAlias.map(_.alias).getOrElse(f.identifier.sql) -> f.identifier.reportedType.typeId + }.toMap + } + + /** A NUMBER converted to a DATE is refused by name, as PostgreSQL refuses it (`cannot cast type + * integer to date`) -- in a query, per group and in a computed column; a number converted to a + * TIMESTAMP keeps core's epoch-millisecond conversion. + */ + "a number converted to a DATE" should "be refused by name, in a query and a computed column" in { + forAll( + Table( + ("sql", "refusal"), + ("SELECT (d - d2)::DATE AS v FROM t", "cannot cast BIGINT to DATE"), + ("SELECT CAST(n AS DATE) AS v FROM t", "cannot cast INT to DATE"), + ("SELECT n::DATE AS v FROM t", "cannot cast INT to DATE"), + ("SELECT CAST(x AS DATE) AS v FROM t", "cannot cast DOUBLE to DATE"), + ("SELECT CAST(1 AS DATE) AS v FROM t", "cannot cast BIGINT to DATE"), + ( + "SELECT CAST(YEAR(d) AS DATE) AS v FROM t", + "to DATE in expression: CAST(YEAR(d) AS DATE)" + ), + ("SELECT TRY_CAST(n AS DATE) AS v FROM t", "cannot cast INT to DATE"), + ("SELECT CONVERT(n, DATE) AS v FROM t", "cannot cast INT to DATE"), + ("SELECT id FROM t WHERE (d - d2)::DATE = d", "cannot cast BIGINT to DATE"), + ("SELECT g, (MAX(d) - MIN(d))::DATE AS v FROM t GROUP BY g", "cannot cast BIGINT to DATE") + ) + ) { (sql, refusal) => + withClue(s"[$sql] ") { + parsed(sql) + resolved(sql).swap.getOrElse(fail("accepted once the types are known")) should include( + refusal + ) + } + } + Parser( + "CREATE TABLE z (id INT, n INT, c DATE SCRIPT AS (CAST(n AS DATE)), PRIMARY KEY (id))" + ) match { + case Left(error) => + error.msg should include("Column 'c': Type mismatch: cannot cast INT to DATE") + case other => fail(s"a computed column CAST(n AS DATE) was accepted: $other") + } + forAll( + Table( + "sql", + "SELECT (d - d2)::TIMESTAMP AS v FROM t", + "SELECT CAST(n AS TIMESTAMP) AS v FROM t", + "SELECT (d - d2)::BIGINT AS v FROM t", + "SELECT CAST(ts AS DATE) AS v FROM t", + "SELECT CAST('2024-01-31' AS DATE) AS v FROM t" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + } + + /** A `CASE` or a `NULLIF` used as an operand is ONE value (issue #380): the operator applies to + * every branch -- never `c ? n : 0 + 1`, the ELSE branch plus one -- and a NULL it produces + * makes the result NULL rather than failing the script. + */ + "a CASE or a NULLIF used as an operand" should + "be rendered as one value, in a query and in a computed column" in { + val notNull = fieldPainless("SELECT (CASE WHEN n > 0 THEN 1 ELSE 0 END) + 1 AS v FROM t") + withClue(notNull) { + notNull should not include ": 0 + 1" + notNull should include("? 1 : 0) + 1") + } + Seq( + "SELECT CASE WHEN n > 0 THEN n ELSE 0 END + 1 AS v FROM t", + "SELECT (CASE WHEN n > 0 THEN n ELSE 0 END) + 1 AS v FROM t", + "SELECT CASE WHEN n > 0 THEN n END + 1 AS v FROM t" + ).foreach { sql => + val script = fieldPainless(sql) + withClue(s"[$sql] $script ") { + script should not include ": 0 + 1" + script should fullyMatch regex + """.*def (lv\d+) = \(.*\); \(\1 == null\) \? null : \(\1 \+ 1\)""" + } + } + val nullIf = fieldPainless("SELECT NULLIF(n, 3) + 1 AS v FROM t") + withClue(nullIf) { + nullIf should fullyMatch regex """.*\((param\d+) == null\) \? null : \(\1 \+ 1\)""" + } + val stored = ingestPainless( + "CREATE TABLE z (id INT, n INT, c BIGINT SCRIPT AS (CASE WHEN n > 0 THEN n ELSE 0 END + 1), " + + "c2 INT SCRIPT AS (NULLIF(n, 3) + 1), PRIMARY KEY (id))" + ) + withClue(stored) { + stored should not include ": 0 + 1" + stored should include("== null) ? null : (") + } + } + + /** `GREATEST` / `LEAST` over whole numbers keep the winner's whole type (issue #380): Painless's + * `Math.max` / `Math.min` take `double`s only. + */ + "GREATEST / LEAST over whole numbers" should "choose the winner by a comparison" in { + val greatest = fieldPainless("SELECT GREATEST(n, 2) AS v FROM t") + withClue(greatest) { + greatest should not include "Math.max" + greatest should include(">= 2 ?") + } + val least = fieldPainless("SELECT LEAST(n, big) AS v FROM t") + withClue(least) { + least should not include "Math.min" + least should include(" <= ") + } + fieldPainless("SELECT GREATEST(x, 2) AS v FROM t") should include("Math.max") + // two literals: the winner is the literal itself, converted to the BIGINT both are -- never a + // conditional over constants, which Elasticsearch 6.8 refuses as a whole script + val literals = fieldPainless("SELECT GREATEST(2, 4) AS v FROM t") + literals should endWith("((long) 4)") + literals should not include "?" + } + + /** `GREATEST` / `LEAST` answer the common type of ALL their arguments on every row, as PostgreSQL + * types them, and report it: each argument is read by the type it is declared with -- a `CAST` + * by the type it casts to, never by its column's -- and per document the value is converted to + * that type on every path, the argument that survives a NULL included. + */ + "GREATEST / LEAST" should "answer and report the common type of their arguments" in { + reported( + "SELECT GREATEST(CAST(n AS DOUBLE), 1) AS a, GREATEST(n, x) AS b, GREATEST(n, 2) AS c, " + + "GREATEST(n, CAST(n AS BIGINT)) AS d, LEAST(n, r) AS e, GREATEST(ABS(x), 1) AS f FROM t" + ) shouldBe Map( + "a" -> "DOUBLE", + "b" -> "DOUBLE", + "c" -> "BIGINT", + "d" -> "BIGINT", + "e" -> "REAL", + "f" -> "DOUBLE" + ) + // a CAST to DOUBLE is a DOUBLE: compared by Math.max, never as a whole number + val cast = fieldPainless("SELECT GREATEST(CAST(n AS DOUBLE), 1) AS v FROM t") + withClue(cast) { + cast should include("Math.max(") + cast should not include ">= 1 ?" + cast should include("((double) (") + } + // the argument that survives a NULL is converted with the winner: the value is read once + val survivor = fieldPainless("SELECT GREATEST(n, x) AS v FROM t") + withClue(survivor) { + survivor should fullyMatch regex + """.*def (lv\d+) = \(.*\); \(\1 == null \? \1 : \(def\) \(\(double\) \1\)\)""" + } + // a whole literal past the range of an int is a long literal beside a BIGINT + fieldPainless("SELECT GREATEST(n, 3000000000) AS v FROM t") should include( + ">= 3000000000L ? param1 : 3000000000L)" + ) + } + + /** `ABS` is typed by its argument, as PostgreSQL's `abs(integer) -> integer`: a quoted fraction + * beside `ABS` of an integer is refused by name, a whole number read; its value is that integer. + */ + "ABS" should "be typed by its argument" in { + reported("SELECT ABS(n) AS a, ABS(big) AS b, ABS(x) AS c FROM t") shouldBe Map( + "a" -> "INT", + "b" -> "BIGINT", + "c" -> "DOUBLE" + ) + forAll( + Table( + ("sql", "refusal"), + ("SELECT id FROM t WHERE ABS(n) = '1.5'", "'INT' is not compatible with 'VARCHAR'"), + ("SELECT id FROM t WHERE ABS(n) = '1.0'", "'INT' is not compatible with 'VARCHAR'"), + ("SELECT id FROM t WHERE ABS(big) = '1.5'", "'BIGINT' is not compatible with 'VARCHAR'"), + ("SELECT id FROM t WHERE ABS(n) IN ('1', '1.5')", "'INT' is not compatible with 'VARCHAR'"), + ( + "SELECT id FROM t WHERE ABS(n) BETWEEN '1' AND '1.5'", + "output 'INT' is not compatible with input 'VARCHAR'" + ), + ("SELECT ABS(n) + '1.5' AS v FROM t", "is not compatible with input 'VARCHAR'"), + ("SELECT id FROM t WHERE ABS(n) + '1.5' > 0", "ABS(n) + '1.5'"), + ( + "SELECT NULLIF(ABS(n), '1.5') AS v FROM t", + "with VARCHAR: NULLIF requires two arguments of comparable types" + ), + ( + "SELECT CASE WHEN ABS(n) = '1.5' THEN 1 ELSE 0 END AS v FROM t", + "'INT' is not compatible with 'VARCHAR'" + ) + ) + ) { (sql, refusal) => + withClue(s"[$sql] ") { + parsed(sql) + resolved(sql).swap.getOrElse(fail("accepted once the types are known")) should include( + refusal + ) + } + } + forAll( + Table( + "sql", + "SELECT id FROM t WHERE ABS(n) = '2'", + "SELECT id FROM t WHERE ABS(x) = '1.5'", + "SELECT id FROM t WHERE ABS(n) > 2" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + fieldPainless("SELECT ABS(n) AS v FROM t") should not include "Math.abs" + fieldPainless("SELECT ABS(x) AS v FROM t") should include("Double.valueOf(Math.abs(") + } + + /** `NULLIF(, x)` is typed by the literal's READING beside `x` -- what it answers + * and what PostgreSQL types it -- never by the VARCHAR the literal is written as. + */ + "NULLIF with a quoted literal first" should "be typed by the literal's reading" in { + reported( + "SELECT NULLIF('2024-01-31', ts) AS a, NULLIF('2024-01-31T10:00:00Z', d) AS b, " + + "NULLIF('3', n) AS c, NULLIF('1.5', x) AS e, NULLIF('true', b) AS f, " + + "NULLIF(ts, '2024-01-31') AS h FROM t" + ) shouldBe Map( + "a" -> "TIMESTAMP", + "b" -> "DATE", + "c" -> "BIGINT", + "e" -> "DOUBLE", + "f" -> "BOOLEAN", + "h" -> "TIMESTAMP" + ) + } + + /** A simple `CASE` answers its `THEN` / `ELSE` values in their own type: the operand is only + * compared, in the type it is compared in, and never converts the results. + */ + "a simple CASE" should "answer its THEN / ELSE values in their own type" in { + reported( + "SELECT CASE s WHEN 'a' THEN 1 ELSE 0 END AS a, CASE x WHEN 1.5 THEN 1 ELSE 0 END AS b FROM t" + ) shouldBe Map("a" -> "BIGINT", "b" -> "BIGINT") + val overText = fieldPainless("SELECT CASE s WHEN 'a' THEN 1 ELSE 0 END AS v FROM t") + withClue(overText) { + overText should include(".compareTo(") + overText should endWith("? 1 : 0") + } + val overDouble = fieldPainless("SELECT CASE x WHEN 1.5 THEN 1 ELSE 0 END AS v FROM t") + withClue(overDouble)(overDouble should endWith("? 1 : 0")) + val stored = ingestPainless( + "CREATE TABLE z (id INT, s KEYWORD, c BIGINT SCRIPT AS (CASE s WHEN '1' THEN 1 ELSE 0 END), " + + "PRIMARY KEY (id))" + ) + withClue(stored)(stored should include("? 1 : 0")) + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala index 8b2ff6407..f39e830cb 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala @@ -117,6 +117,31 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with /** A BOOLEAN column as a CASE condition (`"a CASE WHEN over a BOOLEAN column"`). */ private val flagTable = "date_arithmetic_flag" + /** PostgreSQL's fixture for the operand tests (`"a CASE or a NULLIF used as an operand"` and the + * tests after it), and the computed-column tables and `CREATE TABLE … AS SELECT` targets they + * create. + */ + private val operandName = "date_arithmetic_operand" + private val operandTables = Seq( + operandName, + "date_arithmetic_operand_cc", + "date_arithmetic_greatest_cc", + "date_arithmetic_greatest_type_cc", + "date_arithmetic_greatest_long_cc", + "date_arithmetic_greatest_double", + "date_arithmetic_greatest_cast", + "date_arithmetic_greatest_whole", + "date_arithmetic_tocast", + "date_arithmetic_tocast_ctas", + "date_arithmetic_abs_cc", + "date_arithmetic_abs_ctas", + "date_arithmetic_nullif_ts", + "date_arithmetic_nullif_d", + "date_arithmetic_nullif_n", + "date_arithmetic_case_cc", + "date_arithmetic_case_ctas" + ) + private val DayMillis: Long = 86400000L /** One document: `None` is a column the document does not carry. @@ -299,7 +324,7 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with } override def afterAll(): Unit = { - Seq( + (Seq( table, computedTable, rawIndex, @@ -317,7 +342,9 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with nullifTable, booleanTable, flagTable - ).foreach(t => Try(Await.result(client.run(s"DROP TABLE IF EXISTS $t"), 60.seconds))) + ) ++ operandTables).foreach(t => + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $t"), 60.seconds)) + ) super.afterAll() } @@ -354,6 +381,29 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with private final case class Bool(value: Boolean) extends Expected private case object Null extends Expected + /** A whole number, answered in an integer type (`Integer`, `Long`) -- as a SQL engine answers an + * integer expression -- never as a `Double` or a string. + */ + private final case class Whole(value: Long) extends Expected + + /** A number answered as a NUMBER, never as the string spelling it. */ + private final case class Numeric(value: Double) extends Expected + + /** A number answered in exactly this class, on every row -- what tells a `Double` from an + * `Integer`, which `Num` and `Numeric` accept alike: a `Double` for a fractional type; for a + * whole one the class Elasticsearch's JSON hands it back in, an `Integer` when it fits one and a + * `Long` past it (a BIGINT 3 is read as an `Integer` 3, as `CAST(n AS BIGINT)` itself is). + */ + private final case class Classed(value: Double, cls: Class[_]) extends Expected + + private def doubleOf(value: Double): Expected = Classed(value, classOf[java.lang.Double]) + + private def wholeOf(value: Long): Expected = + Classed( + value.toDouble, + if (value.isValidInt) classOf[java.lang.Integer] else classOf[java.lang.Long] + ) + /** An instant read from the clock while the statement ran: between `from` and `to`. */ private final case class Within(from: Instant, to: Instant) extends Expected @@ -440,8 +490,34 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with case s: String if Try(s.toBoolean).toOption.contains(want) => None case other => Some(s"$other instead of $want") } + case Whole(want) => + scalar(actual) match { + case n @ (_: java.lang.Integer | _: java.lang.Long | _: java.lang.Short | + _: java.lang.Byte) if n.asInstanceOf[Number].longValue() == want => + None + case other => Some(s"${withClass(other)} instead of the whole number $want") + } + case Numeric(want) => + scalar(actual) match { + case n: java.lang.Number + if Math.abs(n.doubleValue() - want) <= 1e-9 * Math.max(1.0, Math.abs(want)) => + None + case other => Some(s"${withClass(other)} instead of the number $want") + } + case Classed(want, cls) => + scalar(actual) match { + case n: java.lang.Number + if n.getClass == cls && + Math.abs(n.doubleValue() - want) <= 1e-9 * Math.max(1.0, Math.abs(want)) => + None + case other => Some(s"${withClass(other)} instead of $want (${cls.getSimpleName})") + } } + /** A value with its class: `3.0 (Double)`, `"1" (String)`. */ + private def withClass(value: Any): String = + Option(value).fold("null")(v => s"$v (${v.getClass.getSimpleName})") + /** A row-level expression and its value for each document. */ private final case class RowForm(sql: String, value: Row => Expected) @@ -1717,6 +1793,683 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with ) } + /** PostgreSQL 16's fixture for the operand tests below, over the same rows: an INT that is + * positive, negative and absent, a DOUBLE with a fraction, a KEYWORD holding digits, a BOOLEAN + * absent from one document, two DATEs and a TIMESTAMP. Created on first use. + */ + private lazy val operandTable: String = { + run( + s"CREATE TABLE $operandName (id KEYWORD, g KEYWORD, s KEYWORD, n INT, x DOUBLE, d DATE, " + + "d2 DATE, ts TIMESTAMP, b BOOLEAN, PRIMARY KEY (id))" + ) + bulk(operandName, operandDocs) + operandName + } + + private val operandDocs = List( + """{"id":"r1","g":"g1","s":"1","n":3,"x":0.25,"d":"2024-01-31","d2":"2024-02-01","ts":"2024-01-31T10:30:00Z","b":true}""", + """{"id":"r2","g":"g1","s":"5","n":5,"x":1.5,"d":"2024-03-01","d2":"2024-02-28","ts":"2024-03-01T06:00:00Z","b":false}""", + """{"id":"r3","g":"g2","s":"12","n":-2,"d":"2023-12-25","ts":"2023-12-25T23:59:59Z"}""", + """{"id":"r4","g":"g3","x":2.0,"d2":"2024-01-31","b":true}""", + """{"id":"r5","g":"g3","s":"3","n":10,"x":-1.25,"d":"2024-02-29","d2":"2024-02-29","ts":"2024-02-29T00:00:00Z","b":false}""" + ) + + private def answers( + r1: Expected, + r2: Expected, + r3: Expected, + r4: Expected, + r5: Expected + ): Map[String, Expected] = + Map("r1" -> r1, "r2" -> r2, "r3" -> r3, "r4" -> r4, "r5" -> r5) + + /** The statement is refused, by a message carrying every fragment, before anything runs. */ + private def refusedBy(sql: String, fragments: String*): (String, Either[String, Seq[String]]) = + sql -> (Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) if fragments.forall(error.message.contains) => Right(Nil) + case Right(ElasticFailure(error)) => Right(Seq(s"refused as [${error.message}]")) + case Right(other) => Right(Seq(s"answered $other instead of the refusal")) + case Left(t) => Left(t.toString) + }) + + /** No table named `name` exists: DESCRIBE fails on it. */ + private def absent(name: String): (String, Either[String, Seq[String]]) = + s"DESCRIBE TABLE $name" -> (Try( + Await.result(client.run(s"DESCRIBE TABLE $name"), 60.seconds) + ).toEither match { + case Right(ElasticFailure(_)) => Right(Nil) + case other => Right(Seq(s"the table exists: $other")) + }) + + /** The type DESCRIBE reports for `field` of `table`, as a verdict. */ + private def declares(table: String, field: String, sqlType: String) = + s"DESCRIBE TABLE $table ($field)" -> described(table, field).map { t => + if (t.contains(sqlType)) Nil else Seq(s"$field is declared $t instead of $sqlType") + } + + /** `CREATE TABLE AS