diff --git a/THIRD-PARTY-NOTICES b/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..dadeb4c1 --- /dev/null +++ b/THIRD-PARTY-NOTICES @@ -0,0 +1,63 @@ +THIRD-PARTY NOTICES + +This project includes portions of the third-party software listed below, each distributed under +its own license, reproduced here. + +================================================================================ +{fmt} 6.1.2 +https://github.com/fmtlib/fmt/tree/6.1.2 + +Used in: core/src/main/scala/app/softnetwork/elastic/client/FloatingPointText.scala +A Scala port of {fmt}'s floating-point formatting (include/fmt/format-inl.h: `fp`, `multiply`, +`get_cached_power` and its tables, `bigint`, `grisu_gen_digits`, `grisu_shortest_handler`, +`fallback_format`, the shortest branch of `format_float`; include/fmt/format.h: the `float_writer` +layout of the `{}` specifier and `write_exponent`). + +License (https://raw.githubusercontent.com/fmtlib/fmt/6.1.2/LICENSE.rst): + +Copyright (c) 2012 - present, Victor Zverovich + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- Optional exception to the license --- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into a machine-executable object form of such +source code, you may redistribute such embedded portions in such object form +without including the above copyright and permission notices. + +================================================================================ +DuckDB 1.5.5 +https://github.com/duckdb/duckdb/tree/v1.5.5 + +Used in: core/src/main/scala/app/softnetwork/elastic/client/TemporalText.scala +The text layout of a DATE, a TIME and a TIMESTAMP cast to VARCHAR, translated from +`DateToStringCast` and `TimeToStringCast` (src/include/duckdb/common/types/cast_helpers.hpp) and +`StringFromTimestamp` / `StringAsTime` (src/common/operator/string_cast.cpp). + +License (https://raw.githubusercontent.com/duckdb/duckdb/v1.5.5/LICENSE): + +Copyright 2018-2025 Stichting DuckDB Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala index a56d5cb5..fd491351 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala @@ -18,8 +18,16 @@ package app.softnetwork.elastic.sql.bridge import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import app.softnetwork.elastic.sql.query.SingleSearch import com.sksamuel.elastic4s.requests.searches.SearchRequest -import com.sksamuel.elastic4s.requests.searches.aggs.{AbstractAggregation, Aggregation} +import com.sksamuel.elastic4s.requests.searches.aggs.{ + AbstractAggregation, + Aggregation, + CardinalityAggregation, + KeyedFiltersAggregation, + SumAggregation, + ValueCountAggregation +} import com.sksamuel.elastic4s.requests.searches.aggs.pipeline.BucketScriptPipelineAgg import scala.jdk.CollectionConverters._ @@ -35,8 +43,11 @@ import scala.jdk.CollectionConverters._ * The script is sent as the object Elasticsearch documents -- `source`, `lang`, `params` -- for * exactly the `bucket_script`s that carry params, so the source stays the SAME text from one * request to the next and Elasticsearch compiles it once: inlining the clock instead would compile - * a new script on every request, against the cluster's compilation rate limit. Every other body is - * returned untouched, unparsed. + * a new script on every request, against the cluster's compilation rate limit. + * + * The same builder never writes a `gap_policy` either (and elastic4s 7.17 to 9.0 knows only `skip` + * and `insert_zeros`), so the `keep_values` of a whole-table calculation is written here too + * ([[keepingValues]]). Every other body is returned untouched, unparsed. */ object BucketScriptParams { @@ -53,13 +64,63 @@ object BucketScriptParams { case _ => Nil }.toMap - /** `body`, the JSON of `search`, with the params of its `bucket_script`s bound. */ + /** The `bucket_script`s of the whole-table bucket (issue #413) whose every operand is a count or + * a sum -- `_count`, `value_count`, `cardinality`, `sum` -- by name. + * + * Over ZERO documents Elasticsearch takes every `buckets_path` but `_count` for a gap, and its + * default gap policy skips the calculation: `COUNT(x) * 2` answered NULL where SQL answers 0. + * `keep_values` hands the script the value instead -- 0, or 0.0 for a sum (core's `SUM` over no + * value, never NULL) -- and these four metrics are never `NaN` or infinite, so the script never + * reads one. A calculation over a `MIN`, a `MAX`, an `AVG`... keeps the default: there + * `keep_values` would hand the script `NaN`, which a cast turns into a value -- + * `DATEDIFF(MAX(d), MIN(d))` reads `(long) NaN`, 0, and answered 0 days over no document instead + * of NULL (MEASURED on 8.18.3) -- while skipping answers NULL, as SQL does. + * + * 🔴 Elasticsearch 7.17 and later only: 6.8 refuses the policy (`Invalid gap policy: + * [keep_values], accepted values: [insert_zeros, skip]`), and its bridge does not send it. + */ + private def keepingValues(aggs: Seq[AbstractAggregation]): Set[String] = + aggs + .collectFirst { + case wholeTable: KeyedFiltersAggregation + if wholeTable.name == SingleSearch.WholeTableHavingAgg => + wholeTable.subaggs + } + .map { subaggs => + val counts: Set[String] = subaggs.collect { + case count: ValueCountAggregation => count.name + case distinct: CardinalityAggregation => distinct.name + case sum: SumAggregation => sum.name + }.toSet + subaggs.collect { + case script: BucketScriptPipelineAgg + if script.bucketsPaths.values.forall(path => path == "_count" || counts(path)) => + script.name + }.toSet + } + .getOrElse(Set.empty) + + /** `body`, the JSON of `search`, with the params of its `bucket_script`s bound and the gap policy + * of its whole-table calculations written. + */ def bind(body: String, search: SearchRequest): String = { val params = withParams(search.aggs) - if (params.isEmpty) body + val keepValues = keepingValues(search.aggs) + if (params.isEmpty && keepValues.isEmpty) body else { val root = mapper.readTree(body) - bind(root, params) + if (params.nonEmpty) bind(root, params) + if (keepValues.nonEmpty) + root.path("aggs").path(SingleSearch.WholeTableHavingAgg).path("aggs") match { + case wholeTable: ObjectNode => + keepValues.foreach { name => + wholeTable.path(name).path("bucket_script") match { + case bucketScript: ObjectNode => bucketScript.put("gap_policy", "keep_values") + case _ => + } + } + case _ => + } mapper.writeValueAsString(root) } } diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 0dc21aad..c8e51957 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -65,7 +65,8 @@ import com.sksamuel.elastic4s.requests.searches.aggs.{ NestedAggregation, StatsAggregation, TermsAggregation, - TermsOrder + TermsOrder, + ValueCountAggregation } import com.sksamuel.elastic4s.requests.searches.aggs.pipeline.BucketScriptPipelineAgg import com.sksamuel.elastic4s.requests.searches.sort.FieldSort @@ -730,11 +731,15 @@ object ElasticAggregation { val script = metricSelectorForBucket(criteria, None, references) if (script.isEmpty) None else { + val documentCounts = documentCountNames(aggs) val bucketSelector = bucketSelectorAggregation( "having_filter", now(Script(script)), - extractMetricsPathForBucket(criteria, None, references) + readingDocumentCount( + extractMetricsPathForBucket(criteria, None, references), + documentCounts + ) ) Some( KeyedFiltersAggregation( @@ -743,13 +748,91 @@ object ElasticAggregation { app.softnetwork.elastic.sql.query.SingleSearch.WholeTableHavingBucket -> matchAllQuery() ), - subaggs = aggs.map(_.agg) :+ bucketSelector + subaggs = aggs.map(agg => readingDocumentCount(agg.agg, documentCounts)) :+ + bucketSelector ) ) } case _ => None } + /** The names of the aggregations of the whole-table bucket that count its DOCUMENTS: `COUNT(*)` + * (and `COUNT(_id)`, `COUNT(DISTINCT *)`), the `value_count` on `_index` that + * [[countAggregation]] emits -- a field every document carries exactly once. + */ + private def documentCountNames(aggs: Seq[ElasticAggregation]): Set[String] = + aggs + .map(_.agg) + .collect { + case count: ValueCountAggregation + if count.field.contains("_index") && count.script.isEmpty => + count.name + } + .toSet + + /** A pipeline of the whole-table bucket reads `COUNT(*)` as the bucket's own document count, + * `_count` -- equal to it by construction -- because Elasticsearch never takes `_count` for a + * gap. Over ZERO documents every other `buckets_path` is one, so `COUNT(*) * 2` was skipped + * (NULL) and `HAVING COUNT(*) = 0` dropped the bucket (no row); SQL answers `0` and one row. + * MEASURED on 6.8.23, 7.17.29 and 8.18.3: `_count` is `0` there on every major. + */ + private def readingDocumentCount( + paths: Map[String, String], + documentCounts: Set[String] + ): Map[String, String] = + if (documentCounts.isEmpty) paths + else + paths.map { case (param, path) => + param -> (if (documentCounts.contains(path)) "_count" else path) + } + + private def readingDocumentCount( + agg: AbstractAggregation, + documentCounts: Set[String] + ): AbstractAggregation = + agg match { + case script: BucketScriptPipelineAgg + if script.bucketsPaths.values.exists(documentCounts.contains) => + script.copy(bucketsPaths = readingDocumentCount(script.bucketsPaths, documentCounts)) + case other => other + } + + /** The synthetic whole-table bucket for a calculation over aggregates with NO `GROUP BY` (issue + * #413, [[app.softnetwork.elastic.sql.query.SingleSearch.wholeTableCalculation]]). + * + * The calculation is a `bucket_script`, and Elasticsearch accepts one only inside a multi-bucket + * aggregation: at the top level of `aggs` the search was refused on every major (6.8: `Only + * sibling pipeline aggregations are allowed at the top level`; 7.17 and 8.18: `must be declared + * inside of another aggregation`). The root aggregations therefore move inside the keyed + * `filters` aggregation [[wholeTableHavingAggregation]] builds -- the same name, the same one + * `match_all` bucket, which `parseAggregations` already reads as transparent -- and the + * calculation is evaluated in that bucket exactly as it is in a `GROUP BY` bucket. + * + * Not under a `HAVING`: that statement is [[wholeTableHavingAggregation]] 's, and when that one + * declines (no condition of the predicate resolves at this level) wrapping here would hang the + * calculation WITHOUT the predicate -- a filter silently dropped. It keeps the emission it had. + */ + def wholeTableCalculationAggregation( + request: app.softnetwork.elastic.sql.query.SingleSearch, + aggs: Seq[ElasticAggregation] + ): Option[Aggregation] = + if ( + request.wholeTableCalculation && request.having.flatMap(_.criteria).isEmpty && + aggs.exists(_.aggType.isBucketScript) + ) { + val documentCounts = documentCountNames(aggs) + Some( + KeyedFiltersAggregation( + app.softnetwork.elastic.sql.query.SingleSearch.WholeTableHavingAgg, + Seq( + app.softnetwork.elastic.sql.query.SingleSearch.WholeTableHavingBucket -> + matchAllQuery() + ), + subaggs = aggs.map(agg => readingDocumentCount(agg.agg, documentCounts)) + ) + ) + } else None + /** Generates the bucket_selector script for a given bucket */ def metricSelectorForBucket( diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index d3a5a285..3c371668 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -233,10 +233,13 @@ package object bridge { // No buckets. A `HAVING` with no `GROUP BY` filters the ONE implicit whole-table group, // so the metric aggregations move inside a synthetic single-bucket `filters` aggregation // that the `having_filter` bucket_selector can hang from; without it the predicate was - // silently discarded (see ElasticAggregation.wholeTableHavingAggregation). + // silently discarded (see ElasticAggregation.wholeTableHavingAggregation). A calculation + // over the aggregates needs the same parent, its `bucket_script` being refused at the top + // level (issue #413, ElasticAggregation.wholeTableCalculationAggregation). case Nil => ElasticAggregation .wholeTableHavingAggregation(request, aggs, notNestedReferences) + .orElse(ElasticAggregation.wholeTableCalculationAggregation(request, aggs)) .map(Seq(_)) .getOrElse(aggs.map(_.agg)) case aggs => diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/HavingFunctionEmissionSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/HavingFunctionEmissionSpec.scala index 718bd174..fb6fda4e 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/HavingFunctionEmissionSpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/HavingFunctionEmissionSpec.scala @@ -438,11 +438,13 @@ class HavingFunctionEmissionSpec extends AnyFlatSpec with Matchers { // --------------------------------------------------------------------------------------------- "a whole-table HAVING over a function of an aggregate" should "hang the selector off the synthetic bucket" in { + // `COUNT(*)` is read as the bucket's own `_count` there (issue #413): over no document every + // other path is a gap, and `HAVING COUNT(*) = 0` dropped its one row queryOf("SELECT COUNT(*) AS c FROM t HAVING GREATEST(COUNT(*), 0) > 1") shouldBe Seq( """{"query":{"match_all":{}},"size":0,"_source":false,""", """"aggs":{"__whole_table_having__":{"filters":{"filters":{"_all":{"match_all":{}}}},""", """"aggs":{"c":{"value_count":{"field":"_index"}},""", - """"having_filter":{"bucket_selector":{"buckets_path":{"c":"c"},""", + """"having_filter":{"bucket_selector":{"buckets_path":{"c":"_count"},""", """"script":{"source":"(params.c == null ? 0 : Math.max(params.c, 0)) > 1"}}}}}}}""" ).mkString } diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/WholeTableCalculationSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/WholeTableCalculationSpec.scala new file mode 100644 index 00000000..ddcf542f --- /dev/null +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/WholeTableCalculationSpec.scala @@ -0,0 +1,157 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime +import scala.jdk.CollectionConverters._ + +/** Issue #413 -- a calculation over aggregates with NO `GROUP BY` is evaluated in ONE bucket. + * + * Its `bucket_script` used to sit at the top level of `aggs`, which Elasticsearch refuses on every + * major (6.8: `Only sibling pipeline aggregations are allowed at the top level`; 7.17 and 8.18: + * `must be declared inside of another aggregation`). It now hangs, with the metrics it reads, from + * the single-bucket `filters` aggregation a `HAVING` without `GROUP BY` already uses, and the + * statement is an aggregation (`size` 0), not a row query. Each shape below was EXECUTED on + * Elasticsearch 6.8.23, 7.17.29 and 8.18.3 and answered one row. + */ +class WholeTableCalculationSpec extends AnyFlatSpec with Matchers { + + import scala.language.implicitConversions + + implicit def timestamp: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) + case other => throw new IllegalArgumentException(s"Not a single search: $other") + } + + private val mapper = new ObjectMapper() + + private def bodyOf(sql: String): JsonNode = { + val select: ElasticSearchRequest = SelectStatement(sql) + mapper.readTree(select.query) + } + + private def names(node: JsonNode): Seq[String] = + Option(node).map(_.fieldNames().asScala.toSeq.sorted).getOrElse(Nil) + + private val wrapper = SingleSearch.WholeTableHavingAgg + + "a calculation over aggregates without GROUP BY" should "hang from the whole-table bucket" in { + Seq( + "SELECT MAX(n) - MIN(n) AS v FROM t" -> Seq("max_n", "min_n", "v"), + "SELECT COUNT(*) * 2 AS v FROM t" -> Seq("count_all", "v"), + "SELECT MAX(d) + 1 AS v FROM t" -> Seq("max_d", "v"), + "SELECT MAX(n) AS m, MAX(n) - MIN(n) AS v FROM t" -> Seq("m", "min_n", "v") + ).foreach { case (sql, inside) => + withClue(s"[$sql] ") { + val body = bodyOf(sql) + // an aggregation, not a row query: no document is fetched + body.get("size").asInt() shouldBe 0 + names(body.get("aggs")) shouldBe Seq(wrapper) + val bucket = body.get("aggs").get(wrapper) + bucket + .get("filters") + .get("filters") + .get(SingleSearch.WholeTableHavingBucket) + .has( + "match_all" + ) shouldBe true + names(bucket.get("aggs")) shouldBe inside + bucket.get("aggs").get("v").has("bucket_script") shouldBe true + } + } + } + + it should "keep the HAVING's selector beside it, in the same bucket" in { + val bucket = + bodyOf("SELECT COUNT(*) * 2 AS v FROM t HAVING COUNT(*) > 1").get("aggs").get(wrapper) + names(bucket.get("aggs")) shouldBe Seq("count_all", "having_filter", "v") + } + + it should "leave a statement it does not cover as it was" in { + // a column beside the calculation has no value for the whole table: unchanged, still refused + // by Elasticsearch -- not answered with a NULL calculation per document + val mixed = bodyOf("SELECT id, MAX(n) - MIN(n) AS v FROM t") + names(mixed.get("aggs")) shouldBe Seq("max_n", "min_n", "v") + // per group: the GROUP BY bucket is the parent, as before + names(bodyOf("SELECT g, MAX(n) - MIN(n) AS v FROM t GROUP BY g").get("aggs")) shouldBe Seq("g") + // a plain aggregate needs no parent + names(bodyOf("SELECT MAX(n) AS m FROM t").get("aggs")) shouldBe Seq("m") + } + + private def calculation(sql: String): JsonNode = + bodyOf(sql).get("aggs").get(wrapper).get("aggs").get("v").get("bucket_script") + + private def gapPolicyOf(sql: String): Option[String] = + Option(calculation(sql).get("gap_policy")).map(_.asText()) + + "over zero documents, the whole-table bucket" should "read COUNT(*) as its own _count" in { + // Elasticsearch never takes `_count` for a gap: `COUNT(*) * 2` is 0 and `HAVING COUNT(*) = 0` + // keeps the bucket over no document, on every major (MEASURED on 6.8.23, 7.17.29, 8.18.3) + calculation("SELECT COUNT(*) * 2 AS v FROM t") + .get("buckets_path") + .get("count_all") + .asText() shouldBe + "_count" + val selector = bodyOf("SELECT COUNT(*) AS c FROM t HAVING COUNT(*) = 0") + .get("aggs") + .get(wrapper) + .get("aggs") + .get("having_filter") + .get("bucket_selector") + selector.get("buckets_path").elements().asScala.map(_.asText()).toSeq shouldBe Seq("_count") + // the COUNT(*) column itself is still the count it was + names( + bodyOf("SELECT COUNT(*) AS c FROM t HAVING COUNT(*) = 0").get("aggs").get(wrapper).get("aggs") + ) shouldBe Seq("c", "having_filter") + // a count of a column is not the bucket's document count + calculation("SELECT COUNT(n) * 2 AS v FROM t") + .get("buckets_path") + .get("count_n") + .asText() shouldBe + "count_n" + } + + it should "keep the value of a count or a sum, never hand a MIN / MAX's NaN to the script" in { + // Elasticsearch 7.17 and later: COUNT(x) is 0 and SUM(x) 0.0 over no document + gapPolicyOf("SELECT COUNT(n) * 2 AS v FROM t") shouldBe Some("keep_values") + gapPolicyOf("SELECT COUNT(*) + COUNT(n) AS v FROM t") shouldBe Some("keep_values") + gapPolicyOf("SELECT SUM(x) * 2 AS v FROM t") shouldBe Some("keep_values") + gapPolicyOf("SELECT COUNT(DISTINCT g) * 10 AS v FROM t") shouldBe Some("keep_values") + gapPolicyOf("SELECT COUNT(*) * 2 AS v FROM t") shouldBe Some("keep_values") + // a calculation reading a MIN, a MAX or an AVG keeps the default (NULL over no document) + gapPolicyOf("SELECT MAX(n) - MIN(n) AS v FROM t") shouldBe None + gapPolicyOf("SELECT DATEDIFF(MAX(d), MIN(d)) AS v FROM t") shouldBe None + gapPolicyOf("SELECT COUNT(*) + MAX(n) AS v FROM t") shouldBe None + gapPolicyOf("SELECT AVG(n) * 2 AS v FROM t") shouldBe None + // per group, and in a HAVING's selector: as before + bodyOf("SELECT g, COUNT(n) * 2 AS v FROM t GROUP BY g").toString.contains("gap_policy") shouldBe + false + bodyOf("SELECT COUNT(n) AS c FROM t HAVING COUNT(n) = 0").toString.contains( + "gap_policy" + ) shouldBe + false + } +} diff --git a/build.sbt b/build.sbt index 3e18e814..e2106e0b 100644 --- a/build.sbt +++ b/build.sbt @@ -232,7 +232,15 @@ lazy val core = project scalacOptions ++= Seq( "-language:experimental.macros", "-Ymacro-debug-lite" - ) + ), + // core holds code ported from third-party software (FloatingPointText from {fmt}, + // TemporalText from DuckDB): their notices ship in the jar, as META-INF/THIRD-PARTY-NOTICES, + // and through the class directory into every assembly that bundles core + Compile / resourceGenerators += Def.task { + val notices = (Compile / resourceManaged).value / "META-INF" / "THIRD-PARTY-NOTICES" + IO.copyFile((ThisBuild / baseDirectory).value / "THIRD-PARTY-NOTICES", notices) + Seq(notices) + }.taskValue ) .dependsOn( macros % "compile->compile;test->test;it->it" @@ -431,7 +439,9 @@ def cliProject(esVersion: String, ss: Def.SettingsDefinition*): Project = { case PathList("META-INF", "services", "java.net.spi.InetAddressResolverProvider") => MergeStrategy.discard case PathList("META-INF", "services", _ @_*) => MergeStrategy.concat - case PathList("META-INF", _ @_*) => MergeStrategy.discard + // the notices of the third-party code core ports ({fmt}, DuckDB) stay in the bundle + case PathList("META-INF", "THIRD-PARTY-NOTICES") => MergeStrategy.concat + case PathList("META-INF", _ @_*) => MergeStrategy.discard case "reference.conf" => MergeStrategy.concat case "README.md" => MergeStrategy.first case "LICENSE" => MergeStrategy.first diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index 3ef86598..8cd2351f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -229,7 +229,9 @@ trait ElasticConversion { explodeNested, retainDocumentId, rowInvariants.lift(leg).getOrElse(ListMap.empty), - outputFields = leg0.map(_ => fields).getOrElse(Seq.empty) + outputFields = leg0.map(_ => fields).getOrElse(Seq.empty), + // ...and in the type the result gives each column (`SetOperationValues`) + converters = leg0.flatMap(_.converters).orNull ) } else { Seq.empty @@ -332,7 +334,8 @@ trait ElasticConversion { explodeNested: Boolean = true, retainDocumentId: Boolean = false, rowInvariants: ListMap[String, Any] = ListMap.empty, - outputFields: Seq[String] = Seq.empty + outputFields: Seq[String] = Seq.empty, + converters: Array[Any => Any] = null )(implicit context: ConversionContext): Seq[ListMap[String, Any]] = { val hitsNode = Option(json.path("hits").path("hits")) .filter(_.isArray) @@ -390,8 +393,13 @@ trait ElasticConversion { if (outputFields.size == fields.size) fields.zip(outputFields) else fields.map(f => (f, f)) val effective = pairs.filterNot(_._1 == "*") + // `converters` (a `UNION ALL` leg's, aligned with `fields`) follow the same "*" filter + val convs: Array[Any => Any] = + if ((converters ne null) && converters.length == fields.size) + fields.indices.filterNot(i => fields(i) == "*").map(i => converters(i)).toArray + else null if (effective.isEmpty) rows - else rows.map(rowProjector(effective.map(_._1), effective.map(_._2))) + else rows.map(rowProjector(effective.map(_._1), effective.map(_._2), convs)) } def findKeyValue(path: String, map: Map[String, Any]): Option[Any] = { @@ -523,11 +531,31 @@ trait ElasticConversion { protected def rowProjector( sourceFields: Seq[String], targetFields: Seq[String] + )(implicit context: ConversionContext): ListMap[String, Any] => ListMap[String, Any] = + rowProjector(sourceFields, targetFields, null) + + /** [[rowProjector]] that also CONVERTS the value of some columns as it places them -- a `UNION + * ALL` leg whose column has another type than the result's (`SetOperationValues`). + * + * `converters` is aligned with `sourceFields`: the function for the column at that position, or + * `null` for none; `null` itself when nothing converts, which is the projection above, + * unchanged. A converting projection always rebuilds the row -- a value changes -- and does it + * in the one rebuild the projection performs, never as a second pass over the rows. + */ + protected def rowProjector( + sourceFields: Seq[String], + targetFields: Seq[String], + converters: Array[Any => Any] )(implicit context: ConversionContext): ListMap[String, Any] => ListMap[String, Any] = { if (sourceFields.isEmpty) identity else { val fieldArr: Array[String] = sourceFields.toArray val len = fieldArr.length + // decided once per stream: `null` unless at least one column of THIS projection converts + val convs: Array[Any => Any] = + if ((converters ne null) && converters.length == len && converters.exists(_ ne null)) + converters + else null // A target list of a different length cannot be matched positionally against this one, so // the projection degrades to a plain normalization rather than guessing an alignment. val outArr: Array[String] = @@ -577,6 +605,8 @@ trait ElasticConversion { case EntityContext => false case _ => true } + // a converted value changes the row, so a converting projection never hands a row back as is + val rebuilds = renames || (convs ne null) if (fieldIndex.size() != len) { // Duplicate SOURCE names cannot hold distinct positions in a row map. // * without a rename this is the historical degenerate shape — keep the exact legacy @@ -588,7 +618,22 @@ trait ElasticConversion { // hand-written duplicate projection reaches. if (!renames) { val requestedSet = sourceFields.toSet - row => normalizeRowOrdered(row, sourceFields, requestedSet) + if (convs eq null) row => normalizeRowOrdered(row, sourceFields, requestedSet) + else { + // the column a repeated name keeps is its FIRST position, and so is its conversion + val convertByName = new java.util.HashMap[String, Any => Any](len * 2) + var j = 0 + while (j < len) { + if (!convertByName.containsKey(fieldArr(j)) && (convs(j) ne null)) + convertByName.put(fieldArr(j), convs(j)) + j += 1 + } + row => + normalizeRowOrdered(row, sourceFields, requestedSet).map { entry => + val convert = convertByName.get(entry._1) + if (convert eq null) entry else entry._1 -> convert(entry._2) + } + } } else { val sourceSet = sourceFields.toSet row => { @@ -597,8 +642,10 @@ trait ElasticConversion { while (j < len) { if (outFirst(j)) { row.get(fieldArr(j)) match { - case Some(v) => builder += outArr(j) -> v - case None => if (nullFillMissing) builder += outArr(j) -> null + case Some(v) => + builder += outArr(j) -> (if ((convs eq null) || (convs(j) eq null)) v + else convs(j)(v)) + case None => if (nullFillMissing) builder += outArr(j) -> null } } j += 1 @@ -648,30 +695,42 @@ trait ElasticConversion { } } } - // Under a rename the loop may have stopped on `passthrough` with entries still to come; - // none of them can be a source name (a row map's keys are unique and all `len` of them - // were just consumed), so they are extras — subject to the same `targetNames` rule. - if (renames) { + // Under a rename -- or a conversion -- the loop may have stopped on `passthrough` with + // entries still to come; none of them can be a source name (a row map's keys are unique + // and all `len` of them were just consumed), so they are extras -- subject, under a + // rename, to the same `targetNames` rule. + if (rebuilds) { while (it.hasNext) { val entry = it.next() - if (!targetNames.contains(entry._1)) { + if (!renames || !targetNames.contains(entry._1)) { if (extras eq null) extras = new ListBuffer[(String, Any)] extras += entry } } } // An in-order strict prefix needs no rebuild either when missing fields are skipped - if (!renames && (passthrough || (inOrder && !nullFillMissing))) row + if (!rebuilds && (passthrough || (inOrder && !nullFillMissing))) row else { val builder = ListMap.newBuilder[String, Any] var j = 0 - while (j < len) { - if (outFirst(j)) { - if (seen(j)) builder += outArr(j) -> values(j) - else if (nullFillMissing) builder += outArr(j) -> null + if (convs eq null) + while (j < len) { + if (outFirst(j)) { + if (seen(j)) builder += outArr(j) -> values(j) + else if (nullFillMissing) builder += outArr(j) -> null + } + j += 1 + } + else + while (j < len) { + if (outFirst(j)) { + if (seen(j)) { + val convert = convs(j) + builder += outArr(j) -> (if (convert eq null) values(j) else convert(values(j))) + } else if (nullFillMissing) builder += outArr(j) -> null + } + j += 1 } - j += 1 - } if (extras ne null) extras.foreach(builder += _) builder.result() } @@ -1126,8 +1185,15 @@ trait ElasticConversion { // genuine `MAX` of `0.0` needs at least one document, where this rule cannot fire. On the // majors that were already correct the value is `null` anyway, so this is a no-op for // them beyond making the NULL explicit rather than an absent key. + // + // A calculation (`bucket_script`) is not forced: over no document Elasticsearch skips one + // that reads any other value than a count or a sum -- it is absent, NULL -- and evaluates + // one over counts and sums only (`_count`, and `keep_values` from 7.17), whose answer is + // the calculation's (`COUNT(*) * 2` is 0), issue #413. val emptyInput = - docCount.contains(0L) && aggregations.get(name).exists(_.nullOverEmptyInput) + docCount.contains(0L) && aggregations + .get(name) + .exists(agg => agg.nullOverEmptyInput && agg.aggType != AggregationType.BucketScript) Option(value.get("value")) .filter(_ => !emptyInput) .filter(!_.isNull) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/FloatingPointText.scala b/core/src/main/scala/app/softnetwork/elastic/client/FloatingPointText.scala new file mode 100644 index 00000000..c36ac417 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/FloatingPointText.scala @@ -0,0 +1,785 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file contains a Scala port of portions of the {fmt} library, version 6.1.2 + * (https://github.com/fmtlib/fmt/tree/6.1.2): + * + * - include/fmt/format-inl.h: the `fp` class (assign, assign_with_boundaries, + * assign_float_with_boundaries, normalize), `multiply`, `get_cached_power` and its tables of + * cached powers of 10, `bigint`, `grisu_gen_digits`, `grisu_shortest_handler`, + * `fallback_format` and the shortest-representation branch of `format_float`; + * - include/fmt/format.h: the `float_writer` layout of the `{}` specifier and + * `write_exponent`. + * + * The ported parts were read from the copy of {fmt} that DuckDB 1.5.5 embeds (third_party/fmt), + * where they are identical to upstream {fmt} 6.1.2 (DuckDB's own additions to that copy, the + * thousand separators, are not ported). {fmt} is distributed under the following license: + * + * Copyright (c) 2012 - present, Victor Zverovich + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * --- Optional exception to the license --- + * + * As an exception, if, as a result of your compiling your source code, portions + * of this Software are embedded into a machine-executable object form of such + * source code, you may redistribute such embedded portions in such object form + * without including the above copyright and permission notices. + */ + +package app.softnetwork.elastic.client + +/** A binary floating-point value as TEXT, spelled as DuckDB 1.5.5 spells it when it casts a + * `DOUBLE` or a `REAL` to `VARCHAR` (`250000000000.0`, `1e-05`, `1e+16`, `-0.0`, `nan`, `inf`). + * + * DuckDB writes such a value with the `{}` format of the fmt library it embeds (fmt 6.1.2): the + * shortest digits the Grisu algorithm finds, fmt's own bignum fallback where Grisu cannot decide, + * then fmt's layout -- plain digits for a decimal exponent in [-4, 16), `d.ddde±XX` otherwise, and + * always a fractional part. This object is a PORT of exactly that code, not an approximation of + * it: the fallback does not always find the shortest digits (DuckDB spells the double `7.168e25` + * `7.1680000000000004e+25`, and the float `4073260.75` with all of its digits, where the shortest + * float would be `4073260.8`), and those spellings are DuckDB's answer too. Java's own + * `Double.toString` differs from it on layout (`2.5E11`) and, before JDK 19, on digits. MEASURED + * against DuckDB 1.5.5 on 3.2 million values (2.08 million doubles, 1.12 million floats): every + * spelling identical. + * + * A call allocates about 250 to 290 bytes -- its digit buffer, the Grisu handler, the builder and + * the result -- and about 1.1 KB for a value the bignum fallback spells, a small fraction of + * values (Grisu decides the others). MEASURED over the same values. + */ +private[client] object FloatingPointText { + + /** `CAST( AS VARCHAR)`. */ + def double(value: Double): String = spell(value, binary32 = false) + + /** `CAST( AS VARCHAR)`: the float's own boundaries for Grisu, as fmt formats a `float`. */ + def real(value: Float): String = spell(value.toDouble, binary32 = true) + + private def spell(value: Double, binary32: Boolean): String = { + val negative = java.lang.Double.doubleToRawLongBits(value) < 0 + val magnitude = if (negative) -value else value + if (java.lang.Double.isNaN(magnitude)) { if (negative) "-nan" else "nan" } + else if (java.lang.Double.isInfinite(magnitude)) { if (negative) "-inf" else "inf" } + else { + val digits = new Array[Char](DigitCapacity) + val exponent = new Array[Int](1) + val count = shortestDigits(magnitude, binary32, digits, exponent) + val text = new java.lang.StringBuilder(DigitCapacity) + if (negative) text.append('-') + layout(text, digits, count, exponent(0)) + text.toString + } + } + + private final val DigitCapacity = 40 + + // --------------------------------------------------------------------------------------------- + // fmt's layout of `{}` (float_writer: general format, precision -1, trailing zeros on) + // --------------------------------------------------------------------------------------------- + + /** `digits` (`count` of them) times 10 ^ `exp`, laid out as fmt's `{}` lays it out. */ + private def layout( + text: java.lang.StringBuilder, + digits: Array[Char], + count: Int, + exp: Int + ): Unit = { + val point = count + exp // the decimal point sits after this many digits + val scientific = point - 1 + if (scientific < -4 || scientific >= 16) { + text.append(digits(0)) + if (count > 1) text.append('.').append(digits, 1, count - 1) + text.append('e') + var e = scientific + if (e < 0) { text.append('-'); e = -e } + else text.append('+') + if (e >= 100) { + val hundreds = e / 100 + if (e >= 1000) text.append(('0' + hundreds / 10).toChar) + text.append(('0' + hundreds % 10).toChar) + e %= 100 + } + text.append(('0' + e / 10).toChar).append(('0' + e % 10).toChar) + } else if (count <= point) { + text.append(digits, 0, count) + var zeros = point - count + while (zeros > 0) { text.append('0'); zeros -= 1 } + text.append(".0") + } else if (point > 0) { + text.append(digits, 0, point).append('.').append(digits, point, count - point) + } else { + text.append('0') + if (point != 0 || count != 0) { + text.append('.') + var zeros = -point + while (zeros > 0) { text.append('0'); zeros -= 1 } + text.append(digits, 0, count) + } + } + } + + // --------------------------------------------------------------------------------------------- + // fmt's format_float (shortest): Grisu, then fallback_format + // --------------------------------------------------------------------------------------------- + + /** Unsigned 64-bit comparisons: fmt's significands are `uint64_t`. */ + @inline private def ult(a: Long, b: Long): Boolean = (a ^ Long.MinValue) < (b ^ Long.MinValue) + @inline private def ule(a: Long, b: Long): Boolean = (a ^ Long.MinValue) <= (b ^ Long.MinValue) + + private final val ImplicitBit = 1L << 52 + private final val SignificandMask = ImplicitBit - 1 + private final val ExponentMask = 0x7ff0000000000000L + + /** A handmade floating-point number `f * 2 ^ e` (fmt's `fp`). */ + private final class Fp(var f: Long, var e: Int) + + /** `fp::assign`: the significand and exponent of a POSITIVE double; true when its predecessor is + * closer than its successor (a normal power of two other than the smallest). + */ + private def assign(d: Double, fp: Fp): Boolean = { + val u = java.lang.Double.doubleToRawLongBits(d) + var f = u & SignificandMask + var biased = ((u & ExponentMask) >>> 52).toInt + val predecessorCloser = f == 0 && biased > 1 + if (biased != 0) f += ImplicitBit else biased = 1 + fp.f = f + fp.e = biased - 1023 - 52 + predecessorCloser + } + + /** `normalize`. */ + private def normalize(fp: Fp, shift: Int): Unit = { + val shiftedImplicitBit = ImplicitBit << shift + while ((fp.f & shiftedImplicitBit) == 0) { + fp.f <<= 1 + fp.e -= 1 + } + val offset = 64 - 52 - shift - 1 + fp.f <<= offset + fp.e -= offset + } + + /** `multiply`: `lhs * rhs / 2 ^ 64`, rounded half up (unsigned). */ + private def multiply(lhs: Long, rhs: Long): Long = { + val mask = 0xffffffffL + val a = lhs >>> 32 + val b = lhs & mask + val c = rhs >>> 32 + val d = rhs & mask + val ac = a * c + val bc = b * c + val ad = a * d + val bd = b * d + val mid = (bd >>> 32) + (ad & mask) + (bc & mask) + (1L << 31) + ac + (ad >>> 32) + (bc >>> 32) + (mid >>> 32) + } + + /** The decimal digits of `value` (> 0 or +0.0) into `digits`, their count returned and the + * decimal exponent of the LAST digit in `exponent(0)`. + */ + private def shortestDigits( + value: Double, + binary32: Boolean, + digits: Array[Char], + exponent: Array[Int] + ): Int = + if (value <= 0) { + digits(0) = '0' + exponent(0) = 0 + 1 + } else { + val fp = new Fp(0L, 0) + var lower = 0L + var upper = 0L + if (binary32) { + // assign_float_with_boundaries + assign(value, fp) + val minNormalE = -125 - 53 + var halfUlp = 1L << (53 - 24 - 1) + if (minNormalE > fp.e) halfUlp <<= (minNormalE - fp.e) + val up = new Fp(fp.f + halfUlp, fp.e) + normalize(up, 0) + val low = fp.f - (halfUlp >>> (if (fp.f == ImplicitBit && fp.e > minNormalE) 1 else 0)) + lower = low << (fp.e - up.e) + upper = up.f + } else { + // assign_with_boundaries + val lowerCloser = assign(value, fp) + val lowF = if (lowerCloser) (fp.f << 2) - 1 else (fp.f << 1) - 1 + val lowE = if (lowerCloser) fp.e - 2 else fp.e - 1 + val up = new Fp((fp.f << 1) + 1, fp.e - 1) + normalize(up, 1) + lower = lowF << (lowE - up.e) + upper = up.f + } + normalize(fp, 0) + // a cached power of 10 bringing the exponent into [-60, -32] + val index0 = (((-60 - (fp.e + 64)) + 64 - 1).toLong * 0x4d104d42L + 0xffffffffL) >> 32 + val index = (index0.toInt - -348 - 1) / 8 + 1 + val cachedExp10 = -348 + index * 8 + val cachedF = Pow10Significands(index) + val valueF = multiply(fp.f, cachedF) + val valueE = fp.e + Pow10Exponents(index) + 64 + lower = multiply(lower, cachedF) - 1 + upper = multiply(upper, cachedF) + 1 + val handler = new ShortestHandler(digits, upper - valueF) + val kappa = new Array[Int](1) + if (grisuGenDigits(upper, valueE, upper - lower, kappa, handler) == Error) { + val exp10 = kappa(0) + handler.size - cachedExp10 - 1 + fallbackFormat(value, digits, exp10, exponent) + } else { + exponent(0) = kappa(0) - cachedExp10 + handler.size + } + } + + private final val More = 0 + private final val Done = 1 + private final val Error = 2 + + /** `grisu_shortest_handler`: Grisu's digit-by-digit `round_weed`. */ + private final class ShortestHandler(val buf: Array[Char], val diff: Long) { + var size = 0 + + private def round(d: Long, divisor: Long, remainder0: Long, error: Long): Long = { + var remainder = remainder0 + while ( + ult(remainder, d) && !ult(error - remainder, divisor) && + (ult(remainder + divisor, d) || !ult(d - remainder, remainder + divisor - d)) + ) { + buf(size - 1) = (buf(size - 1) - 1).toChar + remainder += divisor + } + remainder + } + + def onDigit( + digit: Char, + divisor: Long, + remainder0: Long, + error: Long, + exp: Int, + integral: Boolean + ): Int = { + buf(size) = digit + size += 1 + if (!ult(remainder0, error)) More + else { + val unit = if (integral) 1L else PowersOf10(-exp) + val up = (diff - 1) * unit + val remainder = round(up, divisor, remainder0, error) + val down = (diff + 1) * unit + if ( + ult(remainder, down) && !ult(error - remainder, divisor) && + (ult(remainder + divisor, down) || ult(remainder + divisor - down, down - remainder)) + ) Error + else if (ule(2 * unit, remainder) && ule(remainder, error - 4 * unit)) Done + else Error + } + } + } + + private def countDigits(n: Long): Int = { + var count = 1 + var rest = n + while (rest >= 10) { + rest /= 10 + count += 1 + } + count + } + + /** `grisu_gen_digits` for the scaled upper boundary `valueF * 2 ^ valueE`; the decimal exponent + * (kappa) it ends on in `kappa(0)`. + */ + private def grisuGenDigits( + valueF: Long, + valueE: Int, + error0: Long, + kappa: Array[Int], + handler: ShortestHandler + ): Int = { + val shift = -valueE + val one = 1L << shift + var integral = valueF >>> shift + var fractional = valueF & (one - 1) + var error = error0 + var exp = countDigits(integral) + var result = More + // the integral part: up to 10 digits + while (result == More && exp > 0) { + val divisor = PowersOf10(exp - 1) + val digit = integral / divisor + integral = integral % divisor + exp -= 1 + val remainder = (integral << shift) + fractional + result = handler.onDigit( + ('0' + digit).toChar, + PowersOf10(exp) << shift, + remainder, + error, + exp, + integral = true + ) + } + // the fractional part + while (result == More) { + fractional *= 10 + error *= 10 + val digit = ('0' + (fractional >>> shift)).toChar + fractional &= one - 1 + exp -= 1 + result = handler.onDigit(digit, one, fractional, error, exp, integral = false) + } + kappa(0) = exp + result + } + + /** `fallback_format`: Steele & White's (FPP)² over fmt's bignum, double boundaries (fmt's "TODO: + * handle float") -- the digits into `digits`, their count returned, the exponent of the last one + * in `exponent(0)`. + */ + private def fallbackFormat( + d: Double, + digits: Array[Char], + exp10: Int, + exponent: Array[Int] + ): Int = { + val numerator = new Bignum + val denominator = new Bignum + val lower = new Bignum + val upperStore = new Bignum + var upper: Bignum = null + val value = new Fp(0L, 0) + val shift = if (assign(d, value)) 2 else 1 + val significand = value.f << shift + if (value.e >= 0) { + numerator.assign(significand) + numerator.shiftLeft(value.e) + lower.assign(1L) + lower.shiftLeft(value.e) + if (shift != 1) { + upperStore.assign(1L) + upperStore.shiftLeft(value.e + 1) + upper = upperStore + } + denominator.assignPow10(exp10) + denominator.shiftLeft(1) + } else if (exp10 < 0) { + numerator.assignPow10(-exp10) + lower.assign(numerator) + if (shift != 1) { + upperStore.assign(numerator) + upperStore.shiftLeft(1) + upper = upperStore + } + numerator.multiply64(significand) + denominator.assign(1L) + denominator.shiftLeft(shift - value.e) + } else { + numerator.assign(significand) + denominator.assignPow10(exp10) + denominator.shiftLeft(shift - value.e) + lower.assign(1L) + if (shift != 1) { + upperStore.assign(1L << 1) + upper = upperStore + } + } + if (upper == null) upper = lower + val even = if ((value.f & 1) == 0) 1 else 0 + var count = 0 + var finished = false + while (!finished) { + val digit = numerator.divmodAssign(denominator) + val low = Bignum.compare(numerator, lower) - even < 0 + val high = Bignum.addCompare(numerator, upper, denominator) + even > 0 + digits(count) = ('0' + digit).toChar + count += 1 + if (low || high) { + if (!low) digits(count - 1) = (digits(count - 1) + 1).toChar + else if (high) { + val result = Bignum.addCompare(numerator, numerator, denominator) + if (result > 0 || (result == 0 && digit % 2 != 0)) + digits(count - 1) = (digits(count - 1) + 1).toChar + } + finished = true + } else { + numerator.multiply32(10) + lower.multiply32(10) + if (upper ne lower) upper.multiply32(10) + } + } + exponent(0) = exp10 - (count - 1) + count + } + + /** fmt's `bigint`, bit for bit: 32-bit "bigits", least significant first, scaled by 2 ^ (32 * + * `exp`). Its representation is part of the answer -- `compare` reads it, not only the value -- + * so it is reproduced operation by operation. + */ + private final class Bignum { + var bigits: Array[Int] = new Array[Int](40) + var size: Int = 0 + var exp: Int = 0 + + private def reserve(n: Int): Unit = + if (n > bigits.length) { + val grown = new Array[Int](math.max(n, bigits.length * 2)) + System.arraycopy(bigits, 0, grown, 0, size) + bigits = grown + } + + def resize(n: Int): Unit = { + reserve(n) + size = n + } + + def pushBack(bigit: Int): Unit = { + reserve(size + 1) + bigits(size) = bigit + size += 1 + } + + def numBigits: Int = size + exp + + def assign(n0: Long): Unit = { + var n = n0 + var count = 0 + // fmt's do-while: at least one bigit, also for 0 + var more = true + while (more) { + reserve(count + 1) + bigits(count) = n.toInt + count += 1 + n >>>= 32 + more = n != 0 + } + size = count + exp = 0 + } + + def assign(other: Bignum): Unit = { + resize(other.size) + System.arraycopy(other.bigits, 0, bigits, 0, other.size) + exp = other.exp + } + + def shiftLeft(shift0: Int): Unit = { + exp += shift0 / 32 + val shift = shift0 % 32 + if (shift != 0) { + var carry = 0 + var i = 0 + val n = size + while (i < n) { + val c = bigits(i) >>> (32 - shift) + bigits(i) = (bigits(i) << shift) + carry + carry = c + i += 1 + } + if (carry != 0) pushBack(carry) + } + } + + def multiply32(value: Int): Unit = { + val wide = value & 0xffffffffL + var carry = 0L + var i = 0 + val n = size + while (i < n) { + val result = (bigits(i) & 0xffffffffL) * wide + carry + bigits(i) = result.toInt + carry = result >>> 32 + i += 1 + } + if (carry != 0) pushBack(carry.toInt) + } + + def multiply64(value: Long): Unit = { + val mask = 0xffffffffL + val lowerHalf = value & mask + val upperHalf = value >>> 32 + var carry = 0L + var i = 0 + val n = size + while (i < n) { + val bigit = bigits(i) & mask + val result = bigit * lowerHalf + (carry & mask) + carry = bigit * upperHalf + (result >>> 32) + (carry >>> 32) + bigits(i) = result.toInt + i += 1 + } + while (carry != 0) { + pushBack((carry & mask).toInt) + carry >>>= 32 + } + } + + private def removeLeadingZeros(): Unit = { + var top = size - 1 + while (top > 0 && bigits(top) == 0) top -= 1 + size = top + 1 + } + + /** `subtract_bigits`: the new borrow. */ + private def subtractBigits(index: Int, other: Int, borrow: Int): Int = { + reserve(index + 1) + val result = (bigits(index) & 0xffffffffL) - (other & 0xffffffffL) - (borrow & 0xffffffffL) + bigits(index) = result.toInt + (result >>> 63).toInt + } + + /** `subtract_aligned`, including its last loop, which keeps subtracting at ONE index. */ + private def subtractAligned(other: Bignum): Unit = { + var borrow = 0 + var i = other.exp - exp + var j = 0 + while (j != other.size) { + borrow = subtractBigits(i, other.bigits(j), borrow) + i += 1 + j += 1 + } + while (borrow > 0) borrow = subtractBigits(i, 0, borrow) + removeLeadingZeros() + } + + def assignPow10(exp10: Int): Unit = + if (exp10 == 0) assign(1L) + else { + var bitmask = 1 + while (exp10 >= bitmask) bitmask <<= 1 + bitmask >>= 1 + assign(5L) + bitmask >>= 1 + while (bitmask != 0) { + square() + if ((exp10 & bitmask) != 0) multiply32(5) + bitmask >>= 1 + } + shiftLeft(exp10) + } + + /** `square`, its 128-bit accumulator as two words (fmt's `accumulator`). */ + private def square(): Unit = { + val n = java.util.Arrays.copyOf(bigits, size) + val count = size + val resultCount = 2 * count + resize(resultCount) + var sumLow = 0L + var sumHigh = 0L + def add(term: Long): Unit = { + sumLow += term + if (ult(sumLow, term)) sumHigh += 1 + } + def shiftSum(): Unit = { + sumLow = (sumHigh << 32) | (sumLow >>> 32) + sumHigh >>>= 32 + } + var index = 0 + while (index < count) { + var i = 0 + var j = index + while (j >= 0) { + add((n(i) & 0xffffffffL) * (n(j) & 0xffffffffL)) + i += 1 + j -= 1 + } + bigits(index) = sumLow.toInt + shiftSum() + index += 1 + } + index = count + while (index < resultCount) { + var j = count - 1 + var i = index - j + while (i < count) { + add((n(i) & 0xffffffffL) * (n(j) & 0xffffffffL)) + i += 1 + j -= 1 + } + bigits(index) = sumLow.toInt + shiftSum() + index += 1 + } + removeLeadingZeros() + exp *= 2 + } + + /** `divmod_assign`: this becomes this % divisor; the quotient is returned. */ + def divmodAssign(divisor: Bignum): Int = + if (Bignum.compare(this, divisor) < 0) 0 + else { + val count = size + val expDifference = exp - divisor.exp + if (expDifference > 0) { + resize(count + expDifference) + var i = count - 1 + var j = i + expDifference + while (i >= 0) { + bigits(j) = bigits(i) + i -= 1 + j -= 1 + } + java.util.Arrays.fill(bigits, 0, expDifference, 0) + exp -= expDifference + } + var quotient = 0 + // fmt's do-while: this >= divisor here, so it subtracts at least once + var more = true + while (more) { + subtractAligned(divisor) + quotient += 1 + more = Bignum.compare(this, divisor) >= 0 + } + quotient + } + } + + private object Bignum { + + /** `compare`: reads the representation (a lower bigit run decides, zero or not). */ + def compare(lhs: Bignum, rhs: Bignum): Int = { + val lhsBigits = lhs.numBigits + val rhsBigits = rhs.numBigits + if (lhsBigits != rhsBigits) { if (lhsBigits > rhsBigits) 1 else -1 } + else { + var i = lhs.size - 1 + var j = rhs.size - 1 + var end = i - j + if (end < 0) end = 0 + var answer = 0 + var decided = false + while (!decided && i >= end) { + val a = lhs.bigits(i) & 0xffffffffL + val b = rhs.bigits(j) & 0xffffffffL + if (a != b) { + answer = if (a > b) 1 else -1 + decided = true + } else { + i -= 1 + j -= 1 + } + } + if (decided) answer + else if (i != j) { if (i > j) 1 else -1 } + else 0 + } + } + + /** `add_compare`: the sign of `lhs1 + lhs2 - rhs`. */ + def addCompare(lhs1: Bignum, lhs2: Bignum, rhs: Bignum): Int = { + val maxLhsBigits = math.max(lhs1.numBigits, lhs2.numBigits) + val rhsBigits = rhs.numBigits + if (maxLhsBigits + 1 < rhsBigits) -1 + else if (maxLhsBigits > rhsBigits) 1 + else { + def bigitAt(n: Bignum, i: Int): Long = + if (i >= n.exp && i < n.numBigits) n.bigits(i - n.exp) & 0xffffffffL else 0L + var borrow = 0L + val minExp = math.min(math.min(lhs1.exp, lhs2.exp), rhs.exp) + var i = rhsBigits - 1 + var answer = 0 + var decided = false + while (!decided && i >= minExp) { + val sum = bigitAt(lhs1, i) + bigitAt(lhs2, i) + val rhsBigit = bigitAt(rhs, i) + if (sum > rhsBigit + borrow) { + answer = 1 + decided = true + } else { + borrow = rhsBigit + borrow - sum + if (borrow > 1) { + answer = -1 + decided = true + } else { + borrow <<= 32 + i -= 1 + } + } + } + if (decided) answer else if (borrow != 0) -1 else 0 + } + } + } + + /** `powers_of_10_64`: 10 ^ 0 to 10 ^ 19, unsigned. */ + private val PowersOf10: Array[Long] = { + val powers = new Array[Long](20) + var p = 1L + var i = 0 + while (i < 20) { + powers(i) = p + p *= 10 + i += 1 + } + powers + } + + /** Normalized 64-bit significands of 10 ^ k, k = -348, -340, ..., 340 (fmt's table). */ + private val Pow10Significands: Array[Long] = Array( + 0xfa8fd5a0081c0288L, 0xbaaee17fa23ebf76L, 0x8b16fb203055ac76L, 0xcf42894a5dce35eaL, + 0x9a6bb0aa55653b2dL, 0xe61acf033d1a45dfL, 0xab70fe17c79ac6caL, 0xff77b1fcbebcdc4fL, + 0xbe5691ef416bd60cL, 0x8dd01fad907ffc3cL, 0xd3515c2831559a83L, 0x9d71ac8fada6c9b5L, + 0xea9c227723ee8bcbL, 0xaecc49914078536dL, 0x823c12795db6ce57L, 0xc21094364dfb5637L, + 0x9096ea6f3848984fL, 0xd77485cb25823ac7L, 0xa086cfcd97bf97f4L, 0xef340a98172aace5L, + 0xb23867fb2a35b28eL, 0x84c8d4dfd2c63f3bL, 0xc5dd44271ad3cdbaL, 0x936b9fcebb25c996L, + 0xdbac6c247d62a584L, 0xa3ab66580d5fdaf6L, 0xf3e2f893dec3f126L, 0xb5b5ada8aaff80b8L, + 0x87625f056c7c4a8bL, 0xc9bcff6034c13053L, 0x964e858c91ba2655L, 0xdff9772470297ebdL, + 0xa6dfbd9fb8e5b88fL, 0xf8a95fcf88747d94L, 0xb94470938fa89bcfL, 0x8a08f0f8bf0f156bL, + 0xcdb02555653131b6L, 0x993fe2c6d07b7facL, 0xe45c10c42a2b3b06L, 0xaa242499697392d3L, + 0xfd87b5f28300ca0eL, 0xbce5086492111aebL, 0x8cbccc096f5088ccL, 0xd1b71758e219652cL, + 0x9c40000000000000L, 0xe8d4a51000000000L, 0xad78ebc5ac620000L, 0x813f3978f8940984L, + 0xc097ce7bc90715b3L, 0x8f7e32ce7bea5c70L, 0xd5d238a4abe98068L, 0x9f4f2726179a2245L, + 0xed63a231d4c4fb27L, 0xb0de65388cc8ada8L, 0x83c7088e1aab65dbL, 0xc45d1df942711d9aL, + 0x924d692ca61be758L, 0xda01ee641a708deaL, 0xa26da3999aef774aL, 0xf209787bb47d6b85L, + 0xb454e4a179dd1877L, 0x865b86925b9bc5c2L, 0xc83553c5c8965d3dL, 0x952ab45cfa97a0b3L, + 0xde469fbd99a05fe3L, 0xa59bc234db398c25L, 0xf6c69a72a3989f5cL, 0xb7dcbf5354e9beceL, + 0x88fcf317f22241e2L, 0xcc20ce9bd35c78a5L, 0x98165af37b2153dfL, 0xe2a0b5dc971f303aL, + 0xa8d9d1535ce3b396L, 0xfb9b7cd9a4a7443cL, 0xbb764c4ca7a44410L, 0x8bab8eefb6409c1aL, + 0xd01fef10a657842cL, 0x9b10a4e5e9913129L, 0xe7109bfba19c0c9dL, 0xac2820d9623bf429L, + 0x80444b5e7aa7cf85L, 0xbf21e44003acdd2dL, 0x8e679c2f5e44ff8fL, 0xd433179d9c8cb841L, + 0x9e19db92b4e31ba9L, 0xeb96bf6ebadf77d9L, 0xaf87023b9bf0ee6bL + ) + + /** Binary exponents of the same powers of 10 (fmt's table). */ + private val Pow10Exponents: Array[Int] = Array( + -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954, -927, -901, -874, + -847, -821, -794, -768, -741, -715, -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, + -422, -396, -369, -343, -316, -289, -263, -236, -210, -183, -157, -130, -103, -77, -50, -24, 3, + 30, 56, 83, 109, 136, 162, 189, 216, 242, 269, 295, 322, 348, 375, 402, 428, 455, 481, 508, 534, + 561, 588, 614, 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, 907, 933, 960, 986, 1013, 1039, + 1066 + ) +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/JavaDatePattern.scala b/core/src/main/scala/app/softnetwork/elastic/client/JavaDatePattern.scala new file mode 100644 index 00000000..c20625f5 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/JavaDatePattern.scala @@ -0,0 +1,940 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import java.text.ParsePosition +import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle, SignStyle} +import java.time.temporal.{ + ChronoField, + IsoFields, + Temporal, + TemporalAccessor, + TemporalField, + TemporalQueries, + TemporalUnit, + ValueRange, + WeekFields +} +import java.time.{DayOfWeek, Instant, LocalDate, LocalTime, ZoneId, ZoneOffset, ZonedDateTime} +import java.util.Locale +import scala.util.Try + +/** A custom `java.time` date pattern, read as ONE Elasticsearch major reads it -- Elasticsearch 7 + * and 8, whose custom patterns are `java.time` patterns, and Elasticsearch 6.8 behind a leading + * `8`. Every reading below is MEASURED, major by major (each pattern letter at each width, alone + * and combined, with edge values: week-year boundaries, names, 12-hour clocks, fractions, zones; + * the instant read back from the doc value), and written from that behaviour and Java's own API: + * + * - first, the text is parsed by `java.time` with the pattern's letters, built field by field + * from `DateTimeFormatterBuilder` with the texts and the week rules THAT major's JVM gives + * them + * -- never the JVM core runs on -- and resolved as Elasticsearch does (STRICTLY on 7 and 8): a + * value it does not parse or resolve is not this alternative's, and the next one is tried; + * - then Elasticsearch completes what `java.time` resolved: a resolved date as is; else a year + * (of era) with its month and day, or its day of year, or its month, else -- with no resolved + * time -- January the first, any other field ignored; else a week-based year: the first day of + * its week (week 1 when none, a week past the year's last rolling over); else a month: of + * 1970, its day or the first; the time `java.time` resolved, or midnight -- an unresolved time + * ignored when there is a date, a resolved one with no date on 1970-01-01; no date and no + * time: the value is REJECTED, as is a date that does not exist (the next alternative never + * tried). + * + * A letter or a width no measurement covers, a combination of letter classes outside the measured + * ones (a zone id beside another zone included), a zone other than an offset, `UTC`, `GMT` and + * `UT` (a region id, an abbreviation, a long name: its offset comes from the zone data of the JVM + * reading it -- `IST` is UTC on 8.18.3 and Israel time on 7.17.29), and a reading that would + * depend on the Java version core runs on (a localized offset, a fraction after digits of a + * variable width; on 6.8, an AM/PM with no hour, a week past its week-based year's last) are not + * guessed: they are [[MappedDateFormat.Unmeasured]], and the statement fails, naming the field and + * the format. + */ +private[client] object JavaDatePattern { + + import MappedDateFormat._ + + // --------------------------------------------------------------------------------------------- + // The measured rules of a major + // --------------------------------------------------------------------------------------------- + + /** What a major's `java.time` reading takes from its JVM, MEASURED: the week rules, the texts of + * each text letter (a text two values share -- the narrow `J` -- is the LAST one's; none: the + * letter reads a number), the letters and widths whose values were read back, and the classes of + * letters read in combination. + */ + final case class Rules( + label: String, + weeks: WeekFields, + texts: Map[(Char, Int), Seq[(Long, String)]], + letters: Map[Char, Set[Int]], + dateShapes: Set[String], + timeShapes: Set[String], + zoneKinds: Set[String], + resolver: ResolverStyle = ResolverStyle.STRICT, + alignedField: TemporalField = ChronoField.ALIGNED_WEEK_OF_MONTH, + strictWeeks: Boolean = true + ) + + private def numbered(texts: Seq[String], from: Long = 1L): Seq[(Long, String)] = + texts.zipWithIndex.map { case (t, i) => (i + from, t) } + + private val MonthsShort = + numbered( + Seq("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") + ) + private val MonthsFull = numbered( + Seq( + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December" + ) + ) + private val MonthsNarrow = + numbered(Seq("J", "F", "M", "A", "M", "J", "J", "A", "S", "O", "N", "D")) + private val MonthsDigits = numbered((1 to 12).map(_.toString)) + // ISO order: Monday is 1 + private val DaysShort = numbered(Seq("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")) + private val DaysFull = + numbered(Seq("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")) + private val DaysNarrow = numbered(Seq("M", "T", "W", "T", "F", "S", "S")) + private val QuartersShort = numbered(Seq("Q1", "Q2", "Q3", "Q4")) + private val QuartersFull = + numbered(Seq("1st quarter", "2nd quarter", "3rd quarter", "4th quarter")) + private val QuartersDigits = numbered(Seq("1", "2", "3", "4")) + private val AmPm = numbered(Seq("AM", "PM"), from = 0L) + + private def eras(bc: String, ad: String): Seq[(Long, String)] = Seq(0L -> bc, 1L -> ad) + + private def textsOf( + eraFull: Seq[(Long, String)], + monthsNarrow: Seq[(Long, String)], + quarters: Map[(Char, Int), Seq[(Long, String)]] + ): Map[(Char, Int), Seq[(Long, String)]] = { + val era = Map( + ('G', 1) -> eras("BC", "AD"), + ('G', 2) -> eras("BC", "AD"), + ('G', 3) -> eras("BC", "AD"), + ('G', 4) -> eraFull, + ('G', 5) -> eras("B", "A") + ) + val months = for { + m <- Seq('M', 'L') + (n, t) <- Seq(3 -> MonthsShort, 4 -> MonthsFull, 5 -> monthsNarrow) + } yield (m, n) -> t + val days = for { + e <- Seq('E', 'e', 'c') + (n, t) <- Seq(3 -> DaysShort, 4 -> DaysFull, 5 -> DaysNarrow) ++ + (if (e == 'E') Seq(1 -> DaysShort, 2 -> DaysShort) else Nil) + } yield (e, n) -> t + era ++ months ++ days ++ quarters + (('a', 1) -> AmPm) + } + + private val AllWidths: Map[Char, Set[Int]] = Map( + 'G' -> (1 to 5).toSet, + 'u' -> (1 to 5).toSet, + 'y' -> (1 to 5).toSet, + 'Y' -> (1 to 5).toSet, + 'D' -> (1 to 3).toSet, + 'M' -> (1 to 5).toSet, + 'L' -> (1 to 5).toSet, + 'd' -> Set(1, 2), + 'Q' -> (1 to 5).toSet, + 'q' -> (1 to 5).toSet, + 'w' -> Set(1, 2), + 'W' -> Set(1), + 'E' -> (1 to 5).toSet, + 'e' -> (1 to 5).toSet, + 'c' -> Set(1, 3, 4, 5), + 'F' -> Set(1), + 'a' -> Set(1), + 'h' -> Set(1, 2), + 'K' -> Set(1, 2), + 'k' -> Set(1, 2), + 'H' -> Set(1, 2), + 'm' -> Set(1, 2), + 's' -> Set(1, 2), + 'S' -> (1 to 9).toSet, + 'A' -> (1 to 4).toSet, + 'n' -> (1 to 4).toSet, + 'N' -> (1 to 4).toSet, + 'V' -> Set(2), + 'z' -> (1 to 4).toSet, + 'v' -> Set(1, 4), + 'O' -> Set(1, 4), + 'X' -> (1 to 5).toSet, + 'x' -> (1 to 5).toSet, + 'Z' -> (1 to 5).toSet + ) + + /** The classes of date letters read in combination -- each with every class of time letters below + * (MEASURED on 8.18.3 and 7.17.29: the full product, with every zone): `G` era, `y` year of era, + * `u` year, `Y` week-based year, `M` month (`L`), `d` day, `D` day of year, `Q` quarter (`q`), + * `w` week of week-based year, `W` week of month, `E` day-of-week text (`e`, `c` at 3 and more), + * `e` localized day-of-week number (`c` at 1), `F` aligned week of month. + */ + private val DateShapes: Set[String] = Set( + "", + "D", + "E", + "F", + "G", + "M", + "Q", + "W", + "Y", + "d", + "e", + "u", + "w", + "y", + "Gy", + "MW", + "Md", + "YD", + "YM", + "Ye", + "Yw", + "dD", + "uD", + "uE", + "uM", + "uQ", + "uw", + "wE", + "yD", + "yE", + "yM", + "yQ", + "ye", + "yw", + "MWe", + "MdD", + "MdE", + "Mde", + "YMW", + "YMd", + "YwE", + "Ywe", + "uDE", + "uDe", + "uMD", + "uME", + "uMF", + "uMQ", + "uMW", + "uMd", + "uMw", + "uWe", + "udQ", + "uwe", + "yDe", + "yMD", + "yME", + "yMF", + "yMW", + "yMd", + "yMe", + "yQw", + "ywE", + "ywe", + "GYwe", + "GuMd", + "GyMD", + "GyMd", + "YMWe", + "YMde", + "uMEF", + "uMWE", + "uMWe", + "uMdD", + "uMdE", + "uMdQ", + "uMde", + "uMeF", + "uYMd", + "yMWE", + "yMWe", + "yMdD", + "yMdE", + "yMdQ", + "yMdW", + "yMde", + "yMdw", + "yMeF", + "yYMd", + "GyMdE", + "GyMde" + ) + + /** The classes of time letters read in combination (MEASURED): `a` AM/PM, `h` clock hour of + * AM/PM, `K` hour of AM/PM, `k` clock hour of day, `H` hour of day, `m`, `s`, `S` fraction, `n` + * nano of second, `A` milli of day, `N` nano of day. + */ + private val TimeShapes: Set[String] = Set( + "", + "A", + "H", + "K", + "N", + "S", + "a", + "h", + "k", + "m", + "n", + "s", + "HA", + "Hm", + "Km", + "ah", + "hm", + "km", + "HmN", + "Hms", + "Kms", + "aHm", + "aKm", + "ahm", + "akm", + "kms", + "HmsA", + "HmsS", + "Hmsn", + "ahms", + "HmsSN", + "HmsSn", + "ahmsS" + ) + + private val ZoneKinds: Set[String] = Set("", "offset", "id", "name") + + /** Elasticsearch 8.18.3: weeks start on Sunday, week 1 holds January the first; CLDR English. */ + val Es8: Rules = Rules( + "Elasticsearch 8", + WeekFields.of(DayOfWeek.SUNDAY, 1), + textsOf( + eras("Before Christ", "Anno Domini"), + MonthsNarrow, + Map( + ('Q', 3) -> QuartersShort, + ('Q', 4) -> QuartersFull, + ('Q', 5) -> QuartersDigits, + ('q', 3) -> QuartersShort, + ('q', 4) -> QuartersFull, + ('q', 5) -> QuartersDigits + ) + ), + AllWidths, + DateShapes, + TimeShapes, + ZoneKinds + ) + + /** Elasticsearch 7.17.29: ISO weeks (Monday, 4 days); the full era `CE`, the narrow month and + * quarter their number's text (`1`, not `01`), the full quarter `Q1`, the stand-alone quarter a + * NUMBER (`01` and `5` are read). + */ + val Es7: Rules = Rules( + "Elasticsearch 7", + WeekFields.ISO, + textsOf( + eras("BCE", "CE"), + MonthsDigits, + Map( + ('Q', 3) -> QuartersShort, + ('Q', 4) -> QuartersShort, + ('Q', 5) -> QuartersDigits, + // no stand-alone quarter texts: a number, as `java.time` reads a text it has none for + ('q', 3) -> Nil, + ('q', 4) -> Nil, + ('q', 5) -> Nil + ) + ), + AllWidths, + DateShapes, + TimeShapes, + ZoneKinds + ) + + /** Elasticsearch 6.8's `java.time` mode, selected by a leading `8` (MEASURED on 6.8.23): weeks + * starting on Sunday, Elasticsearch 7's texts, a SMART resolution, `F` the aligned day of week + * in month, a week 53 of a year of 52 read into the next year. + */ + val Es6: Rules = Es7.copy( + label = "Elasticsearch 6", + weeks = WeekFields.of(DayOfWeek.SUNDAY, 1), + resolver = ResolverStyle.SMART, + alignedField = ChronoField.ALIGNED_DAY_OF_WEEK_IN_MONTH, + strictWeeks = false + ) + + // --------------------------------------------------------------------------------------------- + // The pattern + // --------------------------------------------------------------------------------------------- + + private sealed trait Token + private final case class Letters(c: Char, n: Int) extends Token + private final case class Literal(text: String) extends Token + private case object OptionalStart extends Token + private case object OptionalEnd extends Token + + /** The tokens of a `java.time` pattern, or why it is not one core reads. */ + private def tokens(pattern: String): Either[String, Vector[Token]] = { + val out = Vector.newBuilder[Token] + var i = 0 + var depth = 0 + var error: Option[String] = None + while (error.isEmpty && i < pattern.length) { + val c = pattern.charAt(i) + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')) { + var j = i + while (j < pattern.length && pattern.charAt(j) == c) j += 1 + out += Letters(c, j - i) + i = j + } else if (c == '\'') { + val sb = new StringBuilder + var j = i + 1 + var closed = false + while (!closed && j < pattern.length) { + if (pattern.charAt(j) == '\'') { + if (j + 1 < pattern.length && pattern.charAt(j + 1) == '\'') { + sb.append('\''); j += 2 + } else { closed = true; j += 1 } + } else { sb.append(pattern.charAt(j)); j += 1 } + } + if (!closed) error = Some("an unclosed quote") + else out += Literal(if (j == i + 2) "'" else sb.toString) + i = j + } else if (c == '[') { depth += 1; out += OptionalStart; i += 1 } + else if (c == ']') { + if (depth == 0) error = Some("an unopened ']'") + depth -= 1; out += OptionalEnd; i += 1 + } else if (c == '{' || c == '}' || c == '#') error = Some(s"the reserved '$c'") + else { out += Literal(c.toString); i += 1 } + } + error.toLeft(out.result()) + } + + /** A field holding a letter's value in the lexical pass, which says which letters a text carried + * (an optional section may not); its range and its values are the real field's. + */ + private final class Carrier(val letter: Char, val real: TemporalField) extends TemporalField { + def getBaseUnit: TemporalUnit = real.getBaseUnit + def getRangeUnit: TemporalUnit = real.getRangeUnit + def range(): ValueRange = real.range() + def isDateBased: Boolean = real.isDateBased + def isTimeBased: Boolean = real.isTimeBased + def isSupportedBy(temporal: TemporalAccessor): Boolean = false + def rangeRefinedBy(temporal: TemporalAccessor): ValueRange = real.rangeRefinedBy(temporal) + def getFrom(temporal: TemporalAccessor): Long = real.getFrom(temporal) + def adjustInto[R <: Temporal](temporal: R, newValue: Long): R = + real.adjustInto(temporal, newValue) + override def toString: String = s"Carrier($letter)" + } + + /** The class a letter stands for in a shape. */ + private def letterClass(c: Char, n: Int): Char = + c match { + case 'L' => 'M' + case 'q' => 'Q' + case 'e' | 'c' if n >= 3 => 'E' + case 'c' => 'e' + case other => other + } + + private val DateOrder = "GyuYMdDQwWEeF" + private val TimeOrder = "ahKkHmsSnAN" + + private def zoneKind(c: Char): String = + c match { + case 'V' => "id" + case 'z' | 'v' => "name" + case _ => "offset" + } + + /** The formatter of the tokens: with `carriers`, each letter class in its [[Carrier]] (the + * lexical pass), else in its real field (the resolving pass). Built ONLY from + * `DateTimeFormatterBuilder` calls of Java 8, with the major's texts and week rules. + */ + private def formatter( + tokens: Vector[Token], + rules: Rules, + carriers: Option[scala.collection.mutable.Map[Char, Carrier]] + ): DateTimeFormatter = { + val b = new DateTimeFormatterBuilder() + val weeks = rules.weeks + val base2000 = LocalDate.of(2000, 1, 1) + def field(c: Char, n: Int, real: TemporalField): TemporalField = + carriers.fold(real) { registry => + val cls = letterClass(c, n) + registry.getOrElseUpdate(cls, new Carrier(cls, real)) + } + def text(c: Char, n: Int, real: TemporalField): Unit = + rules.texts((c, n)) match { + // no texts: a number, as `java.time` reads a text its locale data has none for + case Nil => b.appendValue(field(c, n, real), 1, 19, SignStyle.NORMAL) + case texts => + val map = new java.util.LinkedHashMap[java.lang.Long, String]() + texts.foreach { case (v, t) => map.put(java.lang.Long.valueOf(v), t) } + b.appendText(field(c, n, real), map) + } + def year(c: Char, n: Int, real: TemporalField): Unit = { + val f = field(c, n, real) + n match { + case 2 => b.appendValueReduced(f, 2, 2, base2000) + case 1 => b.appendValue(f, 1, 19, SignStyle.NORMAL) + case 3 => b.appendValue(f, 3, 19, SignStyle.NORMAL) + case _ => b.appendValue(f, n, 19, SignStyle.EXCEEDS_PAD) + } + } + def number(c: Char, n: Int, real: TemporalField): Unit = + if (n == 1) b.appendValue(field(c, n, real)) else b.appendValue(field(c, n, real), n) + tokens.foreach { + case Literal(t) => b.appendLiteral(t) + case OptionalStart => b.optionalStart() + case OptionalEnd => b.optionalEnd() + case Letters(c, n) => + c match { + case 'G' => text(c, n, ChronoField.ERA) + case 'u' => year(c, n, ChronoField.YEAR) + case 'y' => year(c, n, ChronoField.YEAR_OF_ERA) + case 'Y' => year(c, n, weeks.weekBasedYear()) + case 'M' | 'L' => + if (n <= 2) number(c, n, ChronoField.MONTH_OF_YEAR) + else text(c, n, ChronoField.MONTH_OF_YEAR) + case 'd' => number(c, n, ChronoField.DAY_OF_MONTH) + case 'D' => + n match { + case 1 => b.appendValue(field(c, n, ChronoField.DAY_OF_YEAR)) + case 2 => + b.appendValue(field(c, n, ChronoField.DAY_OF_YEAR), 2, 3, SignStyle.NOT_NEGATIVE) + case _ => b.appendValue(field(c, n, ChronoField.DAY_OF_YEAR), 3) + } + case 'Q' | 'q' => + if (n <= 2) number(c, n, IsoFields.QUARTER_OF_YEAR) + else text(c, n, IsoFields.QUARTER_OF_YEAR) + case 'w' => + b.appendValue(field(c, n, weeks.weekOfWeekBasedYear()), n, 2, SignStyle.NOT_NEGATIVE) + case 'W' => b.appendValue(field(c, n, weeks.weekOfMonth()), 1, 1, SignStyle.NOT_NEGATIVE) + case 'E' => text(c, n, ChronoField.DAY_OF_WEEK) + case 'e' | 'c' => + if (n <= 2) + b.appendValue(field(c, n, weeks.dayOfWeek()), n, n, SignStyle.NOT_NEGATIVE) + else text(c, n, ChronoField.DAY_OF_WEEK) + case 'F' => b.appendValue(field(c, n, rules.alignedField)) + case 'a' => text(c, n, ChronoField.AMPM_OF_DAY) + case 'h' => number(c, n, ChronoField.CLOCK_HOUR_OF_AMPM) + case 'K' => number(c, n, ChronoField.HOUR_OF_AMPM) + case 'k' => number(c, n, ChronoField.CLOCK_HOUR_OF_DAY) + case 'H' => number(c, n, ChronoField.HOUR_OF_DAY) + case 'm' => number(c, n, ChronoField.MINUTE_OF_HOUR) + case 's' => number(c, n, ChronoField.SECOND_OF_MINUTE) + case 'S' => b.appendFraction(field(c, n, ChronoField.NANO_OF_SECOND), n, n, false) + case 'A' => + b.appendValue(field(c, n, ChronoField.MILLI_OF_DAY), n, 19, SignStyle.NOT_NEGATIVE) + case 'n' => + b.appendValue(field(c, n, ChronoField.NANO_OF_SECOND), n, 19, SignStyle.NOT_NEGATIVE) + case 'N' => + b.appendValue(field(c, n, ChronoField.NANO_OF_DAY), n, 19, SignStyle.NOT_NEGATIVE) + case 'V' | 'z' | 'v' => b.appendZoneId() + case 'O' => + b.appendLocalizedOffset( + if (n == 1) java.time.format.TextStyle.SHORT else java.time.format.TextStyle.FULL + ) + case 'X' => + b.appendOffset(Seq("+HHmm", "+HHMM", "+HH:MM", "+HHMMss", "+HH:MM:ss")(n - 1), "Z") + case 'x' => + b.appendOffset( + Seq("+HHmm", "+HHMM", "+HH:MM", "+HHMMss", "+HH:MM:ss")(n - 1), + Seq("+00", "+0000", "+00:00", "+0000", "+00:00")(n - 1) + ) + case 'Z' => + n match { + case 4 => b.appendLocalizedOffset(java.time.format.TextStyle.FULL) + case 5 => b.appendOffset("+HH:MM:ss", "Z") + case _ => b.appendOffset("+HHMM", "+0000") + } + case other => throw new IllegalArgumentException(s"letter $other") + } + } + b.toFormatter(Locale.ROOT).withResolverStyle(rules.resolver) + } + + /** A pattern's two formatters -- the lexical pass's and the resolving pass's -- its letters' + * [[Carrier]] s, and the kind of its zone letter. + */ + private final case class Compiled( + lexical: DateTimeFormatter, + resolving: DateTimeFormatter, + carriers: Map[Char, Carrier], + zone: String, + lexicalPass: Boolean + ) + + /** A custom pattern of one major. */ + final case class Pattern(pattern: String, rules: Rules) extends Alternative { + override def isName: Boolean = false + + private val compiled: Either[String, Compiled] = + tokens(pattern).flatMap { ts => + val letters = ts.collect { case l: Letters => l } + letters.find(l => !rules.letters.get(l.c).exists(_.contains(l.n))) match { + case Some(Letters(c, n)) => + Left(s"the letter '${String.valueOf(c) * n}' is not one core has measured") + case _ if clientDependent(ts, rules).isDefined => Left(clientDependent(ts, rules).get) + case _ => + val registry = scala.collection.mutable.Map.empty[Char, Carrier] + val zones = letters.filter(l => ZoneLetters.indexOf(l.c.toInt) >= 0) + val zone = zones.headOption.fold("")(l => zoneKind(l.c)) + val classes = letters.map(l => letterClass(l.c, l.n)).toSet + // the lexical pass says which letters a text carried (an optional section), checks + // the values older Java resolvers let through (an hour of AM/PM, a week of a week + // year beside a day of week, a week past the week year's last on 6.8) and tells a zone + // name core does not read from a text the format rejects; any other pattern carries + // all its letters, in one parse + val lexicalPass = ts.contains(OptionalStart) || zone == "name" || classes('K') || + (classes('Y') && classes('w') && + (!rules.strictWeeks || classes('e') || classes('E'))) + Try( + Compiled( + formatter(ts, rules, Some(registry)), + formatter(ts, rules, None), + registry.toMap, + zone, + lexicalPass + ) + ).toEither.left + .map(e => s"its letters do not compile: ${e.getMessage}") + .flatMap(c => + // several zone letters: measured only as offsets (`[XXX][X]`) + if (zones.size > 1 && zones.exists(l => zoneKind(l.c) != "offset")) + Left( + s"its zone letters ${zones.map(l => s"'${String.valueOf(l.c) * l.n}'").mkString(", ")}" + + " combine a zone id or name with another zone, which core has not measured" + ) + else if (c.lexicalPass) Right(c) + else shape(c.carriers.keySet, zone).toLeft(c) + ) + } + } + + /** Why the pattern is not read, if it is not. */ + def unread: Option[String] = compiled.left.toOption + + def parse(text: String): Parse = + compiled match { + case Left(reason) => Unmeasured(s"'$pattern' is not a date format core reads: $reason") + case Right(c) if !c.lexicalPass => + // a text the pattern does not parse, told without an exception (a value the response + // parser read as a date is tried in many spellings) + val pos = new ParsePosition(0) + val raw = Try(c.resolving.parseUnresolved(text, pos)).toOption.orNull + if (raw == null || pos.getErrorIndex >= 0 || pos.getIndex != text.length) + unknownZone(text, pos, c).fold[Parse](NotParsed)(Unmeasured) + else resolve(text, c) + case Right(c) => + val pos = new ParsePosition(0) + val raw = Try(c.lexical.parseUnresolved(text, pos)).toOption.orNull + if (raw == null || pos.getErrorIndex >= 0 || pos.getIndex != text.length) + // a zone name is open-ended: a text not read may hold a name Elasticsearch reads + if (c.zone == "name") + Unmeasured(s"a zone in '$text' is not one core reads with '$pattern'") + else unknownZone(text, pos, c).fold[Parse](NotParsed)(Unmeasured) + else + shapeOf(raw, c).orElse(weekPastYear(raw, c)) match { + case Some(reason) => Unmeasured(reason) + case None if !valid(raw, c) => NotParsed + case None => resolve(text, c) + } + } + + /** Why a text a zone id pattern (`VV`) did not parse is not read, when it failed at a zone id: + * whether the JVM core runs on knows a region (`America/Ciudad_Juarez`) depends on its zone + * data, and core reads no region. + */ + private def unknownZone(text: String, pos: ParsePosition, c: Compiled): Option[String] = + if (c.zone != "id" || pos.getErrorIndex < 0 || pos.getErrorIndex >= text.length) None + else { + val at = pos.getErrorIndex + val m = ZoneToken.matcher(text).region(at, text.length) + // the zone id is what failed when `UTC` in its place parses + lazy val asUtc = { + val probe = new ParsePosition(0) + val utc = text.substring(0, at) + "UTC" + text.substring(m.end()) + Try(c.lexical.parseUnresolved(utc, probe)).toOption.exists(_ != null) && + probe.getErrorIndex < 0 && probe.getIndex == utc.length + } + if (m.lookingAt() && fixedId(m.group(), prefixed = true).isEmpty && asUtc) + Some(zoneNotRead(m.group())) + else None + } + + /** The text resolved by `java.time`, then completed as Elasticsearch completes it. */ + private def resolve(text: String, c: Compiled): Parse = + Try(c.resolving.parse(text)).toOption match { + case None => NotParsed + case Some(resolved) => + zoneOf(resolved) match { + case Left(reason) => Unmeasured(reason) + case Right(zone) => completed(resolved, zone, rules).fold[Parse](Invalid)(Parsed) + } + } + + /** Whether every TIME value a text carried is in its field's range, as Elasticsearch's + * `java.time` checks it (MEASURED: an hour of AM/PM 12, a clock hour 0, a milli of day of + * 86400000 are rejected, where a date value its completion ignores is not checked -- a quarter + * 5 in `yyyy-Q`, a month 53 in `YYYY-MM`), and a week of week-based year with a day of week is + * a week of THAT year (week 53 of a year of 52 is rejected, where without a day of week it + * rolls over into the next year) -- checked here rather than left to the `java.time` core runs + * on, whose STRICT resolution of these differs across Java versions. + */ + private def valid(raw: TemporalAccessor, c: Compiled): Boolean = + Try { + // STRICT: every time value; SMART (6.8): the hour of AM/PM, which older Java resolvers + // do not check (the other SMART checks are the same in every Java version measured) + val checked: Carrier => Boolean = + if (rules.resolver == ResolverStyle.STRICT) _.isTimeBased + else _.real == ChronoField.HOUR_OF_AMPM + val inRange = c.carriers.forall { case (_, f) => + !checked(f) || !raw.isSupported(f) || f.range().isValidValue(raw.getLong(f)) + } + def value(cls: Char): Option[Long] = + c.carriers.get(cls).filter(raw.isSupported).map(raw.getLong) + val weekOfYear = (value('Y'), value('w')) match { + case (Some(wby), Some(w)) + if rules.strictWeeks && (value('e').isDefined || value('E').isDefined) => + val first = LocalDate.of(Math.toIntExact(wby), 1, rules.weeks.getMinimalDaysInFirstWeek) + rules.weeks.weekOfWeekBasedYear().rangeRefinedBy(first).isValidValue(w) + case _ => true + } + inRange && weekOfYear + }.getOrElse(false) + + /** Why the letters a text carried are a combination core has not measured, if they are. */ + private def shapeOf(raw: TemporalAccessor, c: Compiled): Option[String] = + shape( + c.carriers.collect { case (cls, f) if raw.isSupported(f) => cls }.toSet, + if (raw.query(TemporalQueries.zone()) == null) "" else c.zone + ) + + /** On Elasticsearch 6.8's java.time mode (SMART), why a week of a week-based year past that + * year's last is not read, if it is: Elasticsearch reads it into the next year (6.8.23, Java + * 15), which Java 21's resolution rejects -- a reading that would depend on the JVM core runs + * on. + */ + private def weekPastYear(raw: TemporalAccessor, c: Compiled): Option[String] = + if (rules.strictWeeks) None + else + ( + c.carriers.get('Y').filter(raw.isSupported).map(raw.getLong), + c.carriers.get('w').filter(raw.isSupported).map(raw.getLong) + ) match { + case (Some(wby), Some(w)) => + val past = Try { + val first = + LocalDate.of(Math.toIntExact(wby), 1, rules.weeks.getMinimalDaysInFirstWeek) + !rules.weeks.weekOfWeekBasedYear().rangeRefinedBy(first).isValidValue(w) + }.getOrElse(false) + if (past) + Some( + s"'$pattern' reads a week past the last of its week-based year, which " + + s"${rules.label}'s java.time mode resolves differently from one Java version to " + + "another" + ) + else None + case _ => None + } + + /** Why `carried` letter classes and a `zone` kind are a combination core has not measured. */ + private def shape(carried: scala.collection.Set[Char], zone: String): Option[String] = { + val date = DateOrder.filter(carried.contains) + val time = TimeOrder.filter(carried.contains) + if (!rules.dateShapes.contains(date) || !rules.timeShapes.contains(time)) + Some( + s"the letters of '$pattern' combine as '$date${if (time.nonEmpty) "|" + time else ""}'," + + " which core has not measured" + ) + // Elasticsearch 6.8 resolves SMARTLY, with its own JVM: an AM/PM with no hour is ignored + // there (6.8.23 bundles Java 15), set to 06:00 or 18:00 from Java 16 on -- a reading that + // would depend on the JVM core runs on + else if (rules.resolver == ResolverStyle.SMART && carried('a') && !"hKkH".exists(carried)) + Some( + s"'$pattern' reads an AM/PM with no hour, which ${rules.label}'s java.time mode resolves " + + "differently from one Java version to another" + ) + else if (!rules.zoneKinds.contains(zone)) + Some(s"the zone of '$pattern' is not one core has measured") + else None + } + } + + private val ZoneLetters = "VzvOXxZ" + + private val ZoneToken = java.util.regex.Pattern.compile("[A-Za-z][A-Za-z0-9_/+~-]*+") + + /** The zone a resolved text gives -- `None` for no zone -- or why core does not read it: only an + * offset, `UTC`, `GMT`, `UT` and their offsets (`GMT+01:00`) are read, the same on every JVM. A + * region id (`Europe/Paris`) takes its offset from the zone data of the JVM reading it, an + * abbreviation or a long name (`CET`, `PST`, `Pacific Time`) from its name data (MEASURED: `IST` + * is UTC on 8.18.3, Israel time on 7.17.29; `America/Mexico_City` is 17:00Z on 8.18.3 where an + * older zone data gives 18:00Z): never Elasticsearch's. + */ + private def zoneOf(resolved: TemporalAccessor): Either[String, Option[ZoneId]] = + Option(resolved.query(TemporalQueries.zone())) match { + case None => Right(None) + case Some(o: ZoneOffset) => Right(Some(o)) + case Some(z) => + fixedId(z.getId, prefixed = true) + .map(o => Some(o: ZoneId)) + .toRight(zoneNotRead(z.getId)) + } + + /** Why a pattern reads differently from one Java version core runs on to another, if it does + * (MEASURED: the corpora read with Java 8, 11, 17 and 21): + * - a localized offset (`O`, `OOOO`, `ZZZZ`: `GMT+1`), which Java 8 does not parse as later + * versions do; + * - a fraction (`S`) right after digits of a variable width (`yyyyMMddHHmmssSSS`): later + * versions keep the fraction's digits apart from the variable width's, Java 8 does not. + */ + private def clientDependent(tokens: Vector[Token], rules: Rules): Option[String] = { + // the adjacent values `java.time` parses together: none, led by a value of a fixed width, or + // led by a value of a variable width -- any other element ends them + var chain = 0 + var reason: Option[String] = None + def fixed(c: Char, n: Int): Option[Boolean] = + c match { + // a text letter with no texts on this major reads a number of a variable width + case _ if rules.texts.get((c, n)).exists(_.isEmpty) => Some(false) + case 'u' | 'y' | 'Y' => Some(n == 2) + case 'M' | 'L' | 'Q' | 'q' if n >= 3 => None + case 'e' | 'c' if n >= 3 => None + case 'e' | 'c' | 'W' => Some(true) + case 'D' => Some(n == 3) + case 'F' | 'A' | 'n' | 'N' => Some(false) + case 'M' | 'L' | 'Q' | 'q' | 'd' | 'w' => Some(n == 2) + case 'h' | 'K' | 'k' | 'H' | 'm' | 's' => Some(n == 2) + case _ => None + } + tokens.foreach { + case Letters(c, n) if c == 'O' || (c == 'Z' && n == 4) => + reason = reason.orElse( + Some( + s"the letter '${String.valueOf(c) * n}' is a localized offset (`GMT+1`), which Java 8 " + + "parses differently from later Java versions" + ) + ) + case Letters('S', n) => + if (chain == 2) + reason = reason.orElse( + Some( + s"the fraction '${"S" * n}' follows digits of a variable width with no separator, " + + "which Java 8 parses differently from later Java versions" + ) + ) + if (chain == 0) chain = 1 + case Letters(c, n) => + chain = fixed(c, n) match { + case Some(true) => if (chain == 0) 1 else chain + case Some(false) => 2 + case None => 0 + } + case _ => chain = 0 + } + reason + } + + /** Elasticsearch's completion of what `java.time` resolved (MEASURED, 7.17.29 and 8.18.3): see + * the object's documentation. `None` when the value is rejected. + */ + private def completed( + t: TemporalAccessor, + zone: Option[ZoneId], + rules: Rules + ): Option[Instant] = + Try { + def has(f: TemporalField) = t.isSupported(f) + def int(f: TemporalField) = Math.toIntExact(t.getLong(f)) + val weeks = rules.weeks + val time = Option(t.query(TemporalQueries.localTime())) + val resolvedDate = Option(t.query(TemporalQueries.localDate())) + // a year with a day of week and no date, with a resolved time: REJECTED (MEASURED, + // `Unsupported field: DayOfYear`), where it is the year's first day with no time + if ( + resolvedDate.isEmpty && time.isDefined && has(ChronoField.YEAR) && + has(ChronoField.DAY_OF_WEEK) + ) + throw new IllegalArgumentException("a year and a day of week with a time") + val date: Option[LocalDate] = resolvedDate.orElse { + val year = + if (has(ChronoField.YEAR)) Some(int(ChronoField.YEAR)) + else if (has(ChronoField.YEAR_OF_ERA)) Some(int(ChronoField.YEAR_OF_ERA)) + else None + year + .flatMap { y => + if (has(ChronoField.MONTH_OF_YEAR) && has(ChronoField.DAY_OF_MONTH)) + Some(LocalDate.of(y, int(ChronoField.MONTH_OF_YEAR), int(ChronoField.DAY_OF_MONTH))) + else if (has(ChronoField.DAY_OF_YEAR)) + Some(LocalDate.ofYearDay(y, int(ChronoField.DAY_OF_YEAR))) + else if (has(ChronoField.MONTH_OF_YEAR)) + Some(LocalDate.of(y, int(ChronoField.MONTH_OF_YEAR), 1)) + // a year alone, with a resolved time, gives no date: 1970-01-01 (MEASURED) + else if (time.isDefined) None + else Some(LocalDate.of(y, 1, 1)) + } + .orElse { + if (has(weeks.weekBasedYear())) { + val first = LocalDate.of( + int(weeks.weekBasedYear()), + 1, + weeks.getMinimalDaysInFirstWeek + ) + val week = + if (has(weeks.weekOfWeekBasedYear())) t.getLong(weeks.weekOfWeekBasedYear()) + else 1L + Some( + first.`with`(weeks.weekOfWeekBasedYear(), week).`with`(weeks.dayOfWeek(), 1L) + ) + } else None + } + .orElse { + if (has(ChronoField.MONTH_OF_YEAR)) + Some( + LocalDate.of( + 1970, + int(ChronoField.MONTH_OF_YEAR), + if (has(ChronoField.DAY_OF_MONTH)) int(ChronoField.DAY_OF_MONTH) else 1 + ) + ) + else None + } + } + val at = zone.getOrElse(ZoneOffset.UTC) + (date, time) match { + case (Some(d), tm) => ZonedDateTime.of(d, tm.getOrElse(LocalTime.MIDNIGHT), at).toInstant + case (None, Some(tm)) => ZonedDateTime.of(LocalDate.of(1970, 1, 1), tm, at).toInstant + case _ => throw new IllegalArgumentException("no date and no time") + } + }.toOption +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/JodaDatePattern.scala b/core/src/main/scala/app/softnetwork/elastic/client/JodaDatePattern.scala new file mode 100644 index 00000000..33da57f4 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/JodaDatePattern.scala @@ -0,0 +1,666 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import java.time.temporal.{ChronoUnit, IsoFields} +import java.time.{Clock, DayOfWeek, Instant, LocalDate, LocalDateTime, ZoneOffset} +import java.util.Locale +import scala.util.Try + +/** A custom Joda-Time date pattern, read as Elasticsearch 6 reads it. Every reading below is + * MEASURED on 6.8.23 (each letter `G C Y x w e E y D M d a K h H k m s S z Z` at each width, alone + * and combined, with edge values; the instant read back from the doc value) and written from that + * behaviour and Java's own API, never from Joda-Time's code: + * + * - first, the text is read left to right, each letter at its place: a number of one digit up to + * a maximum (2 for `M d H h K k m s w`, 3 for `D`, 1 for `e`, the letter count for `S`, `C` + * and a repeated letter; 9 for a year, or its letter count when a number follows), a year + * signed (not a year of era), `yy` `YY` `xx` two digits around the current year minus 30 (any + * other count of digits as written); a name in any case, short or full (`MMM`, `E`, `G`, `a`); + * an offset (`Z`, `ZZ`: `Z`, `z`, `+HH`, `+HHmm`, `+HH:mm`, with seconds, up to 19 hours); a + * zone id (`ZZZ`); one of eleven zone names (`z`: `UT`, `UTC`, `GMT` and the North American + * `EST` ... `PDT`); literal text in any case; + * - then the values are applied to 1970-01-01T00:00, largest range first (the year, the week + * year and the era before the month, the week, the day of year; the day of month; the day of + * week; then the time), each one resetting the smaller fields to their first value, and + * applied a second time, the last one resetting; when the first is a month, a week or a day, + * the year is 1970; a value out of its field's range there REJECTS the text; + * - last, the zone: an offset as given; `UTC` and `GMT` (`ZZZ`), `UT`, `UTC` and `GMT` (`z`) as + * UTC. + * + * A letter or a width no measurement covers, a combination of letter classes outside the measured + * ones, two zone letters, and any other zone id or name -- a region (`Europe/Paris`), an + * abbreviation (`EST`, `PDT`), whose offset comes from zone data, the JVM core runs on's and not + * Elasticsearch's -- are [[MappedDateFormat.Unmeasured]]. + */ +private[client] object JodaDatePattern { + + import MappedDateFormat._ + + // --------------------------------------------------------------------------------------------- + // The fields, ordered as Elasticsearch 6 applies them (MEASURED): by the size of their range, + // then of their unit + // --------------------------------------------------------------------------------------------- + + private sealed abstract class Field(val letter: Char, val range: Int, val unit: Int) { + + /** The local date-time with this field set to `v`; throws when `v` is out of its range. */ + def set(t: LocalDateTime, v: Int): LocalDateTime + + /** The start of this field's unit. */ + def floor(t: LocalDateTime): LocalDateTime + } + + // range and unit sizes: no range (a year) 100, eras 100, centuries 90, years 80, months 70, + // weeks 60, days 50, half days 40, hours 30, minutes 20, seconds 10, millis 0 + private def check(v: Int, min: Int, max: Int): Int = + if (v < min || v > max) throw new IllegalArgumentException(s"$v out of [$min, $max]") + else v + + private def startOfYear(y: Int): LocalDateTime = LocalDateTime.of(y, 1, 1, 0, 0) + + private def weekYear(t: LocalDateTime): Int = t.get(IsoFields.WEEK_BASED_YEAR) + private def week(t: LocalDateTime): Int = t.get(IsoFields.WEEK_OF_WEEK_BASED_YEAR) + + private def weeksIn(weekYear: Int): Int = + IsoFields.WEEK_OF_WEEK_BASED_YEAR.rangeRefinedBy(LocalDate.of(weekYear, 1, 4)).getMaximum.toInt + + private def firstWeekDay(weekYear: Int): LocalDate = + LocalDate + .of(weekYear, 1, 4) + .`with`(IsoFields.WEEK_OF_WEEK_BASED_YEAR, 1L) + .`with`(DayOfWeek.MONDAY) + + private case object Era extends Field('G', 100, 100) { + def set(t: LocalDateTime, v: Int): LocalDateTime = { + check(v, 0, 1) + if ((t.getYear <= 0) != (v == 0)) t.withYear(1 - t.getYear) else t + } + // the first instant of the era: AD 1 for AD, the first representable instant for BC + def floor(t: LocalDateTime): LocalDateTime = + if (t.getYear >= 1) startOfYear(1) + else LocalDateTime.ofInstant(Instant.ofEpochMilli(Long.MinValue), ZoneOffset.UTC) + } + + /** The century of the year's absolute value, its sign kept (MEASURED: a century 0 reads years 0 + * to 99). + */ + private case object CenturyOfEra extends Field('C', 100, 90) { + private def withAbsoluteYear(t: LocalDateTime, v: Int) = + t.withYear(if (t.getYear < 0) -v else v) + def set(t: LocalDateTime, v: Int): LocalDateTime = + withAbsoluteYear(t, check(v, 0, 2922789) * 100 + Math.abs(t.getYear) % 100) + def floor(t: LocalDateTime): LocalDateTime = + startOfYear(withAbsoluteYear(t, (Math.abs(t.getYear) / 100) * 100).getYear) + } + + private case object YearOfEra extends Field('Y', 100, 80) { + def set(t: LocalDateTime, v: Int): LocalDateTime = { + check(v, 1, 292278993) + t.withYear(if (t.getYear <= 0) 1 - v else v) + } + def floor(t: LocalDateTime): LocalDateTime = startOfYear(t.getYear) + } + + private case object Year extends Field('y', 100, 80) { + def set(t: LocalDateTime, v: Int): LocalDateTime = t.withYear(check(v, -292275054, 292278993)) + def floor(t: LocalDateTime): LocalDateTime = startOfYear(t.getYear) + } + + private case object WeekYear extends Field('x', 100, 80) { + def set(t: LocalDateTime, v: Int): LocalDateTime = { + check(v, -292275054, 292278993) + if (v == weekYear(t)) t + else { + val w = Math.min(week(t), weeksIn(v)) + LocalDateTime.of( + firstWeekDay(v).plusWeeks(w - 1L).plusDays(t.getDayOfWeek.getValue - 1L), + t.toLocalTime + ) + } + } + def floor(t: LocalDateTime): LocalDateTime = firstWeekDay(weekYear(t)).atStartOfDay() + } + + private case object MonthOfYear extends Field('M', 80, 70) { + def set(t: LocalDateTime, v: Int): LocalDateTime = t.withMonth(check(v, 1, 12)) + def floor(t: LocalDateTime): LocalDateTime = t.toLocalDate.withDayOfMonth(1).atStartOfDay() + } + + private case object WeekOfWeekYear extends Field('w', 80, 60) { + def set(t: LocalDateTime, v: Int): LocalDateTime = + t.plusWeeks(check(v, 1, weeksIn(weekYear(t))).toLong - week(t)) + def floor(t: LocalDateTime): LocalDateTime = + t.toLocalDate.`with`(DayOfWeek.MONDAY).atStartOfDay() + } + + private case object DayOfYear extends Field('D', 80, 50) { + def set(t: LocalDateTime, v: Int): LocalDateTime = + t.withDayOfYear(check(v, 1, t.toLocalDate.lengthOfYear)) + def floor(t: LocalDateTime): LocalDateTime = t.toLocalDate.atStartOfDay() + } + + private case object DayOfMonth extends Field('d', 70, 50) { + def set(t: LocalDateTime, v: Int): LocalDateTime = + t.withDayOfMonth(check(v, 1, t.toLocalDate.lengthOfMonth)) + def floor(t: LocalDateTime): LocalDateTime = t.toLocalDate.atStartOfDay() + } + + private case object DayOfWeekField extends Field('e', 60, 50) { + def set(t: LocalDateTime, v: Int): LocalDateTime = + t.plusDays(check(v, 1, 7).toLong - t.getDayOfWeek.getValue) + def floor(t: LocalDateTime): LocalDateTime = t.toLocalDate.atStartOfDay() + } + + private case object HalfDay extends Field('a', 50, 40) { + def set(t: LocalDateTime, v: Int): LocalDateTime = + t.withHour(t.getHour % 12 + 12 * check(v, 0, 1)) + def floor(t: LocalDateTime): LocalDateTime = + t.truncatedTo(ChronoUnit.DAYS).withHour(if (t.getHour >= 12) 12 else 0) + } + + private case object HourOfHalfDay extends Field('K', 40, 30) { + def set(t: LocalDateTime, v: Int): LocalDateTime = + t.withHour((t.getHour / 12) * 12 + check(v, 0, 11)) + def floor(t: LocalDateTime): LocalDateTime = t.truncatedTo(ChronoUnit.HOURS) + } + + private case object ClockHourOfHalfDay extends Field('h', 40, 30) { + def set(t: LocalDateTime, v: Int): LocalDateTime = HourOfHalfDay.set(t, check(v, 1, 12) % 12) + def floor(t: LocalDateTime): LocalDateTime = t.truncatedTo(ChronoUnit.HOURS) + } + + private case object HourOfDay extends Field('H', 50, 30) { + def set(t: LocalDateTime, v: Int): LocalDateTime = t.withHour(check(v, 0, 23)) + def floor(t: LocalDateTime): LocalDateTime = t.truncatedTo(ChronoUnit.HOURS) + } + + private case object ClockHourOfDay extends Field('k', 50, 30) { + def set(t: LocalDateTime, v: Int): LocalDateTime = t.withHour(check(v, 1, 24) % 24) + def floor(t: LocalDateTime): LocalDateTime = t.truncatedTo(ChronoUnit.HOURS) + } + + private case object MinuteOfHour extends Field('m', 30, 20) { + def set(t: LocalDateTime, v: Int): LocalDateTime = t.withMinute(check(v, 0, 59)) + def floor(t: LocalDateTime): LocalDateTime = t.truncatedTo(ChronoUnit.MINUTES) + } + + private case object SecondOfMinute extends Field('s', 20, 10) { + def set(t: LocalDateTime, v: Int): LocalDateTime = t.withSecond(check(v, 0, 59)) + def floor(t: LocalDateTime): LocalDateTime = t.truncatedTo(ChronoUnit.SECONDS) + } + + private case object MillisOfSecond extends Field('S', 10, 0) { + def set(t: LocalDateTime, v: Int): LocalDateTime = t.withNano(check(v, 0, 999) * 1000000) + def floor(t: LocalDateTime): LocalDateTime = t + } + + /** The local date-time the values give (MEASURED): see the object's documentation. */ + private def resolve(saved: Seq[(Field, Int)]): LocalDateTime = { + def order(fields: Seq[(Field, Int)]) = fields.sortBy { case (f, _) => (-f.range, -f.unit) } + val sorted0 = order(saved) + val sorted = + if (sorted0.nonEmpty && sorted0.head._1.unit >= 50 && sorted0.head._1.unit <= 70) + order(sorted0 :+ (Year -> 1970)) + else sorted0 + var t = LocalDateTime.of(1970, 1, 1, 0, 0) + sorted.foreach { case (f, v) => t = f.floor(f.set(t, v)) } + sorted.zipWithIndex.foreach { case ((f, v), i) => + t = f.set(t, v) + if (i == sorted.size - 1) t = f.floor(t) + } + t + } + + // --------------------------------------------------------------------------------------------- + // The pattern + // --------------------------------------------------------------------------------------------- + + private sealed trait Element + private final case class Literal(text: String) extends Element + private final case class Number(field: Field, maxDigits: Int, signed: Boolean) extends Element + private final case class TwoDigits(field: Field, lenient: Boolean) extends Element + private final case class Fraction(maxDigits: Int) extends Element + private final case class Text(field: Field, texts: Map[String, Int]) extends Element + private case object Offset extends Element + private case object RegionId extends Element + private case object ZoneName extends Element + + private val Months = Seq( + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december" + ) + private val MonthTexts: Map[String, Int] = + Months.zipWithIndex.flatMap { case (m, i) => Seq(m -> (i + 1), m.take(3) -> (i + 1)) }.toMap + private val DayTexts: Map[String, Int] = + Seq( + "monday", + "tuesday", + "wednesday", + "thursday", + "friday", + "saturday", + "sunday" + ).zipWithIndex.flatMap { case (d, i) => Seq(d -> (i + 1), d.take(3) -> (i + 1)) }.toMap + private val EraTexts: Map[String, Int] = Map("bc" -> 0, "ad" -> 1) + private val HalfDayTexts: Map[String, Int] = Map("am" -> 0, "pm" -> 1) + + /** The zone names Elasticsearch 6 reads with `z`, `zz`, `zzz` (MEASURED, winter and summer: every + * other name is rejected): core reads `UT`, `UTC` and `GMT`, UTC; the North American names take + * their offset from zone data, and are not read. + */ + private val ZoneNames: Seq[String] = + Seq("UT", "UTC", "GMT", "EST", "EDT", "CST", "CDT", "MST", "MDT", "PST", "PDT") + + private val FixedNames: Set[String] = Set("UT", "UTC", "GMT") + + /** A text's values by the elements, its offset in seconds, and why core does not read its zone, + * if it does not. + */ + private final case class Lexed( + saved: Seq[(Field, Int)], + offsetSeconds: Int, + unread: Option[String] + ) + + /** The widths of each letter whose values were read back on 6.8.23. */ + private val Widths: Map[Char, Set[Int]] = Map( + 'G' -> (1 to 5).toSet, + 'C' -> (1 to 4).toSet, + 'Y' -> (1 to 5).toSet, + 'x' -> (1 to 5).toSet, + 'w' -> (1 to 2).toSet, + 'e' -> (1 to 5).toSet, + 'E' -> (1 to 5).toSet, + 'y' -> (1 to 5).toSet, + 'D' -> (1 to 3).toSet, + 'M' -> (1 to 5).toSet, + 'd' -> (1 to 3).toSet, + 'a' -> Set(1, 2, 3), + 'K' -> (1 to 2).toSet, + 'h' -> (1 to 2).toSet, + 'H' -> (1 to 2).toSet, + 'k' -> (1 to 2).toSet, + 'm' -> (1 to 2).toSet, + 's' -> (1 to 2).toSet, + 'S' -> (1 to 9).toSet, + 'z' -> (1 to 3).toSet, + 'Z' -> (1 to 5).toSet + ) + + private val Numeric = "CYxwyDdKhHkmsSe" + + /** The elements of a Joda pattern, or why core does not read it. */ + private def elements(pattern: String): Either[String, Vector[Element]] = { + sealed trait Token + final case class Letters(c: Char, n: Int) extends Token + final case class Quoted(text: String) extends Token + val tokens = Vector.newBuilder[Token] + var i = 0 + var error: Option[String] = None + while (error.isEmpty && i < pattern.length) { + val c = pattern.charAt(i) + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')) { + var j = i + while (j < pattern.length && pattern.charAt(j) == c) j += 1 + tokens += Letters(c, j - i) + i = j + } else if (c == '\'') { + val sb = new StringBuilder + var j = i + 1 + var closed = false + while (!closed && j < pattern.length) { + if (pattern.charAt(j) == '\'') { + if (j + 1 < pattern.length && pattern.charAt(j + 1) == '\'') { + sb.append('\''); j += 2 + } else { closed = true; j += 1 } + } else { sb.append(pattern.charAt(j)); j += 1 } + } + if (!closed) error = Some("an unclosed quote") + else tokens += Quoted(if (j == i + 2) "'" else sb.toString) + i = j + } else { tokens += Quoted(c.toString); i += 1 } + } + val ts = tokens.result() + def numericNext(k: Int): Boolean = + k + 1 < ts.size && (ts(k + 1) match { + case Letters(c, n) => Numeric.indexOf(c.toInt) >= 0 || (c == 'M' && n <= 2) + case _ => false + }) + error.toLeft(()).flatMap { _ => + val out = Vector.newBuilder[Element] + var failure: Option[String] = None + ts.indices.foreach { k => + ts(k) match { + case Quoted(t) => out += Literal(t) + case Letters(c, n) if !Widths.get(c).exists(_.contains(n)) => + failure = failure.orElse( + Some(s"the letter '${String.valueOf(c) * n}' is not one core has measured") + ) + case Letters(c, n) => + val year: Option[Field] = c match { + case 'y' => Some(Year) + case 'Y' => Some(YearOfEra) + case 'x' => Some(WeekYear) + case _ => None + } + def upTo(default: Int) = Math.max(default, n) + out += (c match { + // `YY` is the year, as `yy` (MEASURED: `-2024` and `0` are read) + case 'y' | 'Y' if n == 2 => TwoDigits(Year, lenient = !numericNext(k)) + case 'x' if n == 2 => TwoDigits(WeekYear, lenient = !numericNext(k)) + case 'y' | 'Y' | 'x' => + Number(year.get, if (numericNext(k)) n else 9, signed = c != 'Y') + case 'C' => Number(CenturyOfEra, n, signed = false) + case 'G' => Text(Era, EraTexts) + case 'w' => Number(WeekOfWeekYear, upTo(2), signed = false) + case 'e' => Number(DayOfWeekField, n, signed = false) + case 'E' => Text(DayOfWeekField, DayTexts) + case 'D' => Number(DayOfYear, upTo(3), signed = false) + case 'M' if n >= 3 => Text(MonthOfYear, MonthTexts) + case 'M' => Number(MonthOfYear, 2, signed = false) + case 'd' => Number(DayOfMonth, upTo(2), signed = false) + case 'a' => Text(HalfDay, HalfDayTexts) + case 'K' => Number(HourOfHalfDay, upTo(2), signed = false) + case 'h' => Number(ClockHourOfHalfDay, upTo(2), signed = false) + case 'H' => Number(HourOfDay, upTo(2), signed = false) + case 'k' => Number(ClockHourOfDay, upTo(2), signed = false) + case 'm' => Number(MinuteOfHour, upTo(2), signed = false) + case 's' => Number(SecondOfMinute, upTo(2), signed = false) + case 'S' => Fraction(n) + case 'Z' if n <= 2 => Offset + case 'Z' => RegionId + case _ => ZoneName + }) + } + } + failure.toLeft(out.result()) + } + } + + /** The class a field stands for in a shape. */ + private def classOf(e: Element): Option[Char] = + e match { + case Number(f, _, _) => Some(f.letter) + case TwoDigits(f, _) => Some(f.letter) + case Fraction(_) => Some('S') + case Text(DayOfWeekField, _) => Some('E') + case Text(f, _) => Some(f.letter) + case _ => None + } + + /** The classes of date letters read in combination -- each with every class of time letters + * below, and every zone (MEASURED on 6.8.23): `G` era, `C` century of era, `Y` year of era, `y` + * year, `x` week year, `M` month, `w` week, `D` day of year, `d` day of month, `e` day of week, + * `E` its name. + */ + private val DateShapes: Set[String] = Set( + "", + "C", + "D", + "E", + "G", + "M", + "Y", + "d", + "e", + "w", + "x", + "y", + "Cy", + "Gy", + "Md", + "YD", + "YM", + "Ye", + "Yw", + "we", + "xD", + "xM", + "xe", + "xw", + "yD", + "yE", + "yM", + "ye", + "yw", + "MdE", + "Mde", + "YMd", + "YwE", + "Ywe", + "xMd", + "xwE", + "xwe", + "yDe", + "yMD", + "yME", + "yMd", + "yMe", + "ywE", + "ywe", + "CyMd", + "GYMd", + "GYwe", + "Gxwe", + "GyMD", + "GyMd", + "YMde", + "YyMd", + "xMde", + "yMDd", + "yMdE", + "yMde", + "yMwd", + "yxMd", + "GyMdE", + "GyMde" + ) + + /** The classes of time letters read in combination (MEASURED): `a` half day, `K` `h` its hours, + * `H` `k` the hours of the day, `m`, `s`, `S` fraction. + */ + private val TimeShapes: Set[String] = Set( + "", + "H", + "K", + "S", + "a", + "h", + "k", + "m", + "s", + "Hm", + "Km", + "ah", + "hm", + "km", + "sS", + "Hms", + "aHm", + "aKm", + "ahm", + "akm", + "HmsS", + "ahms" + ) + + private val DateOrder = "GCYyxMwDdeE" + private val TimeOrder = "aKhHkmsS" + + /** A custom Joda pattern of Elasticsearch 6. */ + final case class Pattern(pattern: String, clock: Clock) extends Alternative { + override def isName: Boolean = false + + private val compiled: Either[String, (Vector[Element], String)] = + elements(pattern).flatMap { es => + val classes = es.flatMap(classOf).toSet + val date = DateOrder.filter(classes.contains) + val time = TimeOrder.filter(classes.contains) + val zone = es + .collectFirst { + case Offset => "offset" + case RegionId => "id" + case ZoneName => "name" + } + .getOrElse("") + val shape = s"$date|$time|$zone" + // two zone letters (`ZZ ZZZ`, `ZZ z`): a combination core has not measured + val zones = es.count { + case Offset | RegionId | ZoneName => true + case _ => false + } + if (zones > 1) + Left(s"its letters combine two zones, which core has not measured") + else if (DateShapes.contains(date) && TimeShapes.contains(time)) Right((es, zone)) + else + Left(s"its letters combine as '$shape', which core has not measured") + } + + /** Why the pattern is not read, if it is not. */ + def unread: Option[String] = compiled.left.toOption + + def parse(text: String): Parse = + compiled match { + case Left(reason) => Unmeasured(s"'$pattern' is not a date format core reads: $reason") + case Right((es, _)) => + lex(es, text) match { + case None => NotParsed + case Some(Lexed(_, _, Some(reason))) => Unmeasured(reason) + case Some(Lexed(saved, offsetSeconds, None)) => + Try { + resolve(saved).toInstant(ZoneOffset.UTC).minusSeconds(offsetSeconds.toLong) + }.fold(_ => Invalid, Parsed) + } + } + + /** The values of `text` by the elements, read left to right; `None` when it does not match. */ + private def lex(es: Vector[Element], text: String): Option[Lexed] = { + var pos = 0 + val saved = scala.collection.mutable.ArrayBuffer.empty[(Field, Int)] + var offset = 0 + var unread: Option[String] = None + var ok = true + def digits(from: Int, max: Int): Int = { + var j = from + while (j < text.length && j - from < max && Character.isDigit(text.charAt(j))) j += 1 + j - from + } + val it = es.iterator + while (ok && it.hasNext) { + it.next() match { + case Literal(t) => + if (text.regionMatches(true, pos, t, 0, t.length)) pos += t.length else ok = false + case Number(f, max, signed) => + val sign = + if ( + signed && pos < text.length && (text.charAt(pos) == '-' || text.charAt(pos) == '+') + ) + 1 + else 0 + val n = digits(pos + sign, max) + if (n == 0) ok = false + else { + saved += (f -> Integer.parseInt(text.substring(pos, pos + sign + n).replace("+", ""))) + pos += sign + n + } + case TwoDigits(f, lenient) => + val sign = + if ( + lenient && pos < text.length && (text.charAt(pos) == '-' || text.charAt(pos) == '+') + ) + 1 + else 0 + val n = digits(pos + sign, if (lenient) 9 else 2) + if (n == 0) ok = false + else { + val t = text.substring(pos, pos + sign + n) + saved += (f -> (if (sign == 0 && n == 2) twoDigitYear(t, clock) + else Integer.parseInt(t.replace("+", "")))) + pos += sign + n + } + case Fraction(max) => + val n = digits(pos, max) + if (n == 0) ok = false + else { + saved += (MillisOfSecond -> Integer.parseInt( + (text.substring(pos, pos + n) + "00").take(3) + )) + pos += n + } + case Text(f, texts) => + // the longest name that matches, in any case + val rest = text.substring(pos).toLowerCase(Locale.ROOT) + texts.keys.filter(rest.startsWith).toSeq.sortBy(-_.length).headOption match { + case Some(name) => saved += (f -> texts(name)); pos += name.length + case None => ok = false + } + case Offset => + OffsetText.findPrefixMatchOf(text.substring(pos)) match { + case Some(m) => + offset = offsetSeconds(m.matched) + pos += m.matched.length + case None => ok = false + } + case RegionId => + // `UTC` and `GMT` (MEASURED: `UT` is rejected); any other id takes its offset from + // zone data, not read + RegionText.findPrefixMatchOf(text.substring(pos)) match { + case Some(m) => + if (m.matched != "UTC" && m.matched != "GMT") + unread = unread.orElse(Some(zoneNotRead(m.matched))) + pos += m.matched.length + case None => ok = false + } + case ZoneName => + ZoneNames + .filter(text.startsWith(_, pos)) + .sortBy(-_.length) + .headOption match { + case Some(name) => + if (!FixedNames.contains(name)) unread = unread.orElse(Some(zoneNotRead(name))) + pos += name.length + case None => ok = false + } + } + } + if (ok && pos == text.length) Some(Lexed(saved.toList, offset, unread)) else None + } + } + + private val OffsetText = "[Zz]|[+-]\\d{2}(?::?\\d{2}(?::?\\d{2})?)?".r + private val RegionText = "[A-Za-z][A-Za-z0-9_/+-]*".r + + private def offsetSeconds(t: String): Int = + if (t == "Z" || t == "z") 0 + else { + val sign = if (t.charAt(0) == '-') -1 else 1 + val d = t.substring(1).replace(":", "") + def part(i: Int) = if (d.length >= i + 2) Integer.parseInt(d.substring(i, i + 2)) else 0 + sign * (part(0) * 3600 + part(2) * 60 + part(4)) + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappedDateFormat.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappedDateFormat.scala new file mode 100644 index 00000000..8335d7fd --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappedDateFormat.scala @@ -0,0 +1,1185 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import java.time.format.DateTimeFormatter +import java.time.temporal.{ChronoField, IsoFields} +import java.time.{ + Clock, + Instant, + LocalDate, + LocalDateTime, + LocalTime, + OffsetDateTime, + Year, + ZoneId, + ZoneOffset, + ZonedDateTime +} +import java.util.Locale +import java.util.regex.{Matcher, Pattern => Regex} +import scala.util.Try + +/** A date field's mapping `format`, read as ONE Elasticsearch major reads it: the instant a value + * of the field denotes -- the one that major INDEXED, which its queries and aggregations see, at a + * millisecond's precision. The `||` alternatives are tried in order, and the FIRST one that parses + * a value decides: its instant, or, when the date it parses is not a valid one (a month 31), the + * value's rejection, the next alternative never tried -- as Elasticsearch does. + * + * Elasticsearch 6 parses with Joda-Time, Elasticsearch 7 and 8 with `java.time`, and the same + * mapping reads differently across them ([[MappedDateFormat.Dialect]]). Every reading is MEASURED, + * major by major -- every format of each major's documentation, every pattern letter at each + * width, alone and combined, epochs and JSON numbers, written to a field of the format one value + * at a time and the instant read back from its doc value, on Elasticsearch 6.8.23, 7.17.29 and + * 8.18.3 + * -- and written from that behaviour and Java's own API, never from Elasticsearch's or Joda's + * code: + * + * - `epoch_millis`, `epoch_second` ([[MappedDateFormat.Epoch]]): an integer, a fraction rounded + * down to the millisecond, each major's rules for a negative one; + * - the built-in names, as grammars of their digits, separators, fractions and zones, per major; + * - a custom pattern: a `java.time` pattern on Elasticsearch 7 and 8, and on 6.8 behind a + * leading `8` ([[JavaDatePattern]]), a Joda pattern on Elasticsearch 6 ([[JodaDatePattern]]); + * - a field with no `format`: Elasticsearch's default, + * `strict_date_optional_time||epoch_millis`, which reads `"2024"` and `2024` as the year 2024, + * not 2024 milliseconds. + * + * A value is read only where its reading is the same on every Java version core runs on (8, 11, + * 17, 21) and equal to the instant Elasticsearch indexed. A name, a pattern letter, a width or a + * combination of letters no measurement covers, a zone other than `Z`, an offset, `UTC`, `GMT` and + * `UT` (a region id or a zone name, whose offset comes from zone data -- the JVM core runs on's, + * not Elasticsearch's), a reading that depends on the Java version core runs on, and a value core + * cannot read without guessing -- a spelling or an offset the response parser no longer gives, a + * JSON number whose digits its `double` no longer holds -- are NOT guessed: such a value is + * [[MappedDateFormat.Undecidable]], and the statement reading it fails, naming the field and the + * format. + */ +private[client] final case class MappedDateFormat( + spec: String, + dialect: MappedDateFormat.Dialect, + alternatives: Seq[MappedDateFormat.Alternative] +) { + import MappedDateFormat._ + + /** Every alternative is a name core reads: each reads an ISO text by its ISO fields, so a value + * the response parser already read as an ISO date denotes the same instant whatever the way it + * was spelled, and is read as that instant. + */ + private val namesOnly: Boolean = alternatives.forall(_.isName) + + /** The instant `text` denotes, by the first alternative that parses it. */ + def read(text: String): Outcome = { + val it = alternatives.iterator + var outcome: Outcome = Rejected + var done = false + while (!done && it.hasNext) + it.next().parse(text) match { + case NotParsed => () + case Parsed(instant) => + outcome = Try(Instant.ofEpochMilli(instant.toEpochMilli)).toOption.fold[Outcome]( + Rejected + )(Read) + done = true + case Invalid => done = true + case Unmeasured(reason) => + outcome = Undecidable(reason) + done = true + } + outcome + } + + /** The instant a value of the field denotes, whatever the class the response parser gave it: a + * text as written; a number as its digits; a value it already read as an ISO date or time -- + * which no longer says how it was spelled -- by the spellings it may have had: the first one the + * format reads when every alternative is a name core reads (they read every spelling as the same + * instant), else all of them, which must agree. + */ + def readValue(value: Any): Outcome = + value match { + case s: String => read(s) + case b: java.math.BigDecimal => read(b.toPlainString) + case d: java.lang.Double => byDigits(d.doubleValue(), Math.ulp(d.doubleValue())) + case f: java.lang.Float => byDigits(f.doubleValue(), Math.ulp(f.floatValue()).toDouble) + case n: java.lang.Number => read(n.toString) + // a text with a region (`2024-07-31T10:30:00+05:00[Europe/Paris]`), which the response + // parser read with the zone data of the JVM core runs on -- its written local time and + // offset no longer known (Elasticsearch 8 reads the local time in the region, MEASURED) + case z: ZonedDateTime if !z.getZone.isInstanceOf[ZoneOffset] => + Undecidable(zoneNotRead(z.getZone.getId)) + case _: LocalDate | _: LocalDateTime | _: ZonedDateTime | _: OffsetDateTime | _: LocalTime => + if (namesOnly && !value.isInstanceOf[LocalTime]) firstRead(value) + else bySpellings(value) + case i: Instant => Read(millis(i)) + case other => read(other.toString) + } + + /** A fractional JSON number, which the response parser read as a `double`: Elasticsearch read the + * DIGITS written in the document (MEASURED: `1706697000.5` is 1706697000 milliseconds in an + * `epoch_millis` field, 1706697000500 in an `epoch_second` one, on 6.8.23, 7.17.29 and 8.18.3), + * which the `double` no longer says. Every number written as digits that read as this `double` + * lies within half its unit in the last place: the value is read when every such number its + * format reads gives the same instant -- else it is undecidable (`1706697000000.0` may have been + * written `1706696999999.9999`, a millisecond earlier). + */ + private def byDigits(value: Double, ulp: Double): Outcome = + if (java.lang.Double.isNaN(value) || java.lang.Double.isInfinite(value)) Rejected + else { + val exact = new java.math.BigDecimal(value) + val half = new java.math.BigDecimal(ulp).divide(java.math.BigDecimal.valueOf(2L)) + val low = exact.subtract(half) + val high = exact.add(half) + // every number of 0 to 9 fraction digits that reads as this double: the lowest and the + // highest of each width (a value read with more digits than its format takes is rejected), + // the shortest digits and both ends + val candidates = (0 to 9).flatMap { digits => + Seq( + low.setScale(digits, java.math.RoundingMode.CEILING), + high.setScale(digits, java.math.RoundingMode.FLOOR) + ).filter(d => d.compareTo(low) >= 0 && d.compareTo(high) <= 0) + } ++ Seq(java.math.BigDecimal.valueOf(value).stripTrailingZeros, low, high) + val outcomes = candidates.map(d => read(d.toPlainString)) + outcomes.collect { case Read(i) => i }.distinct match { + case Seq(one) if !outcomes.exists(_.isInstanceOf[Undecidable]) => Read(one) + case Seq() if !outcomes.exists(_.isInstanceOf[Undecidable]) => Rejected + case _ => + Undecidable( + "a fractional JSON number: the response gives it as a double, whose digits read " + + "differently within its precision" + ) + } + } + + /** A value the response parser read as an ISO date, at a format of names only: its first spelling + * the format reads -- the one `java.time` prints, almost always -- gives its instant. None read: + * no alternative reads the value, which Elasticsearch could not have indexed. + */ + private def firstRead(value: Any): Outcome = { + // PER ROW on the extraction path: whether the format reads a value depends only on its SHAPE + // (the digits of its year, whether it has a time, seconds, a fraction of how many digits, an + // offset), so the spellings are read once per shape, and every other value of that shape is its + // own instant + val key = shapeOf(value) + readsShape.get(key) match { + case java.lang.Boolean.TRUE => + Try(millis(instantOf(value))).toOption.fold[Outcome](Rejected)(Read) + case java.lang.Boolean.FALSE => Rejected + case _ => + val it = spellings(value) + var outcome: Outcome = Rejected + while (outcome == Rejected && it.hasNext) + outcome = read(it.next()) match { + case r: Read => r + case _ => Rejected + } + readsShape.put(key, java.lang.Boolean.valueOf(outcome != Rejected)) + outcome + } + } + + /** Whether the format reads a value of a shape ([[shapeOf]]), as it is first met. */ + private val readsShape = new java.util.concurrent.ConcurrentHashMap[Long, java.lang.Boolean]() + + /** A value the response parser read as an ISO date, read back by each ISO spelling it may have + * had ([[spellings]]): they must agree. A date or a time whose text may have carried an offset + * the response parser dropped (`2024-01-31+02:00` is read as the date alone) is undecidable + * where an alternative reads such an offset. + */ + private def bySpellings(value: Any): Outcome = { + val texts = spellings(value).toList + val lostOffset = value match { + case d: LocalDate => + val t = DateTimeFormatter.ISO_LOCAL_DATE.format(d) + Seq(t + "Z", t + "+01:00").exists(parses) + case t: LocalTime => texts.exists(s => Seq(s + "Z", s + "+01:00").exists(parses)) + case _ => false + } + if (lostOffset) + Undecidable( + "the response gives it without the offset it may have been written with, which the " + + "format reads" + ) + else { + val outcomes = texts.map(read) + outcomes.collectFirst { case u: Undecidable => u } match { + case Some(undecidable) => undecidable + case None => + outcomes.collect { case Read(instant) => instant }.distinct match { + case Seq(instant) => Read(instant) + case Seq() => Rejected + case _ => + Undecidable( + "the format reads it differently depending on how it was spelled, which the " + + "response no longer says" + ) + } + } + } + } + + private def parses(text: String): Boolean = + alternatives.exists(_.parse(text) != NotParsed) +} + +private[client] object MappedDateFormat { + + /** How one Elasticsearch major reads a date: ES 6 with Joda-Time, ES 7 and ES 8 with `java.time`, + * with their own week rules, texts and epochs (MEASURED on 6.8.23, 7.17.29 and 8.18.3). The + * other minor versions of a major read as its measured one, and Elasticsearch 9 as Elasticsearch + * 8 (ASSUMED: measured by the CI's legs, not locally). + */ + sealed abstract class Dialect(val major: Int) { + def label: String = s"Elasticsearch $major" + } + + object Dialect { + case object Joda extends Dialect(6) + case object Java7 extends Dialect(7) + case object Java8 extends Dialect(8) + + /** The dialect of an Elasticsearch `version` (`VersionApi.version`), by the predicates of + * [[ElasticsearchVersion]]; `None` for a version before 6 or not a version. + */ + def of(version: String): Option[Dialect] = + Try { + if (ElasticsearchVersion.isEs6(version)) Some(Joda) + else if (ElasticsearchVersion.isEs7(version)) Some(Java7) + else if (ElasticsearchVersion.isEs8OrHigher(version)) Some(Java8) + else None + }.toOption.flatten + } + + /** What a format makes of a value. */ + sealed trait Outcome + + /** The instant, at a millisecond's precision. */ + final case class Read(instant: Instant) extends Outcome + + /** No alternative reads it, or the first that parses it finds no valid date in it. */ + case object Rejected extends Outcome + + /** Core cannot tell the instant without guessing: `reason` says why. */ + final case class Undecidable(reason: String) extends Outcome + + /** What ONE alternative makes of a text. */ + sealed trait Parse + case object NotParsed extends Parse + case object Invalid extends Parse + final case class Parsed(instant: Instant) extends Parse + final case class Unmeasured(reason: String) extends Parse + + trait Alternative { + def parse(text: String): Parse + + /** A name core reads, as opposed to a custom pattern or an unknown name. */ + def isName: Boolean = true + } + + /** A name or a pattern core does not read on this major: a value it decides is undecidable. */ + final case class Unknown(name: String, dialect: Dialect, reason: String = "") + extends Alternative { + def parse(text: String): Parse = + Unmeasured( + s"'$name' is not a date format core reads as ${dialect.label} reads it" + + (if (reason.isEmpty) "" else s": $reason") + ) + override def isName: Boolean = false + } + + /** `epoch_millis` (`unitMillis` 1) or `epoch_second` (`unitMillis` 1000). */ + final case class Epoch(name: String, unitMillis: Long, dialect: Dialect) extends Alternative { + private val shape: Regex = dialect match { + // Joda: a sign, a fraction and an exponent, the value truncated toward zero to the unit + case Dialect.Joda => Regex.compile("[+-]?\\d+(?:\\.\\d+)?(?:[eE][+-]?\\d+)?") + // Elasticsearch 7 and 8: a fraction of 6 digits at most in `epoch_millis`, 9 in `epoch_second` + // (MEASURED) + case _ if unitMillis == 1L => Regex.compile("-?\\d++(?:\\.\\d{1,6}+)?") + case _ => Regex.compile("-?\\d++(?:\\.\\d{1,9}+)?") + } + + def parse(text: String): Parse = + if (!shape.matcher(text).matches()) NotParsed + else if (dialect == Dialect.Joda) + // a value past the instants Elasticsearch 6 holds is not this alternative's (MEASURED: + // `epoch_second||epoch_millis` reads 9223372036854775807 as milliseconds) + Try { + val value = new java.math.BigDecimal(text) + Instant.ofEpochMilli( + java.lang.Math.multiplyExact( + value.setScale(0, java.math.RoundingMode.DOWN).longValueExact(), + unitMillis + ) + ) + }.fold(_ => NotParsed, Parsed) + else if (text.charAt(0) == '-') negative(this, text) + else if (text.indexOf('.') < 0) + // an integer, the common case: no decimal arithmetic + Try( + Instant.ofEpochMilli( + java.lang.Math.multiplyExact(java.lang.Long.parseLong(text), unitMillis) + ) + ).fold(_ => Invalid, Parsed) + else + Try( + Instant.ofEpochMilli( + new java.math.BigDecimal(text) + .multiply(java.math.BigDecimal.valueOf(unitMillis)) + .setScale(0, java.math.RoundingMode.FLOOR) + .longValueExact() + ) + ).fold(_ => Invalid, Parsed) + } + + /** A negative epoch, as each major reads it (MEASURED, every fraction width): + * - Elasticsearch 8 rounds it down to the millisecond (`-1.5` milliseconds is -2, `-1.5` + * seconds -1500 milliseconds), and rejects a negative `epoch_second` whose fraction is zero + * (`-1.0`); + * - Elasticsearch 7 reads a negative `epoch_millis` only when its integer part is a multiple + * of 1000 other than 0, as that integer part (`-1000.5` is -1000; `-86400` and `-1` are + * rejected), and a negative `epoch_second` as its integer part PLUS its fraction, rounded + * down to the millisecond (`-1.5` is -500 milliseconds), an integer part 0 rejected. + */ + private def negative(epoch: Epoch, text: String): Parse = { + val point = text.indexOf('.') + val whole = new java.math.BigDecimal(if (point < 0) text else text.substring(0, point)) + val fraction = + if (point < 0) java.math.BigDecimal.ZERO + else new java.math.BigDecimal("0" + text.substring(point)) + val unit = java.math.BigDecimal.valueOf(epoch.unitMillis) + def floor(d: java.math.BigDecimal): Parse = + Try( + Instant.ofEpochMilli(d.setScale(0, java.math.RoundingMode.FLOOR).longValueExact()) + ).fold(_ => Invalid, Parsed) + epoch.dialect match { + case Dialect.Java7 if whole.signum == 0 => NotParsed + case Dialect.Java7 if epoch.unitMillis == 1L => + if (whole.remainder(java.math.BigDecimal.valueOf(1000L)).signum != 0) NotParsed + else floor(whole) + case Dialect.Java7 => floor(whole.add(fraction).multiply(unit)) + // Elasticsearch 8 rejects a negative `epoch_second` whose fraction is zero (`-1.0`, `-0.0`) + case _ if epoch.unitMillis == 1000L && point >= 0 && fraction.signum == 0 => NotParsed + case _ => floor(new java.math.BigDecimal(text).multiply(unit)) + } + } + + /** How a parsed date is validated: STRICT (`2024-02-30` is no date) or LENIENT (`2024-02-30` is + * `2024-03-01`, Elasticsearch 8's `strict_date`). + */ + private[client] sealed trait Days + private[client] case object StrictDays extends Days + private[client] case object LenientDays extends Days + + /** A day past the month's last is its last (`2024-02-30` is `2024-02-29`: Elasticsearch 7's + * `strict_date_hour_minute_second`, MEASURED). + */ + private[client] case object LastDays extends Days + + private val GroupName = Regex.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>") + + /** The groups a grammar's regex declares. */ + private def groupsOf(regex: Regex): Set[String] = { + val m = GroupName.matcher(regex.pattern()) + var names = Set.empty[String] + while (m.find()) names += m.group(1) + names + } + + /** A text's fields by a grammar: `y`, `mo`, `d`, `doy` (a day of year), `wy`, `ww`, `wd` (an ISO + * week date), `h`, `mi`, `s`, `f` (a fraction's digits) and `z` (a zone), `oy` the year of an + * ordinal date in an alternation; a field the grammar does not hold takes January, the first + * day, midnight, UTC, and a time with no date 1970-01-01 -- as each name was MEASURED to. + */ + private[client] final case class Grammar( + name: String, + regex: Regex, + zone: String => ZoneRead, + days: Days = StrictDays, + endOfDay: Boolean = false, + override val isName: Boolean = true, + timeDropsYear: Boolean = false, + lastDayAt24: Boolean = false + ) extends Alternative { + private val groups = groupsOf(regex) + + private def group(m: Matcher, g: String): Option[String] = + if (groups.contains(g)) Option(m.group(g)).filter(_.nonEmpty) else None + + def parse(text: String): Parse = { + val m = regex.matcher(text) + if (!m.matches()) NotParsed + else + group(m, "z").fold[ZoneRead](Some(Right(ZoneOffset.UTC)))(zone) match { + // a zone the alternative does not read: the text is not this alternative's + case None => NotParsed + case Some(Left(reason)) => Unmeasured(reason) + case Some(Right(zoneId)) => + Try { + // Elasticsearch 7 and 8 read a year with no month, day of year or week, and an hour, + // as a time on 1970-01-01 (MEASURED) + val yearAlone = timeDropsYear && group(m, "h").isDefined && + Seq("mo", "doy", "wy").forall(group(m, _).isEmpty) + // `oy`: the year of an ordinal date in an alternation (a group name is used once) + val year = + if (yearAlone) 1970 + else group(m, "y").orElse(group(m, "oy")).fold(1970)(y => Integer.parseInt(y)) + val month = group(m, "mo").fold(1)(Integer.parseInt) + val day = group(m, "d").fold(1)(Integer.parseInt) + val at24 = group(m, "h").contains("24") && + Seq("mi", "s", "f").forall(g => group(m, g).forall(_.forall(_ == '0'))) + val date = (group(m, "wy"), group(m, "doy")) match { + // an ISO week date: the week (1 when none) of the week-based year, its day (Monday + // when none), the week a week of THAT year + case (Some(wy), _) => + val first = LocalDate.of(Integer.parseInt(wy), 1, 4) + val week = group(m, "ww").fold(1L)(java.lang.Long.parseLong) + IsoFields.WEEK_OF_WEEK_BASED_YEAR + .rangeRefinedBy(first) + .checkValidValue(week, IsoFields.WEEK_OF_WEEK_BASED_YEAR) + first + .`with`(IsoFields.WEEK_OF_WEEK_BASED_YEAR, week) + .`with`( + ChronoField.DAY_OF_WEEK, + group(m, "wd").fold(1L)(java.lang.Long.parseLong) + ) + // an ordinal date + case (None, Some(doy)) => LocalDate.ofYearDay(year, Integer.parseInt(doy)) + case _ => + (if (lastDayAt24 && at24) LastDays else days) match { + case StrictDays => LocalDate.of(year, month, day) + case LenientDays => + LocalDate.of(year, 1, 1).plusMonths(month - 1L).plusDays(day - 1L) + case LastDays => + val first = LocalDate.of(year, month, 1) + first.withDayOfMonth(Math.min(day, first.lengthOfMonth())) + } + } + val hour = group(m, "h").fold(0)(Integer.parseInt) + val minute = group(m, "mi").fold(0)(Integer.parseInt) + val second = group(m, "s").fold(0)(Integer.parseInt) + val nano = group(m, "f").fold(0)(f => Integer.parseInt((f + "00000000").take(9))) + val atEnd = endOfDay && hour == 24 && minute == 0 && second == 0 && nano == 0 + // 24:00 is the next day's midnight, the same day's when there is no date + val dated = Seq("y", "oy", "wy").exists(groups.contains) + ZonedDateTime + .of( + if (atEnd && dated) date.plusDays(1) else date, + if (atEnd) LocalTime.MIDNIGHT else LocalTime.of(hour, minute, second, nano), + zoneId + ) + .toInstant + }.fold(_ => Invalid, Parsed) + } + } + } + + /** Joda's two-digit year (MEASURED on 6.8.23, 2026-10-07: `00`..`45` are 2000..2045, `46`..`99` + * 1946..1999): the year of those digits within 50 years of the current year minus 30. Any other + * digit count, or a sign, is the year as written. + */ + private[client] def twoDigitYear(text: String, clock: Clock): Int = + if (text.length == 2 && text.forall(Character.isDigit)) { + val low = Year.now(clock).getValue - 30 - 50 + low + (((Integer.parseInt(text) - low) % 100) + 100) % 100 + } else Integer.parseInt(text) + + // --------------------------------------------------------------------------------------------- + // Zones + // --------------------------------------------------------------------------------------------- + + /** An offset: hours, hours and minutes, or hours, minutes and seconds, with colons all or none. + */ + private val OffsetText = "[+-]\\d{2}(?::\\d{2}(?::\\d{2})?|\\d{2}(?:\\d{2})?)?" + private val Offset = Regex.compile("([+-])(\\d{2})(?::?(\\d{2})(?::?(\\d{2}))?)?") + + private def offsetOf(text: String): Option[ZoneId] = { + val m = Offset.matcher(text) + if (!m.matches()) None + else + Try { + val sign = if (m.group(1) == "-") -1 else 1 + def part(i: Int) = Option(m.group(i)).fold(0)(Integer.parseInt) + ZoneOffset.ofHoursMinutesSeconds(sign * part(2), sign * part(3), sign * part(4)): ZoneId + }.toOption + } + + /** What an alternative makes of the text of a zone: `None`, no zone it reads (the text is not the + * alternative's); a zone; or why core does not read it (`Left`): the value is [[Unmeasured]]. + */ + private[client] type ZoneRead = Option[Either[String, ZoneId]] + + private def zoneRead(zone: Option[ZoneId]): ZoneRead = zone.map(Right(_)) + + /** The fixed ids, which every JVM and every Elasticsearch major read as UTC. */ + private val FixedIds: Set[String] = Set("UTC", "GMT", "UT") + + /** A fixed id followed by an offset, in one of `ZoneOffset.of`'s forms: a sign, then 1 or 2 + * digits of hours, or 2 with 2 of minutes and 2 of seconds, colons all or none (`GMT+1`, + * `UTC+01:00`). + */ + private val PrefixedOffset = Regex.compile( + "(?:UTC|GMT|UT)([+-])(?:(\\d{1,2}+)|(\\d{2}):(\\d{2})(?::(\\d{2}))?|(\\d{2})(\\d{2})(\\d{2})?)" + ) + + /** A fixed id (`UTC`, `GMT`, `UT`) and, where `prefixed`, a fixed id followed by an offset + * (`GMT+1`): a fixed offset, read here -- never through the zone data of the JVM core runs on. + * `None` for any other text. + */ + private[client] def fixedId(text: String, prefixed: Boolean): Option[ZoneOffset] = + if (FixedIds.contains(text)) Some(ZoneOffset.UTC) + else if (!prefixed) None + else { + val m = PrefixedOffset.matcher(text) + if (!m.matches()) None + else + Try { + val sign = if (m.group(1) == "-") -1 else 1 + def part(groups: Int*) = + groups.flatMap(g => Option(m.group(g))).headOption.fold(0)(Integer.parseInt) + ZoneOffset.ofHoursMinutesSeconds( + sign * part(2, 3, 6), + sign * part(4, 7), + sign * part(5, 8) + ) + }.toOption + } + + private val PrefixedOffsetId = Regex.compile("(?:UTC|GMT|UT)[+-].*") + + /** Why core does not read a zone id or a zone name other than `UTC`, `GMT`, `UT` and their + * offsets: its offset comes from zone data -- the JVM core runs on would give it, not the one + * Elasticsearch indexed it with (MEASURED: `America/Mexico_City`, `Asia/Tehran`, `Asia/Amman`, + * `America/Asuncion` and others read differently on 6.8.23, 8.18.3 and core's JVMs). + */ + private[client] def zoneNotRead(text: String): String = + s"the zone '$text' is a region id or a zone name, whose offset comes from the zone data of " + + "the JVM reading it, not Elasticsearch's: core reads only Z, an offset, UTC, GMT and UT" + + /** `Z` or an offset. */ + private def utcOrOffset(text: String): ZoneRead = + zoneRead(if (text == "Z") Some(ZoneOffset.UTC) else offsetOf(text)) + + /** Joda: `Z`, `z` or an offset. */ + private def jodaOffset(text: String): ZoneRead = + zoneRead(if (text == "Z" || text == "z") Some(ZoneOffset.UTC) else offsetOf(text)) + + /** `Z`, an offset, `UTC`, `GMT` or `UT`; an id holding an offset (`GMT+1`) is not read by the + * names using this (MEASURED: rejected), and any other id is not read by core ([[zoneNotRead]]). + */ + private def regionZone(text: String): ZoneRead = + utcOrOffset(text).orElse { + if (PrefixedOffsetId.matcher(text).matches()) None + else Some(fixedId(text, prefixed = false).toRight(zoneNotRead(text))) + } + + /** `Z`, an offset, `UTC`, `GMT`, `UT` and an id holding an offset (`GMT+1`, as `ZoneOffset.of` + * reads it, else not a zone); any other id is not read by core ([[zoneNotRead]]). + */ + private def anyZone(text: String): ZoneRead = + utcOrOffset(text).orElse { + if (PrefixedOffsetId.matcher(text).matches()) zoneRead(fixedId(text, prefixed = true)) + else Some(fixedId(text, prefixed = false).toRight(zoneNotRead(text))) + } + + private val ZoneIdText = "[A-Za-z][A-Za-z0-9_/+~-]*+" + + // --------------------------------------------------------------------------------------------- + // The names, per major (MEASURED) + // --------------------------------------------------------------------------------------------- + + private def g(text: String): Regex = Regex.compile(text) + + /** Elasticsearch 7 and 8 (`java.time`): every name of their documentation, as MEASURED on 7.17.29 + * and 8.18.3. `es8` is false for Elasticsearch 7, which reads no zone id holding an offset + * (`GMT+1`) where Elasticsearch 8 does, and reads a day past the month's last in + * `strict_date_hour_minute_second` as the month's last. + */ + private def javaNames(es8: Boolean): Map[String, Alternative] = { + // years: strict (4 digits, a sign), strict with 5 and more digits signed, any + val y4 = "(?-?\\d{4})" + val y4p = "(?-?\\d{4}|[+-]\\d{5,9}+)" + val y9 = "(?-?\\d{1,9}+)" + val m2 = "(?\\d{2})" + val m12 = "(?\\d{1,2}+)" + val d2 = "(?\\d{2})" + val d12 = "(?\\d{1,2}+)" + val h2 = "(?\\d{2})" + val h12 = "(?\\d{1,2}+)" + val mi2 = "(?\\d{2})" + val mi12 = "(?\\d{1,2}+)" + val s2 = "(?\\d{2})" + val s12 = "(?\\d{1,2}+)" + val f = "(?\\d{1,9}+)" + val f3 = "(?\\d{1,3}+)" + // a zone: Z, an offset or a zone id, read by `wideZone` (any id `java.time` reads on ES 8, a + // region on ES 7) + val zone = s"(?Z|$OffsetText|$ZoneIdText)" + val wideZone: String => ZoneRead = if (es8) anyZone else regionZone + // the strict week names also read a lower-case `z` (MEASURED, 8.18.3 and 7.17.29) + val zoneOrLowerZ = s"(?Z|z|$OffsetText|$ZoneIdText)" + val zOrUtc: String => ZoneRead = t => if (t == "z") Some(Right(ZoneOffset.UTC)) else wideZone(t) + // date_time: Z or an offset with no seconds + val shortOffset = "(?Z|[+-]\\d{2}(?::?\\d{2})?)" + // the date part of the strict and the lenient date-time names: the month and the day optional + val strictDay = s"$y4(?:-$m2(?:-$d2)?)?" + val anyDay = s"$y9(?:-$m12(?:-$d12)?)?" + val strictDate = s"$y4p-$m2-$d2" + val date = s"$y9-$m12-$d12" + val ordinal = s"$y4-(?\\d{1,3}+)" + val strictOrdinal = s"$y4-(?\\d{3})" + val basicTime = "(?\\d{2})(?\\d{2})(?\\d{2})" + def name( + n: String, + regex: String, + z: String => ZoneRead = utcOrOffset, + days: Days = StrictDays, + endOfDay: Boolean = false, + lastDayAt24: Boolean = false + ): (String, Alternative) = + n -> Grammar(n, g(regex), z, days, endOfDay, timeDropsYear = true, lastDayAt24 = lastDayAt24) + Map( + name( + "strict_date_optional_time", + s"$strictDay(?:T$h2(?::$mi2(?::$s2(?:[.,]$f)?)?)?$zone?)?", + wideZone + ), + name("iso8601", s"$strictDay(?:T$h2(?::$mi2(?::$s2(?:[.,]$f)?)?)?$zone?)?", wideZone), + name( + "strict_date_optional_time_nanos", + s"$strictDay(?:T$h2:$mi2:$s2(?:[.,]$f)?$zone?)?", + wideZone + ), + name( + "date_optional_time", + s"$anyDay(?:T$h12(?::$mi12(?::$s12(?:[.,]$f)?)?$zone?)?)?", + regionZone + ), + name("strict_date_time", s"${strictDay}T$h2:$mi2:$s2(?:\\.$f)?$zone", wideZone), + name("date_time", s"${anyDay}T$h12:$mi12(?::$s12\\.$f)?$shortOffset"), + name("strict_date_time_no_millis", s"${strictDay}T$h2:$mi2:$s2$zone", wideZone), + name("date_time_no_millis", s"${anyDay}T$h12:$mi12(?::$s12)?$zone?", regionZone), + name("strict_date", strictDate, days = LenientDays), + name("date", s"$y9(?:-$m12(?:-$d12)?)?"), + name("strict_date_hour", s"${strictDate}T$h2", days = LastDays, endOfDay = true), + name("date_hour", s"${anyDay}T$h12"), + name("strict_date_hour_minute", s"${strictDate}T$h2:$mi2", days = LastDays, endOfDay = true), + name("date_hour_minute", s"${anyDay}T$h12:$mi12"), + name( + "strict_date_hour_minute_second", + s"${strictDate}T$h2:$mi2:$s2", + days = if (es8) StrictDays else LastDays, + endOfDay = true, + lastDayAt24 = true + ), + name("date_hour_minute_second", s"${anyDay}T$h12:$mi12:$s12"), + name("strict_date_hour_minute_second_fraction", s"${strictDay}T$h2:$mi2:$s2\\.$f"), + name("date_hour_minute_second_fraction", s"${anyDay}T$h2:$mi2:$s2\\.$f"), + name("strict_date_hour_minute_second_millis", s"${strictDay}T$h2:$mi2:$s2\\.$f"), + name("date_hour_minute_second_millis", s"${anyDay}T$h2:$mi2:$s2\\.$f3"), + name("strict_hour", h2, endOfDay = true), + name("hour", h12), + name("strict_hour_minute", s"$h2:$mi2"), + name("hour_minute", s"$h12:$mi12"), + name("strict_hour_minute_second", s"$h2:$mi2:$s2"), + name("hour_minute_second", s"$h12:$mi12:$s12"), + name("strict_hour_minute_second_fraction", s"$h2:$mi2:$s2\\.$f"), + name("hour_minute_second_fraction", s"$h2:$mi2:$s2\\.$f"), + name("strict_hour_minute_second_millis", s"$h2:$mi2:$s2\\.$f"), + name("hour_minute_second_millis", s"$h2:$mi2:$s2\\.$f3"), + name("strict_time", s"$h2:$mi2:$s2\\.$f$zone", wideZone), + name("time", s"$h2:$mi2:$s2\\.$f$zone", wideZone), + name("strict_time_no_millis", s"$h2:$mi2:$s2$zone", wideZone), + name("time_no_millis", s"$h12:$mi12:$s12$zone", wideZone), + name("strict_t_time", s"T$h2:$mi2:$s2\\.$f$zone", wideZone), + name("t_time", s"T$h2:$mi2:$s2\\.$f$zone", wideZone), + name("strict_t_time_no_millis", s"T$h2:$mi2:$s2$zone", wideZone), + name("t_time_no_millis", s"T$h12:$mi12:$s12$zone", wideZone), + name("strict_ordinal_date", strictOrdinal), + name("ordinal_date", ordinal), + name( + "strict_ordinal_date_time", + s"${strictOrdinal}T$h2:$mi2(?::$s2\\.$f)?$zone", + wideZone + ), + name("ordinal_date_time", s"${ordinal}T$h12:$mi12(?::$s2\\.$f)?$zone", wideZone), + name("strict_ordinal_date_time_no_millis", s"${strictOrdinal}T$h2:$mi2:$s2$zone", wideZone), + name("ordinal_date_time_no_millis", s"${ordinal}T$h12:$mi12:$s12$zone", wideZone), + name("basic_date", "(?\\d{4})(?\\d{2})(?\\d{1,2})"), + name( + "basic_date_time", + s"(?\\d{4})(?\\d{2})(?\\d{2})T$basicTime\\.$f$zone", + wideZone + ), + name( + "basic_date_time_no_millis", + s"(?\\d{4})(?\\d{2})(?\\d{2})T$basicTime$zone", + wideZone + ), + name("basic_ordinal_date", "(?\\d{4})(?\\d{3})"), + name("basic_ordinal_date_time", s"(?\\d{4})(?\\d{3})T$basicTime\\.$f$zone", wideZone), + name( + "basic_ordinal_date_time_no_millis", + s"(?\\d{4})(?\\d{3})T$basicTime$zone", + wideZone + ), + name("basic_time", s"$basicTime\\.$f$zone", wideZone), + name("basic_time_no_millis", s"$basicTime$zone", wideZone), + name("basic_t_time", s"T$basicTime\\.$f$zone", wideZone), + name("basic_t_time_no_millis", s"T$basicTime$zone", wideZone), + name("basic_week_date", "(?\\d{4})W(?\\d{2})(?\\d)"), + name("strict_basic_week_date", "(?\\d{4})W(?\\d{2})(?\\d)"), + name( + "basic_week_date_time", + s"(?\\d{4})W(?\\d{2})(?\\d)T$basicTime\\.$f$zone", + wideZone + ), + name( + "strict_basic_week_date_time", + s"(?\\d{4})W(?\\d{2})(?\\d)T$basicTime\\.$f$zone", + wideZone + ), + name( + "basic_week_date_time_no_millis", + s"(?\\d{4})W(?\\d{2})(?\\d)T$basicTime$zone", + wideZone + ), + name( + "strict_basic_week_date_time_no_millis", + s"(?\\d{4})W(?\\d{2})(?\\d)T$basicTime$zone", + wideZone + ), + name("strict_week_date", "(?-?\\d{4})-W(?\\d{2})-(?\\d)"), + name("week_date", "(?-?\\d{4})-W(?\\d{1,2}+)-(?\\d)"), + name( + "strict_week_date_time", + s"(?-?\\d{4})-W(?\\d{2})-(?\\d)T$h2:$mi2:$s2\\.$f$zoneOrLowerZ", + zOrUtc + ), + name( + "week_date_time", + s"(?-?\\d{4})-W(?\\d{1,2}+)-(?\\d)T$h2:$mi2:$s2\\.$f$zone", + wideZone + ), + name( + "strict_week_date_time_no_millis", + s"(?-?\\d{4})-W(?\\d{2})-(?\\d)T$h2:$mi2:$s2$zoneOrLowerZ", + zOrUtc + ), + name( + "week_date_time_no_millis", + s"(?-?\\d{4})-W(?\\d{1,2}+)-(?\\d)T$h12:$mi12:$s12$zone", + wideZone + ), + name("strict_weekyear", "(?-?\\d{4}|[+-]\\d{5,9}+)"), + name("weekyear", "(?-?\\d{1,9}+)"), + name("strict_weekyear_week", "(?-?\\d{4})-W(?\\d{2})"), + name("weekyear_week", "(?-?\\d{1,9}+)-W(?\\d{1,2}+)"), + name("strict_weekyear_week_day", "(?-?\\d{4})-W(?\\d{2})-(?\\d{1,2}+)"), + name("weekyear_week_day", "(?-?\\d{1,9}+)-W(?\\d{1,2}+)-(?\\d{1,2}+)"), + name("strict_year_month_day", s"$y4(?:-$m2(?:-$d2)?)?"), + name("year_month_day", date), + name("strict_year_month", s"$y4-$m2"), + name("year_month", s"$y9-$m12"), + name("strict_year", y4), + name("year", y9) + ) + } + + /** Elasticsearch 6 (Joda-Time): every name of its documentation, as MEASURED on 6.8.23. */ + private val jodaNames: Map[String, Alternative] = { + val y4 = "(?[+-]?\\d{4})" + val y9 = "(?[+-]?\\d{1,9}+)" + val m2 = "(?\\d{2})" + val m12 = "(?\\d{1,2}+)" + val d2 = "(?\\d{2})" + val d12 = "(?\\d{1,2}+)" + val h2 = "(?\\d{2})" + val h12 = "(?\\d{1,2}+)" + val mi2 = "(?\\d{2})" + val mi12 = "(?\\d{1,2}+)" + val s2 = "(?\\d{2})" + val s12 = "(?\\d{1,2}+)" + val f = "(?\\d{1,9}+)" + val f3 = "(?\\d{1,3}+)" + val zone = s"(?[Zz]|$OffsetText)" + val strictDate = s"$y4-$m2-$d2" + val date = s"$y9-$m12-$d12" + val ordinal = s"$y9-(?\\d{1,3}+)" + val strictOrdinal = s"$y4-(?\\d{3})" + val week = "(?[+-]?\\d{1,9}+)-W(?\\d{1,2}+)-(?\\d)" + val strictWeek = "(?[+-]?\\d{4})-W(?\\d{2})-(?\\d)" + val basicTime = "(?\\d{2})(?\\d{2})(?\\d{2})" + val basicWeek = "(?\\d{4})W(?\\d{2})(?\\d)" + def name(n: String, regex: String): (String, Alternative) = + n -> Grammar(n, g(regex), jodaOffset) + Map( + // a calendar, a week or an ordinal date, then a time (MEASURED) + name( + "strict_date_optional_time", + s"(?:$y4(?:-$m2(?:-$d2)?)?|(?[+-]?\\d{4})-W(?\\d{2})(?:-(?\\d))?|" + + s"(?[+-]?\\d{4})-(?\\d{3}))(?:T$h2(?::$mi2(?::$s2(?:[.,]$f)?)?)?$zone?)?" + ), + name( + "date_optional_time", + s"(?:$y9(?:-$m12(?:-$d12)?)?|(?[+-]?\\d{1,9}+)-W(?\\d{1,2}+)(?:-(?\\d))?|" + + s"(?[+-]?\\d{1,9}+)-(?\\d{3}))(?:T$h12(?::$mi12(?::$s12(?:[.,]$f)?)?)?$zone?)?" + ), + name("strict_date_time", s"${strictDate}T$h2:$mi2:$s2\\.$f$zone"), + name("date_time", s"${date}T$h12:$mi12:$s12\\.$f$zone"), + name("strict_date_time_no_millis", s"${strictDate}T$h2:$mi2:$s2$zone"), + name("date_time_no_millis", s"${date}T$h12:$mi12:$s12$zone"), + name("strict_date", strictDate), + name("date", date), + name("strict_date_hour", s"${strictDate}T$h2"), + name("date_hour", s"${date}T$h12"), + name("strict_date_hour_minute", s"${strictDate}T$h2:$mi2"), + name("date_hour_minute", s"${date}T$h12:$mi12"), + name("strict_date_hour_minute_second", s"${strictDate}T$h2:$mi2:$s2"), + name("date_hour_minute_second", s"${date}T$h12:$mi12:$s12"), + name("strict_date_hour_minute_second_fraction", s"${strictDate}T$h2:$mi2:$s2\\.$f"), + name("date_hour_minute_second_fraction", s"${date}T$h2:$mi2:$s2\\.$f"), + name("strict_date_hour_minute_second_millis", s"${strictDate}T$h2:$mi2:$s2\\.$f3"), + name("date_hour_minute_second_millis", s"${date}T$h2:$mi2:$s2\\.$f3"), + name("strict_hour", "(?\\d{2}|\\+\\d)"), + name("hour", h12), + name("strict_hour_minute", s"$h2:$mi2"), + name("hour_minute", s"$h12:$mi12"), + name("strict_hour_minute_second", s"$h2:$mi2:$s2"), + name("hour_minute_second", s"$h12:$mi12:$s12"), + name("strict_hour_minute_second_fraction", s"$h2:$mi2:$s2\\.$f"), + name("hour_minute_second_fraction", s"$h2:$mi2:$s2\\.$f"), + name("strict_hour_minute_second_millis", s"$h2:$mi2:$s2\\.$f3"), + name("hour_minute_second_millis", s"$h2:$mi2:$s2\\.$f3"), + name("strict_time", s"$h2:$mi2:$s2\\.$f$zone"), + name("time", s"$h2:$mi2:$s2\\.$f$zone"), + name("strict_time_no_millis", s"$h2:$mi2:$s2$zone"), + name("time_no_millis", s"$h12:$mi12:$s12$zone"), + name("strict_t_time", s"T$h2:$mi2:$s2\\.$f$zone"), + name("t_time", s"T$h2:$mi2:$s2\\.$f$zone"), + name("strict_t_time_no_millis", s"T$h2:$mi2:$s2$zone"), + name("t_time_no_millis", s"T$h12:$mi12:$s12$zone"), + name("strict_ordinal_date", strictOrdinal), + name("ordinal_date", ordinal), + name("strict_ordinal_date_time", s"${strictOrdinal}T$h2:$mi2:$s2\\.$f$zone"), + name("ordinal_date_time", s"${ordinal}T$h2:$mi2:$s2\\.$f$zone"), + name("strict_ordinal_date_time_no_millis", s"${strictOrdinal}T$h2:$mi2:$s2$zone"), + name("ordinal_date_time_no_millis", s"${ordinal}T$h12:$mi12:$s12$zone"), + name("basic_date", "(?\\d{4})(?\\d{2})(?\\d{2})"), + name("basic_date_time", s"(?\\d{4})(?\\d{2})(?\\d{2})T$basicTime\\.$f$zone"), + name("basic_date_time_no_millis", s"(?\\d{4})(?\\d{2})(?\\d{2})T$basicTime$zone"), + name("basic_ordinal_date", "(?\\d{4})(?\\d{3})"), + name("basic_ordinal_date_time", s"(?\\d{4})(?\\d{3})T$basicTime\\.$f$zone"), + name("basic_ordinal_date_time_no_millis", s"(?\\d{4})(?\\d{3})T$basicTime$zone"), + name("basic_time", s"$basicTime\\.$f$zone"), + name("basic_time_no_millis", s"$basicTime$zone"), + name("basic_t_time", s"T$basicTime\\.$f$zone"), + name("basic_t_time_no_millis", s"T$basicTime$zone"), + name("basic_week_date", basicWeek), + name("strict_basic_week_date", basicWeek), + name("basic_week_date_time", s"${basicWeek}T$basicTime\\.$f$zone"), + name("strict_basic_week_date_time", s"${basicWeek}T$basicTime\\.$f$zone"), + name("basic_week_date_time_no_millis", s"${basicWeek}T$basicTime$zone"), + name("strict_basic_week_date_time_no_millis", s"${basicWeek}T$basicTime$zone"), + name("strict_week_date", strictWeek), + name("week_date", week), + name("strict_week_date_time", s"${strictWeek}T$h2:$mi2:$s2\\.$f$zone"), + name("week_date_time", s"${week}T$h2:$mi2:$s2\\.$f$zone"), + name("strict_week_date_time_no_millis", s"${strictWeek}T$h2:$mi2:$s2$zone"), + name("week_date_time_no_millis", s"${week}T$h12:$mi12:$s12$zone"), + name("strict_weekyear", "(?[+-]?\\d{4})"), + name("weekyear", "(?[+-]?\\d{1,9}+)"), + name("strict_weekyear_week", "(?[+-]?\\d{4})-W(?\\d{2})"), + name("weekyear_week", "(?[+-]?\\d{1,9}+)-W(?\\d{1,2}+)"), + name("strict_weekyear_week_day", strictWeek), + name("weekyear_week_day", week), + name("strict_year_month_day", strictDate), + name("year_month_day", date), + name("strict_year_month", s"$y4-$m2"), + name("year_month", s"$y9-$m12"), + name("strict_year", y4), + name("year", y9) + ) + } + + private lazy val java7Names = javaNames(es8 = false) + private lazy val java8Names = javaNames(es8 = true) + + /** Elasticsearch's built-in format names (its documentation's list): a name core does not read is + * [[Unknown]], never taken for a pattern. Elasticsearch 6 and 7 also take them in camel case + * (`strictDateOptionalTime`). + */ + private val BuiltInNames: Set[String] = snakeNames.toSet ++ camelNames.keySet + + /** Elasticsearch 6 and 7 also read a name in camel case (`strictDateOptionalTime`) as the name + * (MEASURED: every name but `epochMillis` and `epochSecond`, which they refuse); Elasticsearch 8 + * reads it as a pattern. + */ + private lazy val camelNames: Map[String, String] = + snakeNames + .filterNot(n => n == "epoch_millis" || n == "epoch_second") + .map { name => + val parts = name.split('_') + (parts.head + parts.tail.map(_.capitalize).mkString) -> name + } + .filter { case (camel, snake) => camel != snake } + .toMap + + private lazy val snakeNames: Seq[String] = { + val snake = Seq( + "epoch_millis", + "epoch_second", + "date_optional_time", + "strict_date_optional_time", + "strict_date_optional_time_nanos", + "basic_date", + "basic_date_time", + "basic_date_time_no_millis", + "basic_ordinal_date", + "basic_ordinal_date_time", + "basic_ordinal_date_time_no_millis", + "basic_time", + "basic_time_no_millis", + "basic_t_time", + "basic_t_time_no_millis", + "basic_week_date", + "strict_basic_week_date", + "basic_week_date_time", + "strict_basic_week_date_time", + "basic_week_date_time_no_millis", + "strict_basic_week_date_time_no_millis", + "date", + "strict_date", + "date_hour", + "strict_date_hour", + "date_hour_minute", + "strict_date_hour_minute", + "date_hour_minute_second", + "strict_date_hour_minute_second", + "date_hour_minute_second_fraction", + "strict_date_hour_minute_second_fraction", + "date_hour_minute_second_millis", + "strict_date_hour_minute_second_millis", + "date_time", + "strict_date_time", + "date_time_no_millis", + "strict_date_time_no_millis", + "hour", + "strict_hour", + "hour_minute", + "strict_hour_minute", + "hour_minute_second", + "strict_hour_minute_second", + "hour_minute_second_fraction", + "strict_hour_minute_second_fraction", + "hour_minute_second_millis", + "strict_hour_minute_second_millis", + "ordinal_date", + "strict_ordinal_date", + "ordinal_date_time", + "strict_ordinal_date_time", + "ordinal_date_time_no_millis", + "strict_ordinal_date_time_no_millis", + "time", + "strict_time", + "time_no_millis", + "strict_time_no_millis", + "t_time", + "strict_t_time", + "t_time_no_millis", + "strict_t_time_no_millis", + "week_date", + "strict_week_date", + "week_date_time", + "strict_week_date_time", + "week_date_time_no_millis", + "strict_week_date_time_no_millis", + "weekyear", + "strict_weekyear", + "weekyear_week", + "strict_weekyear_week", + "weekyear_week_day", + "strict_weekyear_week_day", + "year", + "strict_year", + "year_month", + "strict_year_month", + "year_month_day", + "strict_year_month_day", + "iso8601" + ) + snake + } + + /** The format of a field, as `dialect` reads it: its `||` alternatives, each a built-in name or a + * custom pattern. `clock` dates Joda's two-digit years. + */ + def apply(spec: String, dialect: Dialect, clock: Clock = Clock.systemUTC()): MappedDateFormat = { + // Elasticsearch 6.8 reads a format with a leading `8` -- every alternative -- with `java.time` + val java6 = dialect == Dialect.Joda && spec.length > 1 && spec.charAt(0) == '8' + MappedDateFormat( + spec, + dialect, + (if (java6) spec.substring(1) else spec) + .split("\\|\\|") + .toSeq + .map(_.trim) + .filter(_.nonEmpty) + .map(alternative(_, dialect, clock, java6)) + ) + } + + private def alternative( + spelled: String, + dialect: Dialect, + clock: Clock, + java6: Boolean + ): Alternative = { + // Elasticsearch 7 and 8 read a format with a leading `8` -- the `java.time` selector of 6.8 -- + // as the format without it (MEASURED: every name and pattern, 0 difference); 6 and 7 read a name + // in camel case as the name + val unprefixed = + if (dialect != Dialect.Joda && spelled.length > 1 && spelled.charAt(0) == '8') + spelled.substring(1) + else spelled + val text = + if (dialect == Dialect.Java8 || java6) unprefixed + else camelNames.getOrElse(unprefixed, unprefixed) + val names = dialect match { + case Dialect.Joda => jodaNames + case Dialect.Java7 => java7Names + case Dialect.Java8 => java8Names + } + // Elasticsearch 6.8's java.time epochs read as Elasticsearch 7's (MEASURED, 0 difference) + val epochs = if (java6) Dialect.Java7 else dialect + text match { + case "epoch_millis" => Epoch(text, 1L, epochs) + case "epoch_second" => Epoch(text, 1000L, epochs) + // the names of Elasticsearch 6.8's java.time mode read their own way, which core does not + case _ if java6 && BuiltInNames.contains(text) => + Unknown(spelled, dialect, "a name of Elasticsearch 6.8's java.time mode ('8')") + case _ if java6 => + val pattern = JavaDatePattern.Pattern(text, JavaDatePattern.Es6) + pattern.unread.fold[Alternative](pattern)(Unknown(spelled, dialect, _)) + case _ => + names.get(text) match { + case Some(name) => name + case None if BuiltInNames.contains(text) => Unknown(spelled, dialect) + case None => + dialect match { + case Dialect.Joda => + val pattern = JodaDatePattern.Pattern(text, clock) + pattern.unread.fold[Alternative](pattern)(Unknown(spelled, dialect, _)) + case Dialect.Java7 | Dialect.Java8 => + val pattern = JavaDatePattern.Pattern( + text, + if (dialect == Dialect.Java7) JavaDatePattern.Es7 else JavaDatePattern.Es8 + ) + pattern.unread.fold[Alternative](pattern)(Unknown(spelled, dialect, _)) + } + } + } + } + + // --------------------------------------------------------------------------------------------- + // Values the response parser already read as ISO dates + // --------------------------------------------------------------------------------------------- + + private def instantOf(value: Any): Instant = + value match { + case z: ZonedDateTime => z.toInstant + case o: OffsetDateTime => o.toInstant + case l: LocalDateTime => l.toInstant(ZoneOffset.UTC) + case d: LocalDate => d.atStartOfDay(ZoneOffset.UTC).toInstant + case other => throw new IllegalArgumentException(s"not a date: $other") + } + + /** What decides whether a name reads a value the response parser read as an ISO date -- the shape + * of its [[spellings]]: its class, its year's sign and digit count, whether its seconds and its + * fraction are zero, its fraction's significant digits, and its offset's (zero, with seconds) + * presence -- a value with a region is never read (`readValue`). Two values of one shape are + * read alike. + */ + private def shapeOf(value: Any): Long = { + def year(y: Int): Long = { + val digits = Math.max(4, String.valueOf(Math.abs(y.toLong)).length).toLong + (if (y < 0) 1L else if (y > 9999) 2L else 0L) | (digits << 2) + } + def time(t: LocalTime): Long = { + val nano = t.getNano + val significant = + if (nano == 0) 0 + else { + var n = nano; var d = 9 + while (n % 10 == 0) { n /= 10; d -= 1 } + d + } + (if (t.getSecond == 0) 1L else 0L) | (if (nano == 0) 2L else 0L) | (significant.toLong << 2) + } + def offset(o: ZoneOffset): Long = + (if (o.getTotalSeconds == 0) 1L else 0L) | (if (o.getTotalSeconds % 60 != 0) 2L else 0L) + value match { + case d: LocalDate => 1L | (year(d.getYear) << 4) + case l: LocalDateTime => 2L | (year(l.getYear) << 4) | (time(l.toLocalTime) << 12) + case o: OffsetDateTime => + 3L | (year(o.getYear) << 4) | (time(o.toLocalTime) << 12) | (offset(o.getOffset) << 20) + case z: ZonedDateTime => + 4L | (year(z.getYear) << 4) | (time(z.toLocalTime) << 12) | (offset(z.getOffset) << 20) + case _ => 0L + } + } + + /** Elasticsearch's `date` keeps milliseconds. */ + private def millis(instant: Instant): Instant = Instant.ofEpochMilli(instant.toEpochMilli) + + /** Every text the response parser reads as `value` (MEASURED against `java.time`'s ISO + * formatters, which it uses): the date as `yyyy-MM-dd`; the time with or without its zero + * seconds, a fraction in any width from its significant digits to nine, or a bare `.`; an offset + * as `Z`, `+00:00`, `-00:00` and with zero seconds (a value with a region is never read, + * `readValue`). + */ + private[client] def spellings(value: Any): Iterator[String] = { + // LAZY: a format of names only reads the first spelling, java.time's own, almost always + def date(d: LocalDate) = DateTimeFormatter.ISO_LOCAL_DATE.format(d) + def times(t: LocalTime): Iterator[String] = { + val hm = f"${t.getHour}%02d:${t.getMinute}%02d" + lazy val hms = f"$hm:${t.getSecond}%02d" + val fractions: Iterator[String] = + if (t.getNano == 0) Iterator("", ".") ++ (1 to 9).iterator.map(w => "." + ("0" * w)) + else { + val nine = f"${t.getNano}%09d" + val significant = nine.reverse.dropWhile(_ == '0').length + (significant to 9).iterator.map(w => "." + nine.take(w)) + } + (if (t.getSecond == 0 && t.getNano == 0) Iterator(hm) else Iterator.empty) ++ + fractions.map(hms + _) + } + def offsets(o: ZoneOffset): Seq[String] = + if (o.getTotalSeconds == 0) Seq("Z", "+00:00", "-00:00", "+00:00:00", "-00:00:00") + else if (o.getTotalSeconds % 60 == 0) Seq(o.getId, o.getId + ":00") + else Seq(o.getId) + def dateTimes(l: LocalDateTime, zones: Seq[String]): Iterator[String] = { + val day = date(l.toLocalDate) + times(l.toLocalTime).flatMap(t => zones.iterator.map(z => s"${day}T$t$z")) + } + value match { + case d: LocalDate => Iterator(date(d)) + case t: LocalTime => times(t) + case l: LocalDateTime => dateTimes(l, Seq("")) + case o: OffsetDateTime => dateTimes(o.toLocalDateTime, offsets(o.getOffset)) + case z: ZonedDateTime => dateTimes(z.toLocalDateTime, offsets(z.getOffset)) + case other => Iterator(other.toString) + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index c264deb3..dbff9581 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -490,7 +490,13 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC // that is not an `IndicesApi`, mappings that cannot be read or that disagree) nothing is // refused on types: Elasticsearch answers, exactly as for a single statement. A branch // without a schema keeps its bare columns `Any`, which pass. - (if (resolved.exists(_.schema.isDefined)) MultiSearch.branchTypes(resolved) + // + // The OPERATORS go with the branches: the one verdict the relational engine's caller asks + // too, typed over the operator tree (only `UNION ALL` reaches this line today). It refuses + // only what fails when the statement runs: a date beside a number, a nested type beside + // another. + (if (resolved.exists(_.schema.isDefined)) + MultiSearch.branchTypes(resolved, multiple.resolvedOperators) else Right(())) match { case Left(reason) => ElasticResult.failure( @@ -2442,12 +2448,27 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC )(implicit context: ConversionContext): Seq[ListMap[String, Any] => ListMap[String, Any]] = { val outputFields = unionAllOutputFieldNames(multiple) if (outputFields.isEmpty) multiple.requests.map(_ => identity[ListMap[String, Any]] _) - else - multiple.requests.map { leg => - rowProjector(unionAllLegFieldNames(leg, outputFields), outputFields) + else { + // ...and each column in the ONE type the result gives it, converted as the row is rebuilt + // (`SetOperationValues`); `None` for every leg of a statement whose columns agree + val converters = SetOperationValues.converters(multiple, setOperationVersion) + multiple.requests.zip(converters).map { case (leg, convert) => + rowProjector(unionAllLegFieldNames(leg, outputFields), outputFields, convert.orNull) } + } } + /** The Elasticsearch version the statement runs on (`VersionApi.version`, cached per client): a + * set operation reads a date field's values as THAT major reads the field's mapping format + * (`SetOperationValues.converters`, which asks for it at most once per statement, and only when + * a converting position reads a date field). `None` for a client that cannot tell it. + */ + private def setOperationVersion: Option[String] = + this match { + case api: VersionApi => api.version.toOption + case _ => None + } + /** The names the result's columns take: the FIRST branch's, SQL-92 §7.10. */ private def unionAllOutputFieldNames(multiple: MultiSearch): Seq[String] = multiple.requests.headOption.map(extractOutputFieldNames).getOrElse(Seq.empty) @@ -2472,13 +2493,19 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC */ private def unionAllLegProjections(multiple: MultiSearch): Seq[LegProjection] = { val outputFields = unionAllOutputFieldNames(multiple) - multiple.requests.map { leg => + // with no declared names there is no column to type, as for the other routes + // (`unionAllRowMappers`) + val converters = + if (outputFields.isEmpty) multiple.requests.map(_ => None) + else SetOperationValues.converters(multiple, setOperationVersion) + multiple.requests.zip(converters).map { case (leg, convert) => LegProjection( leg.fieldAliases, unionAllLegFieldNames(leg, outputFields), leg.nestedHitsMappings, // the plan that built THIS leg's request -- never one made from the merged map - toClientAggregations(leg.sqlAggregations) + toClientAggregations(leg.sqlAggregations), + convert ) } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala b/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala new file mode 100644 index 00000000..e98faf0d --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/SetOperationValues.scala @@ -0,0 +1,925 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.query.MultiSearch +import app.softnetwork.elastic.sql.query.MultiSearch.{ + SetOperationColumn, + SetOperationType, + StructField +} +import app.softnetwork.elastic.sql.query.SingleSearch +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} + +import java.nio.charset.StandardCharsets +import java.time.format.DateTimeFormatter +import java.time.{ + Instant, + LocalDate, + LocalDateTime, + LocalTime, + OffsetDateTime, + OffsetTime, + ZoneOffset, + ZonedDateTime +} +import scala.collection.immutable.ListMap +import scala.jdk.CollectionConverters._ + +/** The values of a `UNION ALL` column, brought to the ONE type the column has + * ([[MultiSearch.columnTypes]]: DuckDB 1.5.5's, or `VARCHAR` where DuckDB gives none), at every + * depth by ONE converter ([[positionConverter]]): every value of a `DOUBLE` position is a + * `Double`, of a `BIGINT` one a `Long`, of a `VARCHAR` one a `String`, of a `TIMESTAMP` one a + * `ZonedDateTime`, of a `VARBINARY` one an `Array[Byte]`, of a `STRUCT` or a `GEO_POINT` one a map + * holding the position's fields in DuckDB's order, of a list a list of its element's type -- never + * a mix, which is what JDBC and Arrow, typing a column from its first value, would otherwise + * report wrongly. + * + * 🔴 PER-ROW work on the extraction path, so it is decided ONCE per statement: [[converters]] is + * `None` for every leg of a statement whose columns all agree -- no function, no rebuild -- and, + * for a leg that does convert, holds a function only at the positions whose column converts. The + * functions are applied as `rowProjector` rebuilds the row it builds anyway, never by a second + * pass. + */ +private[client] object SetOperationValues { + + /** The per-position conversions of each leg of `multiple`, in leg order -- `None` for a leg with + * nothing to convert. `multiple` carries its branches' schemas (`resolveWithSchema`). + * + * `version` is the Elasticsearch version the statement runs on (`VersionApi.version`): a date + * field's values are read as THAT major reads its mapping format ([[MappedDateFormat]]). It is + * asked at most ONCE per statement, and only when a converting position reads a date field. + */ + def converters( + multiple: MultiSearch, + version: => Option[String] + ): Seq[Option[Array[Any => Any]]] = + MultiSearch.columnTypes(multiple.requests) match { + case Right(columns) if columns.exists(_.converts) => + val names = columnNames(multiple, columns.size) + val dates = DateReading(version) + multiple.requests.indices.map(leg => + legConverters(multiple.requests(leg), leg, columns, names, dates) + ) + case _ => multiple.requests.map(_ => None) + } + + /** How a statement reads its date fields: as the Elasticsearch major it runs on reads their + * mapping format ([[MappedDateFormat.Dialect]]). The version is asked once, the first time a + * converting position reads a date field, and never for a statement that has none. + */ + private[client] final class DateReading( + version: => Option[String], + val clock: java.time.Clock = java.time.Clock.systemUTC() + ) { + lazy val elasticVersion: Option[String] = version + lazy val dialect: Option[MappedDateFormat.Dialect] = + elasticVersion.flatMap(MappedDateFormat.Dialect.of) + } + + private[client] object DateReading { + def apply(version: => Option[String]): DateReading = new DateReading(version) + + /** A statement whose Elasticsearch version is not known. */ + def unknown: DateReading = new DateReading(None) + } + + /** The name each column takes: the first declaring branch's, as SQL and DuckDB name it. */ + private def columnNames(multiple: MultiSearch, width: Int): Seq[String] = + multiple.requests + .map(_.select.fieldsWithComputedAliases) + .collectFirst { case fields if fields.size == width => fields.map(_.outputName) } + .getOrElse(Seq.fill(width)("")) + + private def legConverters( + leg: SingleSearch, + index: Int, + columns: Seq[SetOperationColumn], + names: Seq[String], + dates: DateReading + ): Option[Array[Any => Any]] = + // a leg's rows are built from its declared projection, position by position (issue #354); a + // projection of another width -- an opaque `SELECT *` -- has no position to convert at + if (leg.select.fieldsWithComputedAliases.size != columns.size) None + else { + val functions: Array[Any => Any] = columns.zipWithIndex.map { case (column, position) => + // a NULL literal's branch too: its value is Elasticsearch's `[null]`, read as NULL + if (!column.converts || column.branches.lift(index).flatten.isEmpty) null + else { + val item = leg.select.fieldsWithComputedAliases(position) + val field = Option(item.identifier.name).filter(_.nonEmpty).getOrElse(item.outputName) + positionConverter(column.result, column.branches, index, names(position), field, dates) + } + }.toArray + if (functions.forall(_ == null)) None else Some(functions) + } + + /** The conversion of branch `index`'s value at ONE converting position -- a column, a field of a + * struct, the element of a list -- to the position's type, by ONE converter at every depth: + * EVERY branch's value at a converting position is converted, the branch whose type is already + * the position's too, so the position holds one class. + * + * - A scalar is converted by [[converter]]: a `BIGINT` is a `Long`, a `DOUBLE` a `Double`, a + * `TIMESTAMP` a `ZonedDateTime`, a `VARBINARY` bytes, a `VARCHAR` DuckDB's spelling; a + * multi-valued value FAILS, in DuckDB's words ([[nonScalarFailure]]). A date field's value + * is read with its mapping format, as `dates` says the Elasticsearch major reads it; `field` + * is its path, which a failure names. + * - A struct is a map of the position's fields in DuckDB's order ([[structConverter]]), a + * field a branch does not have NULL. + * - A list is a list of converted elements ([[listConverter]]). + * - A point is a `{lat, lon}`, whatever way Elasticsearch stored it ([[geoPoint]]). + * + * Inside a converting struct or list, a position whose branches all agree keeps its values as + * read ([[SetOperationColumn.converts]]); a point is always read as a point. Decided ONCE per + * leg, never per row. + */ + private[client] def positionConverter( + target: SetOperationType, + branches: Seq[Option[SetOperationType]], + index: Int, + name: String, + field: String = "", + dates: DateReading = DateReading.unknown + ): Any => Any = + target.sqlType match { + case SQLTypes.GeoPoint => value => geoPoint(value, name) + case SQLTypes.Struct if target.fields.isEmpty => identity + case SQLTypes.Struct => structConverter(target, branches, index, name, field, dates) + case SQLTypes.Array(_) => listConverter(target, branches, index, name, field, dates) + case result => + val branch = branches.lift(index).flatten + val source = branch.fold[SQLType](SQLTypes.Null)(_.sqlType) + converter( + source, + result, + nonScalarFailure(result, branches.map(_.map(_.sqlType)), index, source, name), + branch.flatMap(_.format), + if (field.isEmpty) name else field, + name, + dates + ) + } + + /** The conversion at a position INSIDE a converting one: [[positionConverter]] where its branches + * disagree, and for a point; its values as read where they agree. + */ + private def innerConverter( + target: SetOperationType, + branches: Seq[Option[SetOperationType]], + index: Int, + name: String, + field: String, + dates: DateReading + ): Any => Any = + if ( + target.sqlType == SQLTypes.GeoPoint || + SetOperationColumn.converts(target, branches.flatten) + ) positionConverter(target, branches, index, name, field, dates) + else identity + + /** The failure of a converting position that a NON-SCALAR value -- a multi-valued field, an + * object -- reaches: the position holds ONE value of its type per row, and such a value has no + * one value to convert. The message is in the words DuckDB uses for the same column, which it + * makes the LIST of that branch's type before failing the cast of the first other branch into it + * (MEASURED on DuckDB 1.5.5): `Unimplemented type for cast (INTEGER -> VARCHAR[]) when casting + * from source column k`, or, the other branch being text, `Type VARCHAR with value 'c' can't be + * cast to the destination type VARCHAR[]` -- whose value, another branch's, is not known here. + */ + private def nonScalarFailure( + result: SQLType, + branchTypes: Seq[Option[SQLType]], + index: Int, + source: SQLType, + name: String + ): String = { + val list = duckDbTypeName(source) + "[]" + branchTypes.zipWithIndex.collectFirst { + case (Some(other), i) if i != index && other != SQLTypes.Null => other + } match { + case Some(SQLTypes.Varchar) => + s"Conversion Error: Type VARCHAR can't be cast to the destination type $list when " + + s"casting from source column $name" + case Some(other) => + s"Conversion Error: Unimplemented type for cast (${duckDbTypeName(other)} -> $list) " + + s"when casting from source column $name" + case None => + s"Conversion Error: Unimplemented type for cast ($list -> " + + s"${duckDbTypeName(result)}) when casting from source column $name" + } + } + + /** The name DuckDB gives a type in its messages. */ + private def duckDbTypeName(t: SQLType): String = + t match { + case SQLTypes.Varchar | SQLTypes.Char | SQLTypes.Text | SQLTypes.Keyword => "VARCHAR" + case SQLTypes.Int => "INTEGER" + case SQLTypes.Real => "FLOAT" + case SQLTypes.Numeric => "DOUBLE" + case SQLTypes.VarBinary => "BLOB" + case SQLTypes.DateTime => "TIMESTAMP" + case SQLTypes.Array(element) => duckDbTypeName(element) + "[]" + case other => other.typeId + } + + /** A struct's type as DuckDB names it: `STRUCT(lat DOUBLE, lon DOUBLE, a INTEGER, b VARCHAR)`. */ + private def duckDbStructName(fields: Seq[StructField]): String = + fields + .map { f => + val typeName = + if (f.fields.nonEmpty) duckDbStructName(f.fields) else duckDbTypeName(f.sqlType) + s"${f.name} $typeName" + } + .mkString("STRUCT(", ", ", ")") + + /** A value reaching a converting `STRUCT` position: a map holding the position's fields, in its + * order -- DuckDB's merged struct, MEASURED. A field the branch does not have is NULL; every + * other field is read under the branch's own name, whatever its case, and converted where its + * branches disagree ([[innerConverter]]): an `INTEGER` beside a `VARCHAR` becomes `'1'`, an + * `INTEGER` beside a `BIGINT` a `Long`, in every branch. + * + * A point branch is read as `{lat, lon}` whatever the way Elasticsearch stored it + * ([[geoPoint]]). A value with several points or several objects FAILS the statement, in + * DuckDB's words, as a multi-valued value does in every converting position; so does a value + * that is not an object. Decided ONCE per leg: the source field each field reads, and its + * conversion. + */ + private[client] def structConverter( + target: SetOperationType, + branches: Seq[Option[SetOperationType]], + index: Int, + name: String, + field: String = "", + dates: DateReading = DateReading.unknown + ): Any => Any = { + val source = branches.lift(index).flatten + val sourceFields = source.fold[Seq[StructField]](Nil)(_.fields) + val read: Any => scala.collection.Map[String, Any] = + if (source.exists(_.sqlType == SQLTypes.GeoPoint)) + value => geoPoint(value, name, target.fields) + else value => objectOf(value, name, sourceFields, target.fields) + val path = if (field.isEmpty) name else field + val plan: Seq[(String, Option[(String, Any => Any)])] = target.fields.map { t => + t.name -> sourceFields.find(_.name.equalsIgnoreCase(t.name)).map { f => + f.name -> innerConverter( + t.valueType, + SetOperationColumn.fieldTypes(branches, t.name), + index, + name, + s"$path.${f.name}", + dates + ) + } + } + value => + read(value) match { + case null => null + case map => + ListMap(plan.map { case (field, from) => + field -> from.map { case (key, convert) => convert(valueOf(map, key)) }.orNull + }: _*) + } + } + + /** A value reaching a converting list position: a list of its elements, each converted where the + * elements' branches disagree ([[innerConverter]]). A single value -- Elasticsearch stores a + * list of one as the value itself -- is a list of one; NULL is NULL, a `NULL` literal's branch + * (Elasticsearch's `[null]`) too. + */ + private[client] def listConverter( + target: SetOperationType, + branches: Seq[Option[SetOperationType]], + index: Int, + name: String, + field: String = "", + dates: DateReading = DateReading.unknown + ): Any => Any = + if (branches.lift(index).flatten.forall(_.sqlType == SQLTypes.Null)) _ => null + else { + val element = innerConverter( + target.element.getOrElse(SetOperationType(SQLTypes.Any)), + SetOperationColumn.elementTypes(branches), + index, + name, + if (field.isEmpty) name else field, + dates + ) + val list: Any => Any = { + case null => null + case l: java.util.List[_] => l.asScala.toList.map(element) + case s: Seq[_] => s.toList.map(element) + case one => List(element(one)) + } + list + } + + /** A map's value for `key`, its case aside -- DuckDB matches a struct's fields whatever their + * case. + */ + private def valueOf(map: scala.collection.Map[String, Any], key: String): Any = + map.get(key) match { + case Some(value) => value + case None => map.collectFirst { case (k, v) if k.equalsIgnoreCase(key) => v }.orNull + } + + private def keyed(map: scala.collection.Map[_, _]): scala.collection.Map[String, Any] = + map.map { case (k, v) => String.valueOf(k) -> (v: Any) } + + /** An object's value: one map, NULL for no value, or the statement's failure. */ + private def objectOf( + value: Any, + name: String, + fields: Seq[StructField], + target: Seq[StructField] + ): scala.collection.Map[String, Any] = + value match { + case null => null + case m: scala.collection.Map[_, _] => keyed(m) + case m: java.util.Map[_, _] => keyed(m.asScala) + case l: java.util.List[_] => objectOf(l.asScala.toList, name, fields, target) + case s: Seq[_] => + if (s.isEmpty) null + else if (s.lengthCompare(1) == 0) objectOf(s.head, name, fields, target) + else + throw new IllegalArgumentException( + s"Conversion Error: Unimplemented type for cast (${duckDbStructName(fields)}[] -> " + + s"${duckDbStructName(target)}) when casting from source column $name" + ) + case other => + throw new IllegalArgumentException( + s"Conversion Error: Could not read '$other' as a STRUCT when casting from source " + + s"column $name" + ) + } + + private val Decimal = """[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?""" + private val LatLon = + ("""\s*(""" + Decimal + """)\s*,\s*(""" + Decimal + """)\s*(?:,\s*""" + Decimal + """\s*)?""").r + private val WktPoint = ("""(?i)\s*POINT\s*\(\s*(""" + Decimal + """)\s+(""" + Decimal + + """)(?:\s+""" + Decimal + """)?\s*\)\s*""").r + private val GeoHash = """(?i)[0-9b-hjkmnp-z]{1,12}""".r + private val Base32 = "0123456789bcdefghjkmnpqrstuvwxyz" + + /** A `GEO_POINT` as `{lat, lon}`, whatever way Elasticsearch accepted it -- MEASURED on 8.18.3, + * whose `_source` keeps each as it was written: an object `{"lat": .., "lon": ..}` (the numbers + * possibly as text), a text `"lat,lon"`, an array `[lon, lat]` (a third number, the altitude, + * ignored), a geohash (the south-west corner of its cell, as Elasticsearch reads one), a WKT + * `POINT (lon lat)` or a GeoJSON `{"type": "Point", "coordinates": [lon, lat]}`. NULL for no + * value; the statement's failure for several points, or for a value that is not a point. + */ + private[client] def geoPoint( + value: Any, + name: String, + target: Seq[StructField] = StructField.GeoPoint + ): scala.collection.Map[String, Any] = { + def point(lat: Any, lon: Any): scala.collection.Map[String, Any] = + (number(lat), number(lon)) match { + case (Some(la), Some(lo)) => + ListMap("lat" -> java.lang.Double.valueOf(la), "lon" -> java.lang.Double.valueOf(lo)) + case _ => notAPoint(value, name) + } + def coordinates(s: Seq[_]): scala.collection.Map[String, Any] = + if (s.isEmpty) null + else if ( + (s.lengthCompare(2) == 0 || s.lengthCompare(3) == 0) && s.forall( + _.isInstanceOf[java.lang.Number] + ) + ) + point(s(1), s.head) + else if (s.lengthCompare(1) == 0) geoPoint(s.head, name, target) + else + throw new IllegalArgumentException( + s"Conversion Error: Unimplemented type for cast (STRUCT(lat DOUBLE, lon DOUBLE)[] -> " + + s"${duckDbStructName(target)}) when casting from source column $name" + ) + value match { + case null => null + case m: scala.collection.Map[_, _] => + val map = keyed(m) + (map.get("lat"), map.get("lon"), map.get("type"), map.get("coordinates")) match { + case (Some(lat), Some(lon), _, _) => point(lat, lon) + case (_, _, Some(t), Some(c: Seq[_])) if String.valueOf(t).equalsIgnoreCase("Point") => + coordinates(c) + case (_, _, Some(t), Some(c: java.util.List[_])) + if String.valueOf(t).equalsIgnoreCase("Point") => + coordinates(c.asScala.toList) + case _ => notAPoint(value, name) + } + case m: java.util.Map[_, _] => geoPoint(m.asScala, name, target) + case l: java.util.List[_] => coordinates(l.asScala.toList) + case s: Seq[_] => coordinates(s) + case s: String => + s match { + case LatLon(lat, lon) => point(lat, lon) + case WktPoint(lon, lat) => point(lat, lon) + case GeoHash() => geoHash(s.toLowerCase) + case _ => notAPoint(value, name) + } + case _ => notAPoint(value, name) + } + } + + private def number(v: Any): Option[Double] = + v match { + case n: java.lang.Number => Some(n.doubleValue()) + case s: String => scala.util.Try(s.trim.toDouble).toOption + case _ => None + } + + /** A geohash's cell, read as Elasticsearch reads it: its south-west corner (MEASURED, 8.18.3: + * `u09tv` is `48.8232421875, 2.3291015625`). + */ + private def geoHash(hash: String): scala.collection.Map[String, Any] = { + var lat = (-90.0, 90.0) + var lon = (-180.0, 180.0) + var even = true + hash.foreach { c => + val bits = Base32.indexOf(c) + Seq(16, 8, 4, 2, 1).foreach { mask => + if (even) { + val mid = (lon._1 + lon._2) / 2 + lon = if ((bits & mask) != 0) (mid, lon._2) else (lon._1, mid) + } else { + val mid = (lat._1 + lat._2) / 2 + lat = if ((bits & mask) != 0) (mid, lat._2) else (lat._1, mid) + } + even = !even + } + } + ListMap("lat" -> java.lang.Double.valueOf(lat._1), "lon" -> java.lang.Double.valueOf(lon._1)) + } + + private def notAPoint(value: Any, name: String): Nothing = + throw new IllegalArgumentException( + s"Conversion Error: Could not read '$value' as a GEO_POINT when casting from source column " + + s"$name: a point is an object {lat, lon}, a text 'lat,lon', an array [lon, lat], a geohash " + + "or a WKT POINT (lon lat)" + ) + + /** The conversion of a value of `source` (the type the branch declares, as + * [[MultiSearch.columnTypes]] folds it) to `target`, the column's: the value is first READ AS + * `source` ([[readAs]]: a text Elasticsearch accepted into the field is the value it indexed), + * then converted -- never from the class its JSON happened to give it. + * + * A `script_fields` value arrives wrapped in Elasticsearch's one-element array; it is unwrapped, + * since the column is ONE value per row, and an empty array, Elasticsearch's spelling of no + * value, is NULL. A value with SEVERAL values -- a multi-valued field, an object -- has no one + * value to convert, and FAILS the statement ([[nonScalarFailure]]). A scalar of a class the + * conversion does not expect is returned as it is. + */ + def converter(source: SQLType, target: SQLType): Any => Any = + converter( + source, + target, + s"Conversion Error: Unimplemented type for cast (${duckDbTypeName(source)}[] -> " + + s"${duckDbTypeName(target)})" + ) + + private[client] def converter(source: SQLType, target: SQLType, nonScalar: String): Any => Any = + converter(source, target, nonScalar, None, "", "", DateReading.unknown) + + /** [[converter]] for a branch that reads a date FIELD, whose mapping `format` -- its own, or + * Elasticsearch's default -- reads its values as the major `dates` names reads them + * ([[readAs]]); `field` is its path and `column` the column it reaches, which a failure names. + */ + private[client] def converter( + source: SQLType, + target: SQLType, + nonScalar: String, + format: Option[String], + field: String, + column: String, + dates: DateReading + ): Any => Any = { + val convert: Any => Any = target match { + case SQLTypes.Varchar => text(source) + case SQLTypes.Double => toDouble + case SQLTypes.Real => toFloat + case SQLTypes.BigInt => toLong + case SQLTypes.Int => toInt + case SQLTypes.SmallInt => toShort + case SQLTypes.TinyInt => toByte + case SQLTypes.Timestamp => toTimestamp + case SQLTypes.VarBinary => bytes(source) + case _ => identity + } + readAs(source, format, field, column, dates) match { + case None => scalarOf(convert, nonScalar) + case Some(read) => scalarOf(read.andThen(convert), nonScalar) + } + } + + /** How a value of a branch is READ AS the type the branch declares, before it is converted: as + * Elasticsearch indexed it -- what its queries and aggregations see -- whatever the JSON + * `_source` holds (MEASURED, field type by field type, `docvalue_fields` against `_source`): + * + * - an integer field (`BIGINT`, `INT`, `SMALLINT`, `TINYINT`) indexes a text or a fractional + * number truncated toward zero: `"5"`, `"5.0"`, `"5.5"`, `5.5` and `"5.5e0"` are 5, `"-5.9"` + * is -5, `"5e2"` is 500; an empty text is no value; + * - a `DOUBLE` or a `REAL` field indexes a text as its number: `"2.5"`, `" 2.5 "`, `"+2.5"`, + * `"1e3"` (1000.0), `"5"` (5.0); an empty text is no value; + * - a `BOOLEAN` field indexes `"true"` and `"false"`, and an empty text as `false`; + * - a `DATE`, `TIMESTAMP` or `TIME` FIELD indexes a value as its mapping `format` reads it -- + * its own, else Elasticsearch's default `strict_date_optional_time||epoch_millis` -- on the + * Elasticsearch major the statement runs on ([[dateByFormat]]): a `TIMESTAMP` is that + * instant, a `DATE` its day and a `TIME` its time of day, in UTC; + * - a computed `DATE` or `TIMESTAMP` (no field, so no format) reads digits (`"1706697000000"`, + * a number) as epoch milliseconds and an ISO text as its instant, a date-only or zone-less + * one in UTC. + * + * `None` for a type read as it comes. Decided once per leg; applied only where a position + * converts. + */ + private def readAs( + source: SQLType, + format: Option[String], + field: String, + column: String, + dates: DateReading + ): Option[Any => Any] = + source match { + case SQLTypes.Date | SQLTypes.Timestamp | SQLTypes.Time if format.isDefined => + Some(dateByFormat(source, format.get, field, column, dates)) + case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | SQLTypes.BigInt => Some(integral) + case SQLTypes.Double => Some(double) + case SQLTypes.Real => Some(real) + case SQLTypes.Boolean => Some(boolean) + case SQLTypes.Date => Some(date) + case SQLTypes.Timestamp => Some(timestamp) + case _ => None + } + + /** A value of a date field read with its mapping format `spec`, as the Elasticsearch major the + * statement runs on reads it ([[MappedDateFormat]], built ONCE per position and leg): a + * `TIMESTAMP` is the instant that major indexed, a `DATE` its day and a `TIME` its time of day, + * in UTC. A value the format does not read -- or one core cannot read without guessing, or any + * value when the major is not known -- FAILS the statement, naming the value, the field and the + * format: never a wrong date, never a value passed on unread. + */ + private def dateByFormat( + source: SQLType, + spec: String, + field: String, + column: String, + dates: DateReading + ): Any => Any = { + val typeName = source match { + case SQLTypes.Date => "DATE" + case SQLTypes.Time => "TIME" + case _ => "TIMESTAMP" + } + val result: Instant => Any = source match { + case SQLTypes.Date => _.atZone(ZoneOffset.UTC).toLocalDate + case SQLTypes.Time => _.atZone(ZoneOffset.UTC).toLocalTime + case _ => _.atZone(ZoneOffset.UTC) + } + def failure(value: Any, detail: String): Nothing = + throw new IllegalArgumentException( + s"Conversion Error: Could not read '$value' as a $typeName when casting from source " + + s"column $column: $detail" + ) + dates.dialect match { + case None => + val detail = + s"the date format '$spec' of field $field reads it as the Elasticsearch version does, " + + dates.elasticVersion.fold("which is not known")(v => s"and core reads dates of no $v") + value => failure(value, detail) + case Some(dialect) => + val format = MappedDateFormat(spec, dialect, dates.clock) + value => + format.readValue(value) match { + case MappedDateFormat.Read(instant) => result(instant) + case MappedDateFormat.Rejected => + failure( + value, + s"the date format '$spec' of field $field does not read it as ${dialect.label} does" + ) + case MappedDateFormat.Undecidable(reason) => + failure( + value, + s"core cannot read it with the date format '$spec' of field $field as " + + s"${dialect.label} does: $reason" + ) + } + } + } + + private val EpochMillis = """-?\d+""".r + + private def decimal(text: String): Option[java.math.BigDecimal] = + scala.util.Try(new java.math.BigDecimal(text)).toOption + + private val integral: Any => Any = { + case s: String => + val t = s.trim + if (t.isEmpty) null + else decimal(t).fold[Any](s)(d => java.lang.Long.valueOf(d.longValue())) + case d @ (_: java.lang.Double | _: java.lang.Float | _: java.math.BigDecimal) => + java.lang.Long.valueOf(d.asInstanceOf[java.lang.Number].longValue()) + case other => other + } + + private val double: Any => Any = { + case s: String => + val t = s.trim + if (t.isEmpty) null + else decimal(t).fold[Any](s)(d => java.lang.Double.valueOf(d.doubleValue())) + case other => other + } + + private val real: Any => Any = { + case s: String => + val t = s.trim + if (t.isEmpty) null + else decimal(t).fold[Any](s)(d => java.lang.Float.valueOf(d.floatValue())) + case other => other + } + + private val boolean: Any => Any = { + case "true" => java.lang.Boolean.TRUE + case "false" | "" => java.lang.Boolean.FALSE + case other => other + } + + private val timestamp: Any => Any = { + case s: String if EpochMillis.pattern.matcher(s).matches() => + Instant.ofEpochMilli(s.toLong).atZone(ZoneOffset.UTC) + case other => toTimestamp(other) + } + + private val date: Any => Any = { + case d: LocalDate => d + case other => + timestamp(other) match { + case z: ZonedDateTime => z.withZoneSameInstant(ZoneOffset.UTC).toLocalDate + case unread => unread + } + } + + /** A value reaching a `VARBINARY` column (DuckDB's `BLOB`) as BYTES -- `Array[Byte]`, the class + * of core's `VARBINARY` cell (`ValueCoercion.inferType`, `JavaValueConversion`). + * + * - A `VARBINARY` value is the bytes of the text Elasticsearch holds for it, a `binary` + * field's base64, in UTF-8: the bytes the relational engine is handed for that field + * (arrow's `VarBinaryVector`) and the bytes the JDBC driver's `getBytes` answers. + * - A text value is the bytes DuckDB's cast from `VARCHAR` to `BLOB` makes of it + * ([[blobOfText]]). + */ + private def bytes(source: SQLType): Any => Any = + source match { + case SQLTypes.VarBinary => { + case null => null + case b: Array[Byte] => b + case other => other.toString.getBytes(StandardCharsets.UTF_8) + } + case _ => { + case null => null + case b: Array[Byte] => b + case other => blobOfText(textOf(other)) + } + } + + /** DuckDB 1.5.5's cast of a `VARCHAR` to a `BLOB`, MEASURED value by value: every ASCII character + * is its byte, `\xHH` (two hexadecimal digits) is the byte `HH`, and the cast FAILS -- the + * statement with it, as DuckDB's does -- on a character outside ASCII, on a backslash followed + * by anything else, and on a backslash with fewer than three bytes after it. The text is read as + * its UTF-8 bytes, left to right, and the first fault met names the failure, in DuckDB's words. + */ + private[client] def blobOfText(text: String): Array[Byte] = { + val in = text.getBytes(StandardCharsets.UTF_8) + val out = new java.io.ByteArrayOutputStream(in.length) + def hex(b: Byte): Int = if (b < 0) -1 else Character.digit(b.toInt, 16) + def escapeFault(detail: String) = + new IllegalArgumentException( + "Conversion Error: Invalid hex escape code encountered in string -> blob conversion of " + + "string \"" + text + "\": " + detail + ) + var i = 0 + while (i < in.length) { + val b = in(i) + if (b == '\\'.toByte) { + if (in.length - i - 1 < 3) throw escapeFault("unterminated escape code at end of blob") + val high = hex(in(i + 2)) + val low = hex(in(i + 3)) + if (in(i + 1) != 'x'.toByte || high < 0 || low < 0) + throw escapeFault(new String(in, i, 4, StandardCharsets.UTF_8)) + out.write(high * 16 + low) + i += 4 + } else if (b < 0) { + // a byte of a character outside ASCII + throw new IllegalArgumentException( + "Conversion Error: Invalid byte encountered in STRING -> BLOB conversion of string \"" + + text + "\". All non-ascii characters must be escaped with hex codes (e.g. \\xAA)" + ) + } else { + out.write(b.toInt) + i += 1 + } + } + out.toByteArray + } + + /** Applies `f` to the ONE value a row holds: a one-element array unwrapped, an empty one NULL, + * and a value of several values -- an array of more, a map -- the statement's failure, + * `nonScalar`. + */ + private def scalarOf(f: Any => Any, nonScalar: String): Any => Any = { + def one(value: Any): Any = + value match { + case null => null + case s: Seq[_] => + if (s.isEmpty) null + else if (s.lengthCompare(1) == 0) one(s.head) + else throw new IllegalArgumentException(nonScalar) + case l: java.util.List[_] => + if (l.isEmpty) null + else if (l.size() == 1) one(l.get(0)) + else throw new IllegalArgumentException(nonScalar) + case _: scala.collection.Map[_, _] | _: java.util.Map[_, _] => + throw new IllegalArgumentException(nonScalar) + case v => f(v) + } + one + } + + /** A text branch's value as its text. A value core re-read as a temporal when the response was + * parsed (issue #418) is spelled back by the ISO formatters, which keep its seconds -- + * `10:30:00`, `2024-01-31T10:30:00Z` -- where `toString` drops them (`10:30`). Bytes are their + * base64 text, the text Elasticsearch holds for a `binary` field. + */ + private def textOf(value: Any): String = + value match { + case s: String => s + case t: LocalTime => DateTimeFormatter.ISO_LOCAL_TIME.format(t) + case t: OffsetTime => DateTimeFormatter.ISO_OFFSET_TIME.format(t) + case d: LocalDateTime => DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(d) + case z: ZonedDateTime => DateTimeFormatter.ISO_ZONED_DATE_TIME.format(z) + case o: OffsetDateTime => DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(o) + case i: Instant => DateTimeFormatter.ISO_INSTANT.format(i) + case b: Array[Byte] => java.util.Base64.getEncoder.encodeToString(b) + case other => other.toString + } + + private val toDouble: Any => Any = { + case null => null + case d: java.lang.Double => d + case n: java.lang.Number => java.lang.Double.valueOf(n.doubleValue()) + case b: java.lang.Boolean => java.lang.Double.valueOf(if (b) 1.0 else 0.0) + case other => other + } + + private val toFloat: Any => Any = { + case null => null + case f: java.lang.Float => f + case n: java.lang.Number => java.lang.Float.valueOf(n.floatValue()) + case b: java.lang.Boolean => java.lang.Float.valueOf(if (b) 1.0f else 0.0f) + case other => other + } + + private val toLong: Any => Any = { + case null => null + case l: java.lang.Long => l + case n: java.lang.Number => java.lang.Long.valueOf(n.longValue()) + case b: java.lang.Boolean => java.lang.Long.valueOf(if (b) 1L else 0L) + case other => other + } + + private val toInt: Any => Any = { + case null => null + case i: java.lang.Integer => i + case n: java.lang.Number => java.lang.Integer.valueOf(n.intValue()) + case b: java.lang.Boolean => java.lang.Integer.valueOf(if (b) 1 else 0) + case other => other + } + + private val toShort: Any => Any = { + case null => null + case s: java.lang.Short => s + case n: java.lang.Number => java.lang.Short.valueOf(n.shortValue()) + case b: java.lang.Boolean => java.lang.Short.valueOf(if (b) 1.toShort else 0.toShort) + case other => other + } + + private val toByte: Any => Any = { + case null => null + case b: java.lang.Byte => b + case n: java.lang.Number => java.lang.Byte.valueOf(n.byteValue()) + case b: java.lang.Boolean => java.lang.Byte.valueOf(if (b) 1.toByte else 0.toByte) + case other => other + } + + /** A `DATE` becomes the instant its day starts at, in UTC -- DuckDB's `CAST(DATE AS TIMESTAMP)`, + * and the reading every date rule of the engine already uses; a `TIMESTAMP` keeps its instant. + */ + private val toTimestamp: Any => Any = { + case null => null + case z: ZonedDateTime => z + case d: LocalDate => d.atStartOfDay(ZoneOffset.UTC) + case l: LocalDateTime => l.atZone(ZoneOffset.UTC) + case o: OffsetDateTime => o.toZonedDateTime + case i: Instant => i.atZone(ZoneOffset.UTC) + case n: java.lang.Number => Instant.ofEpochMilli(n.longValue()).atZone(ZoneOffset.UTC) + case other => other + } + + /** A value as TEXT, spelled as DuckDB 1.5.5 spells its type when it casts it to `VARCHAR`: text, + * a `BOOLEAN` (`true` / `false`), a whole number, a `DOUBLE` or a `REAL` ([[FloatingPointText]]: + * `250000000000.0`, `1e-05`), a `DATE` (`2024-01-31`), a `TIMESTAMP` (`2024-01-31 10:30:00`, + * `2023-12-25 23:59:59.999`) and a `TIME` (`10:30:00`). A temporal is read in UTC, as every date + * rule of the engine reads it. A value Elasticsearch returns as text stays that text: a + * `VARBINARY` value is its base64 text. + */ + private def text(source: SQLType): Any => Any = + source match { + case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | SQLTypes.BigInt => { + case s: String => s + case n: java.lang.Number => java.lang.Long.toString(n.longValue()) + case other => other.toString + } + // a number of unknown width is a DOUBLE to DuckDB: the relational engine loads it as one + case SQLTypes.Double | SQLTypes.Numeric => doubleText + case SQLTypes.Real => realText + case SQLTypes.Date => dateText + case SQLTypes.Timestamp => timestampText + case SQLTypes.Time => timeText + // a temporal of unknown kind is spelled as the kind its value is + case SQLTypes.Temporal => { + case d: LocalDate => dateText(d) + case t: LocalTime => timeText(t) + case other => timestampText(other) + } + // text, a BOOLEAN (`true` / `false`, DuckDB's spelling too) + case _ => textOf + } + + private val doubleText: Any => Any = { + case null => null + case s: String => s + case n: java.lang.Number => FloatingPointText.double(n.doubleValue()) + case other => other.toString + } + + private val realText: Any => Any = { + case null => null + case s: String => s + case n: java.lang.Number => FloatingPointText.real(n.floatValue()) + case other => other.toString + } + + private val dateText: Any => Any = { + case null => null + case s: String => s + case d: LocalDate => TemporalText.date(d) + case z: ZonedDateTime => TemporalText.date(z.withZoneSameInstant(ZoneOffset.UTC).toLocalDate) + case o: OffsetDateTime => TemporalText.date(o.atZoneSameInstant(ZoneOffset.UTC).toLocalDate) + case l: LocalDateTime => TemporalText.date(l.toLocalDate) + case i: Instant => TemporalText.date(i.atZone(ZoneOffset.UTC).toLocalDate) + case n: java.lang.Number => + TemporalText.date(Instant.ofEpochMilli(n.longValue()).atZone(ZoneOffset.UTC).toLocalDate) + case other => other.toString + } + + /** A `DATE` value (a date-only timestamp) is the instant its day starts at, as [[toTimestamp]] + * reads it. + */ + private val timestampText: Any => Any = { + case null => null + case s: String => s + case l: LocalDateTime => TemporalText.timestamp(l) + case z: ZonedDateTime => + TemporalText.timestamp(z.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime) + case o: OffsetDateTime => + TemporalText.timestamp(o.atZoneSameInstant(ZoneOffset.UTC).toLocalDateTime) + case d: LocalDate => TemporalText.timestamp(d.atStartOfDay) + case i: Instant => TemporalText.timestamp(LocalDateTime.ofInstant(i, ZoneOffset.UTC)) + case n: java.lang.Number => + TemporalText.timestamp( + LocalDateTime.ofInstant(Instant.ofEpochMilli(n.longValue()), ZoneOffset.UTC) + ) + case other => other.toString + } + + private val timeText: Any => Any = { + case null => null + case s: String => s + case t: LocalTime => TemporalText.time(t) + case o: OffsetTime => TemporalText.time(o.withOffsetSameInstant(ZoneOffset.UTC).toLocalTime) + case l: LocalDateTime => TemporalText.time(l.toLocalTime) + case z: ZonedDateTime => TemporalText.time(z.withZoneSameInstant(ZoneOffset.UTC).toLocalTime) + case o: OffsetDateTime => TemporalText.time(o.atZoneSameInstant(ZoneOffset.UTC).toLocalTime) + case other => other.toString + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemporalText.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemporalText.scala new file mode 100644 index 00000000..1460a848 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemporalText.scala @@ -0,0 +1,122 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * TemporalText follows the text layout of DuckDB 1.5.5 + * (https://github.com/duckdb/duckdb/tree/v1.5.5) for a DATE, a TIME and a TIMESTAMP cast to + * VARCHAR, whose logic it translates: `DateToStringCast` (YearLength, Format) and + * `TimeToStringCast` (FormatMicros, MicrosLength, Format) of + * src/include/duckdb/common/types/cast_helpers.hpp, and `StringFromTimestamp` / `StringAsTime` of + * src/common/operator/string_cast.cpp -- the year on at least four digits, a year before 1 written + * as its BC year followed by " (BC)", and the fraction of a second without its trailing zeros. + * DuckDB is distributed under the following license: + * + * Copyright 2018-2025 Stichting DuckDB Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and + * associated documentation files (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT + * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package app.softnetwork.elastic.client + +import java.time.{LocalDate, LocalDateTime, LocalTime} + +/** A `DATE`, a `TIMESTAMP` and a `TIME` as TEXT, spelled as DuckDB 1.5.5 casts them to `VARCHAR` + * (`DateToStringCast` / `TimeToStringCast`): the year on at least four digits, `(BC)` after the + * date of a year before 1 (`0044-03-15 (BC)`), a timestamp's date and time separated by a space, + * and the fraction of a second only when there is one, without its trailing zeros (`23:59:59.999`, + * `10:30:00.5`). MEASURED on DuckDB 1.5.5, every case. + */ +private[client] object TemporalText { + + def date(d: LocalDate): String = { + val text = new java.lang.StringBuilder(16) + appendDate(text, d.getYear, d.getMonthValue, d.getDayOfMonth) + text.toString + } + + def timestamp(t: LocalDateTime): String = { + val text = new java.lang.StringBuilder(32) + appendDate(text, t.getYear, t.getMonthValue, t.getDayOfMonth) + text.append(' ') + appendTime(text, t.getHour, t.getMinute, t.getSecond, t.getNano) + text.toString + } + + def time(t: LocalTime): String = { + val text = new java.lang.StringBuilder(18) + appendTime(text, t.getHour, t.getMinute, t.getSecond, t.getNano) + text.toString + } + + private def appendDate(text: java.lang.StringBuilder, year: Int, month: Int, day: Int): Unit = { + // DuckDB's year is proleptic, as java.time's: year 0 is 1 BC + val bc = year <= 0 + val digits = Integer.toString(if (bc) 1 - year else year) + var padding = 4 - digits.length + while (padding > 0) { + text.append('0') + padding -= 1 + } + text.append(digits).append('-') + appendTwoDigits(text, month) + text.append('-') + appendTwoDigits(text, day) + if (bc) text.append(" (BC)") + } + + private def appendTime( + text: java.lang.StringBuilder, + hour: Int, + minute: Int, + second: Int, + nano: Int + ): Unit = { + appendTwoDigits(text, hour) + text.append(':') + appendTwoDigits(text, minute) + text.append(':') + appendTwoDigits(text, second) + if (nano != 0) { + // nine digits, the trailing zeros dropped: `.5`, `.999`, `.123456789` + val fraction = Integer.toString(nano) + var last = fraction.length + while (fraction.charAt(last - 1) == '0') last -= 1 + text.append('.') + var padding = 9 - fraction.length + while (padding > 0) { + text.append('0') + padding -= 1 + } + text.append(fraction, 0, last) + } + } + + private def appendTwoDigits(text: java.lang.StringBuilder, value: Int): Unit = + text.append(('0' + value / 10).toChar).append(('0' + value % 10).toChar) +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala index ad6c4918..6e178179 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -314,12 +314,17 @@ package object client extends SerializationApi { * built its request. A plan made once from the branches' MERGED map (which keeps the LAST * branch's aggregation per name) told a branch to read an aggregation its request had merged * into another, or never emitted: the column came back NULL with HTTP 200. + * @param converters + * aligned with `fields`: how the BRANCH's value at each position reaches the type the result + * gives that column (`SetOperationValues`), `null` where it already has it -- `None` for a + * branch with nothing to convert, which is every branch of a statement whose columns agree */ case class LegProjection( fieldAliases: ListMap[String, String], fields: Seq[String], nestedHits: Map[String, Seq[(String, String)]], - aggregations: ListMap[String, ClientAggregation] + aggregations: ListMap[String, ClientAggregation], + converters: Option[Array[Any => Any]] = None ) /** Retry configuration diff --git a/core/src/test/resources/date-formats/es6.tsv.gz b/core/src/test/resources/date-formats/es6.tsv.gz new file mode 100644 index 00000000..0ffc15e3 Binary files /dev/null and b/core/src/test/resources/date-formats/es6.tsv.gz differ diff --git a/core/src/test/resources/date-formats/es7.tsv.gz b/core/src/test/resources/date-formats/es7.tsv.gz new file mode 100644 index 00000000..8c2d39c1 Binary files /dev/null and b/core/src/test/resources/date-formats/es7.tsv.gz differ diff --git a/core/src/test/resources/date-formats/es8.tsv.gz b/core/src/test/resources/date-formats/es8.tsv.gz new file mode 100644 index 00000000..f89faebb Binary files /dev/null and b/core/src/test/resources/date-formats/es8.tsv.gz differ diff --git a/core/src/test/scala/app/softnetwork/elastic/client/FloatingPointTextSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/FloatingPointTextSpec.scala new file mode 100644 index 00000000..e1506170 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/FloatingPointTextSpec.scala @@ -0,0 +1,127 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** A `DOUBLE` or a `REAL` inside a `VARCHAR` column of a set operation is spelled as DuckDB 1.5.5 + * casts it to `VARCHAR` (the lead's ruling of 2026-10-06). Every expectation below is DuckDB's + * answer to `CAST( AS VARCHAR)`, MEASURED value by value -- the layout (`250000000000.0`, + * `1e-05`, `1e+16`), the digits, and the values where DuckDB's own algorithm does not find the + * shortest digits (`7.1680000000000004e+25`, the float `4073260.75`), which are its answer too. + */ +class FloatingPointTextSpec extends AnyFlatSpec with Matchers { + + /** The value (Python's shortest repr, read back exactly by `parseDouble`) -> DuckDB's text. */ + private val doubles: Seq[(String, String)] = Seq( + "250000000000.0" -> "250000000000.0", + "1e-05" -> "1e-05", + "1e+16" -> "1e+16", + "1000000000000000.0" -> "1000000000000000.0", + "0.0001" -> "0.0001", + "-2.5" -> "-2.5", + "-0.0" -> "-0.0", + "0.0" -> "0.0", + "1.0" -> "1.0", + "100.0" -> "100.0", + "123.456" -> "123.456", + "0.30000000000000004" -> "0.30000000000000004", + "0.3333333333333333" -> "0.3333333333333333", + "1e+22" -> "1e+22", + "1e+23" -> "1e+23", + "9007199254740992.0" -> "9007199254740992.0", + "9999999999999998.0" -> "9999999999999998.0", + "5e-324" -> "5e-324", + "2.2250738585072014e-308" -> "2.2250738585072014e-308", + "1.7976931348623157e+308" -> "1.7976931348623157e+308", + "7.168e+25" -> "7.1680000000000004e+25", + "3.584e+25" -> "3.5840000000000002e+25", + "-250000000000.0" -> "-250000000000.0", + "-1e-05" -> "-1e-05", + "-1e+16" -> "-1e+16", + "1.5e+300" -> "1.5e+300", + "2.82879384806159e+17" -> "2.82879384806159e+17", + "8.41e+21" -> "8.41e+21", + "0.002" -> "0.002", + "4.35" -> "4.35", + "12345.678" -> "12345.678", + "2.5e-05" -> "2.5e-05", + "1.2345e-07" -> "1.2345e-07", + "3.14159" -> "3.14159", + "-123456.789" -> "-123456.789", + "1.2345678901234568e+17" -> "1.2345678901234568e+17", + "5.0" -> "5.0", + "0.1" -> "0.1", + "1e-300" -> "1e-300", + "3e-310" -> "3e-310" + ) + + /** The float (its shortest float32 repr, read back exactly by `parseFloat`) -> DuckDB's text. */ + private val floats: Seq[(String, String)] = Seq( + "1.e-01" -> "0.1", + "2.5e+00" -> "2.5", + "1.e-05" -> "1e-05", + "1.6777216e+07" -> "16777216.0", + "3.4028235e+38" -> "3.4028235e+38", + "1.e-45" -> "1e-45", + "-0.0" -> "-0.0", + "0.0" -> "0.0", + "3.e-01" -> "0.3", + "3.3333334e-01" -> "0.33333334", + "1.e+02" -> "100.0", + "1.e+16" -> "1e+16", + "1.e+15" -> "1000000000000000.0", + "1.e-04" -> "0.0001", + "1.23456e+02" -> "123.456", + "-2.5e+00" -> "-2.5", + "2.5e+11" -> "250000000000.0", + "1.2345679e-01" -> "0.12345679", + "1.1754944e-38" -> "1.1754944e-38", + "7.168e+25" -> "7.168e+25", + "4.0732608e+06" -> "4073260.75", + "-3.614988e+08" -> "-361498816.0", + "-3.9489588e+06" -> "-3948958.75", + "3.2423772e+06" -> "3242377.25", + "5.641915e+07" -> "56419152.0", + "-7.5e-01" -> "-0.75", + "1.25e+00" -> "1.25", + "5.e-01" -> "0.5" + ) + + "A DOUBLE" should "be spelled as DuckDB 1.5.5 casts it to VARCHAR" in { + val wrong = doubles.flatMap { case (value, duck) => + val got = FloatingPointText.double(java.lang.Double.parseDouble(value)) + if (got == duck) None else Some(s"$value: $got instead of $duck") + } + wrong shouldBe empty + } + + it should "spell the values no column stores as DuckDB does" in { + FloatingPointText.double(Double.NaN) shouldBe "nan" + FloatingPointText.double(Double.PositiveInfinity) shouldBe "inf" + FloatingPointText.double(Double.NegativeInfinity) shouldBe "-inf" + } + + "A REAL" should "be spelled as DuckDB 1.5.5 casts it to VARCHAR, from the float's own value" in { + val wrong = floats.flatMap { case (value, duck) => + val got = FloatingPointText.real(java.lang.Float.parseFloat(value)) + if (got == duck) None else Some(s"$value: $got instead of $duck") + } + wrong shouldBe empty + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappedDateFormatSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappedDateFormatSpec.scala new file mode 100644 index 00000000..e66c7221 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappedDateFormatSpec.scala @@ -0,0 +1,301 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.query.TemporalLiterals +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.{Clock, Instant, ZoneOffset, ZonedDateTime} +import java.util.zip.GZIPInputStream +import scala.collection.immutable.ListMap +import scala.io.Source +import scala.util.{Failure, Success, Try} + +/** A date field's values read with its mapping format, as each Elasticsearch major reads it -- + * driven by the corpora MEASURED on Elasticsearch 6.8.23, 7.17.29 and 8.18.3 + * (`date-formats/es.tsv.gz`): every value the major ACCEPTED into a field of each format, + * as written in the document, and the instant it indexed -- each pattern letter at each width, + * alone and combined, every built-in name, epochs and JSON numbers. Each value goes the way core + * receives it: the JSON read by the response parser (an ISO text already a date there), then the + * converter of a converting `TIMESTAMP` position. A value core cannot read without guessing (the + * row's 4th column says why) must FAIL its statement, naming the field and the format. + */ +class MappedDateFormatSpec extends AnyFlatSpec with Matchers { + + /** The day the corpora were measured: Joda's two-digit years depend on it. */ + private val measuredOn = Clock.fixed(Instant.parse("2026-10-07T12:00:00Z"), ZoneOffset.UTC) + + /** A row: the format (empty: none), the value as JSON, the instant indexed, and why core does not + * read it, if it does not. + */ + private def corpus(major: String): Seq[(String, String, Long, Option[String])] = { + val source = Source.fromInputStream( + new GZIPInputStream(getClass.getResourceAsStream(s"/date-formats/$major.tsv.gz")), + "UTF-8" + ) + try source + .getLines() + .filterNot(_.startsWith("#")) + .map { line => + line.split("\t", -1) match { + case Array(format, json, millis) => (format, json, millis.toLong, None) + case Array(format, json, millis, cause) => (format, json, millis.toLong, Some(cause)) + case other => throw new IllegalStateException(s"a corpus row: ${other.mkString("|")}") + } + } + .toList + finally source.close() + } + + /** A document's value as core receives it: its JSON, read by the response parser. */ + private def received(json: String): Any = + ElasticConversion.jsonNodeToAny(JacksonConfig.objectMapper.readTree(json), ListMap.empty) + + private def readsEveryAcceptedValue( + major: String, + version: String, + size: Int, + unread: Int + ): Unit = { + val rows = corpus(major) + rows.size shouldBe size + rows.count(_._4.isDefined) shouldBe unread + val dates = new SetOperationValues.DateReading(Some(version), measuredOn) + val converters = scala.collection.mutable.Map.empty[String, Any => Any] + val wrong = rows.flatMap { case (format, json, millis, cause) => + val spec = if (format.isEmpty) TemporalLiterals.DefaultDateFormat else format + val convert = converters.getOrElseUpdate( + spec, + SetOperationValues.converter( + SQLTypes.Timestamp, + SQLTypes.Timestamp, + "multi", + Some(spec), + "v", + "k", + dates + ) + ) + (cause, Try(convert(received(json)))) match { + case (None, Success(z: ZonedDateTime)) if z.toInstant.toEpochMilli == millis => None + case (Some(_), Failure(e: IllegalArgumentException)) + if e.getMessage.contains("field v") && e.getMessage.contains(s"'$spec'") => + None + case (_, other) => + Some( + s"[$format] $json: indexed ${Instant.ofEpochMilli(millis)}" + + cause.fold("")(c => s" (expected to fail: $c)") + s", read $other" + ) + } + } + withClue(wrong.take(50).mkString("\n", "\n", "\n"))(wrong shouldBe empty) + } + + "Elasticsearch 8.18.3's date formats" should + "read every value it accepted as the instant it indexed" in + readsEveryAcceptedValue("es8", "8.18.3", 36109, 3512) + + "Elasticsearch 7.17.29's date formats" should + "read every value it accepted as the instant it indexed" in + readsEveryAcceptedValue("es7", "7.17.29", 47065, 5351) + + "Elasticsearch 6.8.23's date formats" should + "read every value it accepted as the instant it indexed" in + readsEveryAcceptedValue("es6", "6.8.23", 51515, 5755) + + private def read(spec: String, version: String, value: Any): Try[Any] = + Try( + SetOperationValues.converter( + SQLTypes.Timestamp, + SQLTypes.Timestamp, + "multi", + Some(spec), + "v", + "k", + new SetOperationValues.DateReading(Some(version), measuredOn) + )(value) + ) + + private def instant(t: Try[Any]): Instant = + t match { + case Success(z: ZonedDateTime) => z.toInstant + case other => fail(s"not a date: $other") + } + + "A week-based year (`YYYY`)" should "be read as Elasticsearch 8 indexed it, not as 1970" in { + // MEASURED on 8.18.3: the week-based year's first week, starting on Sunday -- the month and the + // day ignored -- where round 11 read 1970-01-31, 1970-12-31, 1970-01-01, 1970-12-30 + Seq( + "2024-01-31" -> "2023-12-31T00:00:00Z", + "2020-12-31" -> "2019-12-29T00:00:00Z", + "2021-01-01" -> "2020-12-27T00:00:00Z", + "2019-12-30" -> "2018-12-30T00:00:00Z" + ).foreach { case (value, indexed) => + instant(read("YYYY-MM-dd", "8.18.3", value)) shouldBe Instant.parse(indexed) + } + instant(read("YYYY-MM-dd'T'HH:mm:ss", "8.18.3", "2024-06-15T10:30:00")) shouldBe + Instant.parse("2023-12-31T10:30:00Z") + instant(read("YYYY-MM", "8.18.3", "2024-01")) shouldBe Instant.parse("2023-12-31T00:00:00Z") + // Elasticsearch 7 starts the week on Monday (ISO); Elasticsearch 6's `YYYY` is the year of era + instant(read("YYYY-MM-dd", "7.17.29", "2024-01-31")) shouldBe + Instant.parse("2024-01-01T00:00:00Z") + instant(read("YYYY-MM-dd", "6.8.23", "2024-01-31")) shouldBe + Instant.parse("2024-01-31T00:00:00Z") + } + + "A pattern letter, or a combination of letters, no measurement covers" should + "fail the statement, naming the field and the format" in { + def failure(spec: String, version: String, value: Any): String = + read(spec, version, value) match { + case Failure(e: IllegalArgumentException) => e.getMessage + case other => fail(s"not a failure: $other") + } + // a letter (`B`, day periods) no measurement covers + failure("yyyy-MM-dd B", "8.18.3", "2024-01-31 in the morning") should endWith( + "core cannot read it with the date format 'yyyy-MM-dd B' of field v as Elasticsearch 8 " + + "does: 'yyyy-MM-dd B' is not a date format core reads as Elasticsearch 8 reads it: the " + + "letter 'B' is not one core has measured" + ) + // a width no measurement covers + failure("yyyyyyyy-MM-dd", "7.17.29", "2024-01-31") should endWith( + "the letter 'yyyyyyyy' is not one core has measured" + ) + failure("yyyy-MM-dd zzzzz", "6.8.23", "2024-01-31 UTC") should endWith( + "the letter 'zzzzz' is not one core has measured" + ) + // a field no measured completion places: an hour of the day with a clock hour of AM/PM + failure("yyyy-MM-dd HH hh", "8.18.3", "2024-01-31 10 10") should endWith( + "the letters of 'yyyy-MM-dd HH hh' combine as 'yMd|hH', which core has not measured" + ) + failure("yyyy-MM-dd HH hh", "6.8.23", "2024-01-31 10 10") should endWith( + "its letters combine as 'yMd|hH|', which core has not measured" + ) + // a zone name, whose offset comes from the zone data of the JVM reading it + failure("yyyy-MM-dd'T'HH:mm:ss z", "8.18.3", "2024-01-31T10:30:00 CET") should endWith( + "the zone 'CET' is a region id or a zone name, whose offset comes from the zone data of the " + + "JVM reading it, not Elasticsearch's: core reads only Z, an offset, UTC, GMT and UT" + ) + } + + private def failure(spec: String, version: String, value: Any): String = + read(spec, version, value) match { + case Failure(e: IllegalArgumentException) + if e.getMessage.contains("field v") && e.getMessage.contains(s"'$spec'") => + e.getMessage + case other => fail(s"[$spec] $value on $version: not a failure naming the field: $other") + } + + private def received(json: String, spec: String, version: String): Try[Any] = + read(spec, version, received(json)) + + "A zone id or a zone name other than UTC, GMT and UT" should + "fail the statement on every major, naming the field and the format" in { + // a zone's offset comes from zone data, the reading JVM's: MEASURED, these read a different + // instant on the JVMs core runs on than on the Elasticsearch that indexed them (the Elasticsearch + // 7 one, which no measurement covers, fails as well) + val regions = Seq( + // a `java.time` id (`VV`): 2023-07-01T17:00:00Z on 8.18.3, 18:00Z with an older zone data + ("yyyy-MM-dd'T'HH:mm:ss VV", "8.18.3", "\"2023-07-01T12:00:00 America/Mexico_City\""), + ("yyyy-MM-dd'T'HH:mm:ss VV", "7.17.29", "\"2023-07-01T12:00:00 Asia/Tehran\""), + // the default format: 2025-07-01T16:00:00Z on 8.18.3, 15:00Z with Java 17's zone data + ("", "8.18.3", "\"2025-07-01T12:00:00America/Asuncion\""), + // a region in brackets, which the response parser reads with the JVM's zone data: 8.18.3 + // indexed the local time in the region (08:30Z, 10:30Z), the parser gives the offset's + ("yyyy-MM-dd'T'HH:mm:ssXXX'['VV']'", "8.18.3", "\"2024-07-31T10:30:00+05:00[Europe/Paris]\""), + ("yyyy-MM-dd'T'HH:mm:ssXXX'['VV']'", "8.18.3", "\"2024-07-31T10:30:00+05:00[UTC]\""), + // Joda: a zone id (`ZZZ`), an abbreviation (`z`) -- 17:00Z and 14:00Z on 6.8.23 + ("yyyy-MM-dd HH:mm:ss ZZZ", "6.8.23", "\"2023-07-01 12:00:00 America/Mexico_City\""), + ("yyyy-MM-dd HH:mm:ss ZZZ", "6.8.23", "\"2024-01-15 12:00:00 America/Nuuk\""), + ("yyyy-MM-dd HH:mm:ss ZZZ", "6.8.23", "\"2024-01-31 10:30:00 Etc/GMT-3\""), + ("yyyy-MM-dd HH:mm:ss z", "6.8.23", "\"2024-07-31 10:30:00 PDT\"") + ) + regions.foreach { case (format, version, json) => + val spec = if (format.isEmpty) TemporalLiterals.DefaultDateFormat else format + received(json, spec, version) match { + case Failure(e: IllegalArgumentException) + if e.getMessage.contains("field v") && e.getMessage.contains(s"'$spec'") && + e.getMessage.contains("is a region id or a zone name") => + () + case other => fail(s"[$spec] $json on $version: $other") + } + } + // `UTC`, `GMT`, `UT` and an offset are read, the same on every JVM (MEASURED) + instant(received("\"2024-01-31 10:30:00 UTC\"", "yyyy-MM-dd HH:mm:ss VV", "8.18.3")) shouldBe + Instant.parse("2024-01-31T10:30:00Z") + instant(received("\"2024-01-31 10:30:00 UTC\"", "yyyy-MM-dd HH:mm:ss ZZZ", "6.8.23")) shouldBe + Instant.parse("2024-01-31T10:30:00Z") + instant(received("\"2024-01-31 10:30:00 UTC\"", "yyyy-MM-dd HH:mm:ss z", "6.8.23")) shouldBe + Instant.parse("2024-01-31T10:30:00Z") + instant( + received("\"2025-07-01T12:00:00-04:00\"", TemporalLiterals.DefaultDateFormat, "8.18.3") + ) shouldBe Instant.parse("2025-07-01T16:00:00Z") + } + + "A pattern whose reading depends on the Java version core runs on" should + "fail the statement on every Java version, naming the field and the format" in { + // MEASURED on Java 8, 11, 17 and 21: a localized offset is not parsed by Java 8, a fraction + // after a variable width keeps its digits from Java 9 on + failure("yyyy-MM-dd'T'HH:mm:ss O", "8.18.3", "2024-01-31T10:30:00 GMT+1") should endWith( + "the letter 'O' is a localized offset (`GMT+1`), which Java 8 parses differently from later " + + "Java versions" + ) + failure("yyyy-MM-dd ZZZZ", "7.17.29", "2018-12-30 GMT+01:00") should include( + "the letter 'ZZZZ' is a localized offset" + ) + failure("yyyyMMddHHmmssSSS", "8.18.3", "20181230103000000") should endWith( + "the fraction 'SSS' follows digits of a variable width with no separator, which Java 8 " + + "parses differently from later Java versions" + ) + failure("8yyyyMMddHHmmssSSS", "6.8.23", "20181230103000000") should include( + "the fraction 'SSS' follows digits of a variable width" + ) + // Elasticsearch 6.8's java.time mode resolves SMARTLY with its own JVM (Java 15): an AM/PM + // with no hour is ignored there -- 06:00 / 18:00 from Java 16 on (MEASURED: `8yyyy a` "2025 PM" + // indexed 2025-01-01T00:00Z) -- and a week past its week-based year's last is read into the + // next year, which Java 21 rejects (`8YYYY-ww-e` "2020-53-7" indexed 2020-12-26T00:00Z) + failure("8yyyy a", "6.8.23", "2025 PM") should endWith( + "'yyyy a' reads an AM/PM with no hour, which Elasticsearch 6's java.time mode resolves " + + "differently from one Java version to another" + ) + failure("8dd.MM.yyyy a", "6.8.23", "04.07.2024 PM") should include("an AM/PM with no hour") + failure("8YYYY-ww-e", "6.8.23", "2020-53-7") should endWith( + "'YYYY-ww-e' reads a week past the last of its week-based year, which Elasticsearch 6's " + + "java.time mode resolves differently from one Java version to another" + ) + // with an hour, and within the year, they read as Elasticsearch 6.8 indexed them (MEASURED) + instant(read("8yyyy-MM-dd hh a", "6.8.23", "2024-07-04 10 PM")) shouldBe + Instant.parse("2024-07-04T22:00:00Z") + instant(read("8YYYY-ww-e", "6.8.23", "2024-01-1")) shouldBe + Instant.parse("2023-12-31T00:00:00Z") + } + + "A Joda pattern with two zone letters" should "fail the statement, naming the field and the format" in { + // MEASURED on 6.8.23: the offset wins (05:30Z) -- a combination outside the measured product + Seq( + "yyyy-MM-dd HH:mm:ss ZZ ZZZ" -> "2024-07-31 10:30:00 +05:00 Europe/Paris", + "yyyy-MM-dd HH:mm:ss Z ZZZ" -> "2024-07-31 10:30:00 +0500 Europe/Paris", + "yyyy-MM-dd HH:mm:ss ZZ z" -> "2024-07-31 10:30:00 +05:00 PDT" + ).foreach { case (spec, value) => + failure(spec, "6.8.23", value) should endWith( + "its letters combine two zones, which core has not measured" + ) + } + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala index 03602ae1..1810725d 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala @@ -373,7 +373,9 @@ class RelationalClosureGuardSpec extends AnyFlatSpec with Matchers { * refused on types -- Elasticsearch answers. */ it should "ask the branch type check after resolution, and refuse nothing on types without a schema" in { - val sql = "SELECT 1 AS n FROM t UNION ALL SELECT 'a' AS n FROM u" + // a DATE beside a whole number: DuckDB 1.5.5 refuses the pair (`1` beside `'a'` is a VARCHAR + // column there, and is answered since the lead's rule of 2026-10-06) + val sql = "SELECT CURRENT_DATE AS n FROM t UNION ALL SELECT 1 AS n FROM u" Parser(sql) match { case Right(multi: MultiSearch) => MultiSearch.branchTypes(multi.requests).swap.getOrElse("") should include( diff --git a/core/src/test/scala/app/softnetwork/elastic/client/RowNormalizerSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/RowNormalizerSpec.scala index 373ea6ff..d6780c52 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/RowNormalizerSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/RowNormalizerSpec.scala @@ -261,4 +261,53 @@ class RowNormalizerSpec extends AnyFlatSpec with Matchers with ElasticConversion val row = ListMap[String, Any]("whatever" -> 1) project(row, Seq.empty, Seq("a"))(NativeContext) should be theSameInstanceAs row } + + // ── rowProjector with converters: a UNION ALL column brought to the result's type ────────── + + private val toDouble: Any => Any = { + case n: java.lang.Number => java.lang.Double.valueOf(n.doubleValue()) + case other => other + } + + private def convert( + row: ListMap[String, Any], + source: Seq[String], + target: Seq[String], + converters: Array[Any => Any] + )(implicit ctx: ConversionContext): ListMap[String, Any] = + rowProjector(source, target, converters)(ctx)(row) + + "rowProjector with converters" should "be the projection without them when none converts" in { + val row = ListMap[String, Any]("a" -> 1, "b" -> 2, "c" -> 3) + convert(row, fields, fields, null)(NativeContext) should be theSameInstanceAs row + convert(row, fields, fields, Array[Any => Any](null, null, null))(NativeContext) should + be theSameInstanceAs row + } + + it should "convert the columns it is given, in the one rebuild, keeping the row's extras" in { + // 🔴 an in-order row is NOT returned as is: a value changes + val row = ListMap[String, Any]("a" -> 1, "b" -> 2, "c" -> 3, "extra" -> "e") + val converted = + convert(row, fields, fields, Array[Any => Any](null, toDouble, null))(NativeContext) + converted.toList shouldBe List("a" -> 1, "b" -> 2.0, "c" -> 3, "extra" -> "e") + converted("b").getClass shouldBe classOf[java.lang.Double] + converted("a").getClass shouldBe classOf[java.lang.Integer] + } + + it should "convert out of order, null-fill without converting, and rename" in { + val row = ListMap[String, Any]("c" -> 3, "a" -> 1) + convert(row, fields, Seq("x", "y", "z"), Array[Any => Any](toDouble, toDouble, toDouble))( + NativeContext + ).toList shouldBe List("x" -> 1.0, "y" -> null, "z" -> 3.0) + } + + it should "convert under a duplicate projection too" in { + val row = ListMap[String, Any]("a" -> 7, "extra" -> true) + convert(row, Seq("a", "a"), Seq("x", "y"), Array[Any => Any](toDouble, null))( + NativeContext + ).toList shouldBe List("x" -> 7.0, "y" -> 7, "extra" -> true) + convert(row, Seq("a", "a"), Seq("a", "a"), Array[Any => Any](toDouble, null))( + NativeContext + ).toList shouldBe List("a" -> 7.0, "extra" -> true) + } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SetOperationValuesSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationValuesSpec.scala new file mode 100644 index 00000000..614f7bbb --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/SetOperationValuesSpec.scala @@ -0,0 +1,885 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{MultiSearch, TemporalLiterals} +import app.softnetwork.elastic.sql.query.MultiSearch.{SetOperationType, StructField} +import app.softnetwork.elastic.sql.schema.{Column, Table} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.{ + Instant, + LocalDate, + LocalDateTime, + LocalTime, + OffsetDateTime, + ZoneOffset, + ZonedDateTime +} +import scala.collection.immutable.ListMap + +/** The values of a `UNION ALL` column are brought to the ONE type the column has -- DuckDB 1.5.5's, + * or `VARCHAR` where DuckDB gives none -- as the class JDBC and Arrow read the column's type from: + * a `Double` for a `DOUBLE`, a `Long` for a `BIGINT`, a `String` for a `VARCHAR`, a + * `ZonedDateTime` for a `TIMESTAMP`. Only the legs of a column whose branches disagree pay for it. + */ +class SetOperationValuesSpec extends AnyFlatSpec with Matchers { + + import SetOperationValues.converter + + private def kind(v: Any): Class[_] = v.getClass + + "A value reaching a DOUBLE column" should "be a Double, whatever number it was" in { + val f = converter(SQLTypes.Int, SQLTypes.Double) + f(Integer.valueOf(3)) shouldBe 3.0 + kind(f(Integer.valueOf(3))) shouldBe kind(java.lang.Double.valueOf(0)) + kind(f(java.lang.Long.valueOf(3000000000L))) shouldBe kind(java.lang.Double.valueOf(0)) + // Elasticsearch's one-element `script_fields` array is ONE value + f(List(Integer.valueOf(2))) shouldBe 2.0 + kind(f(List(Integer.valueOf(2)))) shouldBe kind(java.lang.Double.valueOf(0)) + // a BOOLEAN is 1 / 0 beside a number (DuckDB) + converter(SQLTypes.Boolean, SQLTypes.Double)(java.lang.Boolean.TRUE) shouldBe 1.0 + f(null) shouldBe (null: Any) + f(List(null)) shouldBe (null: Any) + } + + "A value reaching a BIGINT column" should "be a Long, also when its JSON fitted an int" in { + val f = converter(SQLTypes.BigInt, SQLTypes.BigInt) + kind(f(Integer.valueOf(3))) shouldBe kind(java.lang.Long.valueOf(0)) + kind(converter(SQLTypes.Int, SQLTypes.BigInt)(Integer.valueOf(-2))) shouldBe + kind(java.lang.Long.valueOf(0)) + } + + "A value reaching a REAL column" should "be a Float" in { + kind(converter(SQLTypes.Int, SQLTypes.Real)(Integer.valueOf(3))) shouldBe + kind(java.lang.Float.valueOf(0)) + converter(SQLTypes.Real, SQLTypes.Real)(java.lang.Double.valueOf(0.5)) shouldBe 0.5f + } + + "A DATE reaching a TIMESTAMP column" should "be the instant its day starts at, in UTC" in { + val f = converter(SQLTypes.Date, SQLTypes.Timestamp) + f(LocalDate.parse("2024-01-31")) shouldBe ZonedDateTime.parse("2024-01-31T00:00:00Z") + f(List(LocalDate.parse("2024-01-31"))) shouldBe ZonedDateTime.parse("2024-01-31T00:00:00Z") + // a date written as epoch milliseconds + f(java.lang.Long.valueOf(1706659200000L)) shouldBe + ZonedDateTime.parse("2024-01-31T00:00:00Z").withZoneSameInstant(ZoneOffset.UTC) + // a TIMESTAMP keeps its instant + val ts = ZonedDateTime.parse("2024-01-31T10:30:00Z") + converter(SQLTypes.Timestamp, SQLTypes.Timestamp)(ts) shouldBe ts + } + + "A value reaching a VARCHAR column" should "be spelled as DuckDB spells its type" in { + converter(SQLTypes.Int, SQLTypes.Varchar)(Integer.valueOf(3)) shouldBe "3" + // an INT computed by an aggregation comes back as a Double from Elasticsearch + converter(SQLTypes.Int, SQLTypes.Varchar)(java.lang.Double.valueOf(10.0)) shouldBe "10" + converter(SQLTypes.BigInt, SQLTypes.Varchar)(List(java.lang.Long.valueOf(3000000000L))) shouldBe + "3000000000" + converter(SQLTypes.Date, SQLTypes.Varchar)(LocalDate.parse("2024-01-31")) shouldBe "2024-01-31" + // a DATE column holding a date-time is its UTC day + converter(SQLTypes.Date, SQLTypes.Varchar)(ZonedDateTime.parse("2024-01-31T22:30:00Z")) shouldBe + "2024-01-31" + converter(SQLTypes.Boolean, SQLTypes.Varchar)(java.lang.Boolean.FALSE) shouldBe "false" + converter(SQLTypes.Varchar, SQLTypes.Varchar)(List("a")) shouldBe "a" + converter(SQLTypes.Null, SQLTypes.Varchar)(List(null)) shouldBe (null: Any) + // Elasticsearch's empty array is no value + converter(SQLTypes.Varchar, SQLTypes.Varchar)(Nil) shouldBe (null: Any) + } + + "A text value core re-read as a temporal (issue #418)" should "keep its seconds as text" in { + // a keyword that LOOKS like a date or a time comes back from the response re-read as one; + // `toString` would drop zero seconds (`10:30`, `2024-01-31T10:30Z`), the ISO formatters do not + val f = converter(SQLTypes.Varchar, SQLTypes.Varchar) + f(LocalTime.parse("10:30:00")) shouldBe "10:30:00" + f(LocalTime.parse("10:30:00.5")) shouldBe "10:30:00.5" + f(ZonedDateTime.parse("2024-01-31T10:30:00Z")) shouldBe "2024-01-31T10:30:00Z" + f(ZonedDateTime.parse("2024-01-31T10:30:00+02:00")) shouldBe "2024-01-31T10:30:00+02:00" + f(LocalDateTime.parse("2024-01-31T10:30:00")) shouldBe "2024-01-31T10:30:00" + f(List(LocalDate.parse("2024-01-31"))) shouldBe "2024-01-31" + // the same text reaching a VARBINARY column + converter(SQLTypes.Varchar, SQLTypes.VarBinary)(LocalTime.parse("10:30:00")) + .asInstanceOf[Array[Byte]] + .toSeq shouldBe "10:30:00".getBytes(java.nio.charset.StandardCharsets.UTF_8).toSeq + } + + "A value with several values in a converting column" should "fail the statement" in { + // a multi-valued field or an object has no ONE value of the column's type to convert; the + // message is in DuckDB's words for the same column (MEASURED on DuckDB 1.5.5) + def failure(f: Any => Any, value: Any): String = + intercept[IllegalArgumentException](f(value)).getMessage + val tags = List("a", "b") + failure(converter(SQLTypes.Varchar, SQLTypes.Varchar), tags) shouldBe + "Conversion Error: Unimplemented type for cast (VARCHAR[] -> VARCHAR)" + failure(converter(SQLTypes.Int, SQLTypes.Double), List(1, 2)) shouldBe + "Conversion Error: Unimplemented type for cast (INTEGER[] -> DOUBLE)" + failure(converter(SQLTypes.Varchar, SQLTypes.VarBinary), Map("a" -> 1)) shouldBe + "Conversion Error: Unimplemented type for cast (VARCHAR[] -> BLOB)" + failure(converter(SQLTypes.Timestamp, SQLTypes.Varchar), java.util.Arrays.asList(1, 2)) should + startWith("Conversion Error: Unimplemented type for cast") + } + + // Every spelling below is DuckDB 1.5.5's `CAST( AS VARCHAR)`, MEASURED value by value. + + "A TIMESTAMP reaching a VARCHAR column" should "be spelled as DuckDB spells it, in UTC" in { + val f = converter(SQLTypes.Timestamp, SQLTypes.Varchar) + f(ZonedDateTime.parse("2024-01-31T10:30:00Z")) shouldBe "2024-01-31 10:30:00" + // fractional seconds: only the digits there are + f(ZonedDateTime.parse("2023-12-25T23:59:59.999Z")) shouldBe "2023-12-25 23:59:59.999" + f(ZonedDateTime.parse("2024-01-31T10:30:00.5Z")) shouldBe "2024-01-31 10:30:00.5" + f(ZonedDateTime.parse("2024-01-31T10:30:00.000001Z")) shouldBe "2024-01-31 10:30:00.000001" + f(ZonedDateTime.parse("2024-01-31T10:30:00.123456789Z")) shouldBe + "2024-01-31 10:30:00.123456789" + // a date-only timestamp is its midnight + f(LocalDate.parse("2024-01-31")) shouldBe "2024-01-31 00:00:00" + // before the epoch + f(ZonedDateTime.parse("1969-12-31T23:59:59.999Z")) shouldBe "1969-12-31 23:59:59.999" + // another offset is read in UTC, as every date rule of the engine reads it + f(ZonedDateTime.parse("2024-01-31T12:30:00+02:00")) shouldBe "2024-01-31 10:30:00" + f(LocalDateTime.parse("2024-01-31T10:30:00")) shouldBe "2024-01-31 10:30:00" + // epoch milliseconds, and a computed branch's one-element array + f(java.lang.Long.valueOf(1706697000000L)) shouldBe "2024-01-31 10:30:00" + f(List(ZonedDateTime.parse("2024-01-31T10:30:00Z"))) shouldBe "2024-01-31 10:30:00" + // a year before 1, and one past 9999 + f(LocalDateTime.of(-43, 3, 15, 12, 0)) shouldBe "0044-03-15 (BC) 12:00:00" + f(LocalDateTime.of(0, 12, 31, 0, 0)) shouldBe "0001-12-31 (BC) 00:00:00" + f(LocalDateTime.of(10000, 1, 1, 0, 0)) shouldBe "10000-01-01 00:00:00" + f(LocalDateTime.of(999, 6, 1, 1, 2, 3)) shouldBe "0999-06-01 01:02:03" + f(null) shouldBe (null: Any) + } + + "A DATE reaching a VARCHAR column" should "carry DuckDB's year spelling" in { + val f = converter(SQLTypes.Date, SQLTypes.Varchar) + f(LocalDate.of(-43, 3, 15)) shouldBe "0044-03-15 (BC)" + f(LocalDate.of(1, 1, 1)) shouldBe "0001-01-01" + f(LocalDate.of(10000, 1, 1)) shouldBe "10000-01-01" + } + + "A DOUBLE reaching a VARCHAR column" should "be spelled as DuckDB spells it" in { + val f = converter(SQLTypes.Double, SQLTypes.Varchar) + f(java.lang.Double.valueOf(2.5e11)) shouldBe "250000000000.0" + f(java.lang.Double.valueOf(1e-5)) shouldBe "1e-05" + f(java.lang.Double.valueOf(1e16)) shouldBe "1e+16" + f(java.lang.Double.valueOf(-2.5e11)) shouldBe "-250000000000.0" + f(java.lang.Double.valueOf(-1e-5)) shouldBe "-1e-05" + f(java.lang.Double.valueOf(-0.0)) shouldBe "-0.0" + f(java.lang.Double.valueOf(0.1 + 0.2)) shouldBe "0.30000000000000004" + // DuckDB's own algorithm misses the shortest digits here; its answer is the reference + f(java.lang.Double.valueOf(7.168e25)) shouldBe "7.1680000000000004e+25" + // a whole number written in a DOUBLE column's JSON is still a DOUBLE + f(Integer.valueOf(5)) shouldBe "5.0" + f(List(java.lang.Double.valueOf(1.5))) shouldBe "1.5" + // a number of unknown width is a DOUBLE to DuckDB + converter(SQLTypes.Numeric, SQLTypes.Varchar)(java.lang.Long.valueOf(15)) shouldBe "15.0" + } + + "A REAL reaching a VARCHAR column" should "be spelled from the float's value, as DuckDB does" in { + val f = converter(SQLTypes.Real, SQLTypes.Varchar) + // the JSON of a REAL column is read as a Double; its value is the float's + f(java.lang.Double.valueOf(0.1)) shouldBe "0.1" + f(java.lang.Float.valueOf(-0.75f)) shouldBe "-0.75" + f(java.lang.Double.valueOf(2.5e11)) shouldBe "250000000000.0" + f(java.lang.Float.valueOf(4073260.75f)) shouldBe "4073260.75" + f(Integer.valueOf(16777216)) shouldBe "16777216.0" + } + + "A TIME reaching a VARCHAR column" should "be spelled as DuckDB spells it" in { + val f = converter(SQLTypes.Time, SQLTypes.Varchar) + f(LocalTime.parse("10:30")) shouldBe "10:30:00" + f(LocalTime.parse("10:30:00.5")) shouldBe "10:30:00.5" + f(LocalTime.parse("10:30:00.120")) shouldBe "10:30:00.12" + f(LocalTime.parse("23:59:59.999999")) shouldBe "23:59:59.999999" + f(LocalTime.MIDNIGHT) shouldBe "00:00:00" + f(ZonedDateTime.parse("2024-01-31T10:30:00Z")) shouldBe "10:30:00" + } + + "A value reaching a VARCHAR column DuckDB gives no type to" should "be spelled as text" in { + // the lead's rule of 2026-10-07: a VARBINARY beside a number, a TIME beside a date make a + // VARCHAR column + converter(SQLTypes.VarBinary, SQLTypes.Varchar)("YWI=") shouldBe "YWI=" + // bytes are their base64 text, the text Elasticsearch holds for a `binary` field + converter(SQLTypes.VarBinary, SQLTypes.Varchar)(utf8("ab")) shouldBe "YWI=" + converter(SQLTypes.Time, SQLTypes.Varchar)(LocalTime.parse("10:30")) shouldBe "10:30:00" + } + + "A GEO_POINT in a set operation" should "be a {lat, lon}, whatever way Elasticsearch stored it" in { + // every form Elasticsearch 8.18.3 accepts, MEASURED: `_source` keeps it as it was written + def point(lat: Double, lon: Double) = ListMap("lat" -> lat, "lon" -> lon) + val read = SetOperationValues.geoPoint(_: Any, "k") + read(Map("lat" -> 48.85, "lon" -> 2.35)) shouldBe point(48.85, 2.35) + read(Map("lon" -> "2.35", "lat" -> "48.85")) shouldBe point(48.85, 2.35) + read("48.85,2.35") shouldBe point(48.85, 2.35) + read("48.85, 2.35") shouldBe point(48.85, 2.35) + read(List(2.35, 48.85)) shouldBe point(48.85, 2.35) + read(List(2.35, 48.85, 10.0)) shouldBe point(48.85, 2.35) + read("POINT (2.35 48.85)") shouldBe point(48.85, 2.35) + read("POINT(2.35 48.85)") shouldBe point(48.85, 2.35) + read(Map("type" -> "Point", "coordinates" -> List(2.35, 48.85))) shouldBe point(48.85, 2.35) + // a geohash is the south-west corner of its cell, as Elasticsearch reads it (MEASURED) + read("u09tv") shouldBe point(48.8232421875, 2.3291015625) + read("u09tvw0f6szy") shouldBe point(48.85661389678717, 2.352221868932247) + // one point wrapped in Elasticsearch's array, no point + read(List(Map("lat" -> 1.0, "lon" -> 2.0))) shouldBe point(1.0, 2.0) + read(Nil) shouldBe (null: Any) + read(null) shouldBe (null: Any) + } + + it should "fail the statement, clearly, for several points or for no point" in { + def failure(value: Any): String = + intercept[IllegalArgumentException](SetOperationValues.geoPoint(value, "k")).getMessage + failure(List(List(2.35, 48.85), List(1.0, 2.0))) shouldBe + "Conversion Error: Unimplemented type for cast (STRUCT(lat DOUBLE, lon DOUBLE)[] -> " + + "STRUCT(lat DOUBLE, lon DOUBLE)) when casting from source column k" + failure("nowhere!") shouldBe + "Conversion Error: Could not read 'nowhere!' as a GEO_POINT when casting from source column " + + "k: a point is an object {lat, lon}, a text 'lat,lon', an array [lon, lat], a geohash or a " + + "WKT POINT (lon lat)" + failure(Map("x" -> 1)) should startWith("Conversion Error: Could not read 'Map(x -> 1)'") + } + + private def struct(fields: StructField*): SetOperationType = + SetOperationType(SQLTypes.Struct, fields) + + "A STRUCT or a GEO_POINT reaching a merged STRUCT column" should "hold the column's fields, in order" in { + // DuckDB 1.5.5's merge, MEASURED: STRUCT(lat, lon) beside STRUCT(a, b) is STRUCT(lat, lon, a, b) + val obj = Seq(StructField("a", SQLTypes.Int), StructField("b", SQLTypes.Varchar)) + val merged = struct(StructField.GeoPoint ++ obj: _*) + val branches = Seq(Some(SetOperationType.GeoPoint), Some(struct(obj: _*))) + val fromPoint = SetOperationValues.structConverter(merged, branches, 0, "k") + val fromObject = SetOperationValues.structConverter(merged, branches, 1, "k") + val p = fromPoint("48.85,2.35").asInstanceOf[ListMap[String, Any]] + p shouldBe ListMap("lat" -> 48.85, "lon" -> 2.35, "a" -> null, "b" -> null) + p.keys.toSeq shouldBe Seq("lat", "lon", "a", "b") + val o = fromObject(Map("b" -> "x", "a" -> 1)).asInstanceOf[ListMap[String, Any]] + o shouldBe ListMap("lat" -> null, "lon" -> null, "a" -> 1, "b" -> "x") + o.keys.toSeq shouldBe Seq("lat", "lon", "a", "b") + fromObject(null) shouldBe (null: Any) + // one object wrapped in Elasticsearch's array + fromObject(List(Map("a" -> 2, "b" -> "y"))) shouldBe + ListMap("lat" -> null, "lon" -> null, "a" -> 2, "b" -> "y") + } + + it should "convert a field whose type the merge widened, match fields whatever their case" in { + // an INTEGER beside a VARCHAR is a VARCHAR (DuckDB answers '1'); a point inside an object + // is a {lat, lon} too + val merged = struct( + StructField("a", SQLTypes.Varchar), + StructField("p", SetOperationType.GeoPoint) + ) + val branches = Seq( + Some(struct(StructField("A", SQLTypes.Int), StructField("p", SetOperationType.GeoPoint))), + Some(struct(StructField("a", SQLTypes.Varchar))) + ) + SetOperationValues.structConverter(merged, branches, 0, "k")( + Map("A" -> 1, "p" -> "48.85,2.35") + ) shouldBe ListMap("a" -> "1", "p" -> ListMap("lat" -> 48.85, "lon" -> 2.35)) + } + + it should "fail the statement for several objects, or a value that is no object" in { + val obj = struct(StructField("a", SQLTypes.Int)) + val f = SetOperationValues.structConverter( + struct(StructField.GeoPoint :+ StructField("a", SQLTypes.Int): _*), + Seq(Some(SetOperationType.GeoPoint), Some(obj)), + 1, + "k" + ) + intercept[IllegalArgumentException](f(List(Map("a" -> 1), Map("a" -> 2)))).getMessage shouldBe + "Conversion Error: Unimplemented type for cast (STRUCT(a INTEGER)[] -> STRUCT(lat DOUBLE, " + + "lon DOUBLE, a INTEGER)) when casting from source column k" + intercept[IllegalArgumentException](f("x")).getMessage shouldBe + "Conversion Error: Could not read 'x' as a STRUCT when casting from source column k" + } + + // ── one class per converting position, whatever form core read the value in ─────────────── + + /** A value and its class, arrays by their bytes. */ + private def shown(v: Any): (Any, Class[_]) = + v match { + case null => (null, classOf[Null]) + case b: Array[Byte] => (b.toSeq, classOf[Array[Byte]]) + case other => (other, other.getClass) + } + + "Every form core reads a value in" should "reach the position's one class" in { + // the forms ElasticConversion.jsonNodeToAny makes of a JSON value -- an int, a long, a double + // or a decimal; a date-only, zone-less, zoned or offset text; epoch milliseconds -- and an + // Instant; each also in Elasticsearch's one-element array + val zoned = ZonedDateTime.parse("2024-01-31T10:30:00Z") + val midnight = ZonedDateTime.parse("2024-01-31T00:00:00Z") + val cases: Seq[(SQLType, SQLType, Any, Any)] = Seq( + (SQLTypes.BigInt, SQLTypes.BigInt, Integer.valueOf(1), java.lang.Long.valueOf(1L)), + (SQLTypes.BigInt, SQLTypes.BigInt, java.lang.Long.valueOf(3000000000L), 3000000000L), + (SQLTypes.BigInt, SQLTypes.BigInt, java.lang.Double.valueOf(4.0), java.lang.Long.valueOf(4L)), + (SQLTypes.Int, SQLTypes.BigInt, Integer.valueOf(-2), java.lang.Long.valueOf(-2L)), + (SQLTypes.Boolean, SQLTypes.BigInt, java.lang.Boolean.TRUE, java.lang.Long.valueOf(1L)), + (SQLTypes.Double, SQLTypes.Double, Integer.valueOf(2), java.lang.Double.valueOf(2.0)), + (SQLTypes.Double, SQLTypes.Double, BigDecimal("2.5").bigDecimal, 2.5), + (SQLTypes.Int, SQLTypes.Double, java.lang.Long.valueOf(3L), java.lang.Double.valueOf(3.0)), + (SQLTypes.Timestamp, SQLTypes.Timestamp, LocalDate.parse("2024-01-31"), midnight), + (SQLTypes.Timestamp, SQLTypes.Timestamp, LocalDateTime.parse("2024-01-31T10:30:00"), zoned), + (SQLTypes.Timestamp, SQLTypes.Timestamp, zoned, zoned), + ( + SQLTypes.Timestamp, + SQLTypes.Timestamp, + OffsetDateTime.parse("2024-01-31T12:30:00+02:00"), + ZonedDateTime.parse("2024-01-31T12:30:00+02:00") + ), + (SQLTypes.Timestamp, SQLTypes.Timestamp, Instant.parse("2024-01-31T10:30:00Z"), zoned), + (SQLTypes.Timestamp, SQLTypes.Timestamp, java.lang.Long.valueOf(1706697000000L), zoned), + (SQLTypes.Date, SQLTypes.Timestamp, LocalDate.parse("2024-01-31"), midnight), + (SQLTypes.VarBinary, SQLTypes.VarBinary, "YWI=", utf8("YWI=")), + (SQLTypes.VarBinary, SQLTypes.VarBinary, utf8("ab"), utf8("ab")), + (SQLTypes.Varchar, SQLTypes.VarBinary, "x", utf8("x")), + (SQLTypes.BigInt, SQLTypes.Varchar, Integer.valueOf(1), "1"), + (SQLTypes.Double, SQLTypes.Varchar, Integer.valueOf(2), "2.0"), + (SQLTypes.Timestamp, SQLTypes.Varchar, LocalDate.parse("2024-01-31"), "2024-01-31 00:00:00"), + ( + SQLTypes.Timestamp, + SQLTypes.Varchar, + java.lang.Long.valueOf(1706697000000L), + "2024-01-31 10:30:00" + ), + (SQLTypes.VarBinary, SQLTypes.Varchar, "YWI=", "YWI="), + (SQLTypes.VarBinary, SQLTypes.Varchar, utf8("ab"), "YWI=") + ) + val wrong = cases.flatMap { case (source, target, value, want) => + val f = converter(source, target) + Seq(value, List(value), new java.util.ArrayList[Any](java.util.Arrays.asList(value))) + .flatMap { form => + val got = shown(f(form)) + if (got == shown(want)) None + else Some(s"$source -> $target: $form gave $got instead of ${shown(want)}") + } + } + wrong shouldBe empty + Seq(SQLTypes.BigInt, SQLTypes.Double, SQLTypes.Timestamp, SQLTypes.VarBinary, SQLTypes.Varchar) + .foreach { target => + converter(SQLTypes.Int, target)(null) shouldBe (null: Any) + converter(SQLTypes.Int, target)(List(null)) shouldBe (null: Any) + } + } + + "A value Elasticsearch accepted as text" should "be read as its field's type first, as ES indexed it" in { + // MEASURED on Elasticsearch 8.18.3, field type by field type: what docvalue_fields answers + // (what queries and aggregations see) for each text `_source` holds + val zoned = ZonedDateTime.parse("2024-01-31T10:30:00Z") + val cases: Seq[(SQLType, SQLType, Any, Any)] = Seq( + // integer fields: truncated toward zero; an exponent applied + (SQLTypes.BigInt, SQLTypes.BigInt, "5", java.lang.Long.valueOf(5L)), + (SQLTypes.BigInt, SQLTypes.BigInt, "5.0", java.lang.Long.valueOf(5L)), + (SQLTypes.BigInt, SQLTypes.BigInt, "5.5", java.lang.Long.valueOf(5L)), + (SQLTypes.BigInt, SQLTypes.BigInt, "5.5e0", java.lang.Long.valueOf(5L)), + (SQLTypes.BigInt, SQLTypes.BigInt, "-5.9", java.lang.Long.valueOf(-5L)), + (SQLTypes.BigInt, SQLTypes.BigInt, "5e2", java.lang.Long.valueOf(500L)), + ( + SQLTypes.BigInt, + SQLTypes.BigInt, + java.lang.Double.valueOf(-5.5), + java.lang.Long.valueOf(-5L) + ), + (SQLTypes.Int, SQLTypes.BigInt, " 5 ", java.lang.Long.valueOf(5L)), + (SQLTypes.SmallInt, SQLTypes.Int, "5.5", Integer.valueOf(5)), + (SQLTypes.TinyInt, SQLTypes.Int, "5e1", Integer.valueOf(50)), + (SQLTypes.BigInt, SQLTypes.Double, "5.5", java.lang.Double.valueOf(5.0)), + ( + SQLTypes.BigInt, + SQLTypes.Double, + java.lang.Double.valueOf(5.5), + java.lang.Double.valueOf(5.0) + ), + (SQLTypes.BigInt, SQLTypes.Varchar, "5.0", "5"), + (SQLTypes.BigInt, SQLTypes.Varchar, "5e2", "500"), + // double and float fields + (SQLTypes.Double, SQLTypes.Double, "2.5", java.lang.Double.valueOf(2.5)), + (SQLTypes.Double, SQLTypes.Double, " 2.5 ", java.lang.Double.valueOf(2.5)), + (SQLTypes.Double, SQLTypes.Double, "+2.5", java.lang.Double.valueOf(2.5)), + (SQLTypes.Double, SQLTypes.Double, "1e3", java.lang.Double.valueOf(1000.0)), + (SQLTypes.Double, SQLTypes.Double, "1E3", java.lang.Double.valueOf(1000.0)), + (SQLTypes.Double, SQLTypes.Varchar, "5", "5.0"), + (SQLTypes.Double, SQLTypes.Varchar, "1e3", "1000.0"), + (SQLTypes.Real, SQLTypes.Real, "0.1", java.lang.Float.valueOf(0.1f)), + (SQLTypes.Real, SQLTypes.Double, "1e3", java.lang.Double.valueOf(1000.0)), + // boolean fields: an empty text is false + (SQLTypes.Boolean, SQLTypes.Int, "true", Integer.valueOf(1)), + (SQLTypes.Boolean, SQLTypes.Int, "false", Integer.valueOf(0)), + (SQLTypes.Boolean, SQLTypes.Int, "", Integer.valueOf(0)), + (SQLTypes.Boolean, SQLTypes.Varchar, "", "false"), + // date fields: digits are epoch milliseconds; a DATE is its day, in UTC + (SQLTypes.Timestamp, SQLTypes.Timestamp, "1706697000000", zoned), + ( + SQLTypes.Timestamp, + SQLTypes.Timestamp, + "1706697000", + ZonedDateTime.parse("1970-01-20T18:04:57Z") + ), + (SQLTypes.Timestamp, SQLTypes.Varchar, "1706697000000", "2024-01-31 10:30:00"), + ( + SQLTypes.Date, + SQLTypes.Timestamp, + "1706697000000", + ZonedDateTime.parse("2024-01-31T00:00:00Z") + ), + (SQLTypes.Date, SQLTypes.Timestamp, "-86400000", ZonedDateTime.parse("1969-12-31T00:00:00Z")), + (SQLTypes.Date, SQLTypes.Timestamp, zoned, ZonedDateTime.parse("2024-01-31T00:00:00Z")), + ( + SQLTypes.Date, + SQLTypes.Timestamp, + LocalDateTime.parse("2024-01-31T10:30:00"), + ZonedDateTime.parse("2024-01-31T00:00:00Z") + ), + (SQLTypes.Date, SQLTypes.Varchar, "1706697000000", "2024-01-31") + ) + val wrong = cases.flatMap { case (source, target, value, want) => + val f = converter(source, target) + Seq(value, List(value)).flatMap { form => + val got = shown(f(form)) + if (got == shown(want)) None + else Some(s"$source -> $target: $form gave $got instead of ${shown(want)}") + } + } + wrong shouldBe empty + // an empty text in a number field: no value, as Elasticsearch indexes none + Seq(SQLTypes.BigInt, SQLTypes.Int, SQLTypes.Double, SQLTypes.Real).foreach { source => + converter(source, SQLTypes.Double)("") shouldBe (null: Any) + } + } + + // ── a date field's values, read with its mapping format as the Elasticsearch major reads it ── + // (every measured value of every major: MappedDateFormatSpec) + + private val measuredOn = + java.time.Clock.fixed(Instant.parse("2026-10-07T12:00:00Z"), ZoneOffset.UTC) + + /** The version a statement runs on, by which it reads its date fields. */ + private val es8 = Some("8.18.3") + + /** The converter of a converting position whose branch reads the date field `obj.ts` of mapping + * format `format`, on Elasticsearch `version`. + */ + private def byFormat( + source: SQLType, + target: SQLType, + format: String, + version: Option[String] = Some("8.18.3") + ): Any => Any = + SetOperationValues.converter( + source, + target, + "multi", + Some(format), + "obj.ts", + "k", + new SetOperationValues.DateReading(version, measuredOn) + ) + + private val at1030 = ZonedDateTime.parse("2024-01-31T10:30:00Z") + + "A date field's value" should "be read with its mapping format, as the Elasticsearch major reads it" in { + val ts = SQLTypes.Timestamp + // epoch_second: a number and its text are SECONDS, never 1970's milliseconds + Seq("6.8.23", "7.17.29", "8.18.3").foreach { v => + shown(byFormat(ts, ts, "epoch_second", Some(v))(Integer.valueOf(1706697000))) shouldBe + shown(at1030) + shown(byFormat(ts, ts, "epoch_second", Some(v))("1706697000")) shouldBe shown(at1030) + } + // a fraction of a second: Elasticsearch 8 keeps its milliseconds, Elasticsearch 6 truncates + // it to the second + shown(byFormat(ts, ts, "epoch_second")("1706697000.5")) shouldBe + shown(ZonedDateTime.parse("2024-01-31T10:30:00.500Z")) + shown(byFormat(ts, ts, "epoch_second", Some("6.8.23"))("1706697000.5")) shouldBe shown(at1030) + // the default format reads four digits as a YEAR, as Elasticsearch does, before epoch millis + Seq[Any](Integer.valueOf(2024), "2024").foreach { year => + shown(byFormat(ts, ts, TemporalLiterals.DefaultDateFormat)(year)) shouldBe + shown(ZonedDateTime.parse("2024-01-01T00:00:00Z")) + } + shown(byFormat(ts, ts, TemporalLiterals.DefaultDateFormat)("1706697000000")) shouldBe + shown(at1030) + // an ISO text the response parser does not read is read by the format + shown(byFormat(ts, ts, TemporalLiterals.DefaultDateFormat)("2024-01-31T12:30:00+0200")) shouldBe + shown(at1030) + // Elasticsearch's date keeps milliseconds + shown( + byFormat(ts, ts, TemporalLiterals.DefaultDateFormat)( + ZonedDateTime.parse("2024-01-31T10:30:00.123456789Z") + ) + ) shouldBe shown(ZonedDateTime.parse("2024-01-31T10:30:00.123Z")) + // a custom pattern's text: a ZonedDateTime, not the text + shown(byFormat(ts, ts, "yyyy-MM-dd HH:mm:ss")("2024-01-31 10:30:00")) shouldBe shown(at1030) + // Joda (Elasticsearch 6) reads one or two digits where java.time reads two, and a two-digit + // year around the current year minus 30 + shown(byFormat(ts, ts, "yyyy/MM/dd", Some("6.8.23"))("2024/1/31")) shouldBe + shown(ZonedDateTime.parse("2024-01-31T00:00:00Z")) + shown(byFormat(ts, ts, "dd.MM.yy", Some("6.8.23"))("31.01.46")) shouldBe + shown(ZonedDateTime.parse("1946-01-31T00:00:00Z")) + shown(byFormat(ts, ts, "dd.MM.yy")("31.01.46")) shouldBe + shown(ZonedDateTime.parse("2046-01-31T00:00:00Z")) + // a value the response parser read as an ISO date is read again with the format + shown( + byFormat(ts, ts, "yyyy-dd-MM||strict_date_optional_time")(LocalDate.parse("2024-01-02")) + ) shouldBe + shown(ZonedDateTime.parse("2024-02-01T00:00:00Z")) + shown( + byFormat(ts, ts, "strict_date_hour_minute")(LocalDateTime.parse("2024-01-31T10:30")) + ) shouldBe + shown(at1030) + // a DATE branch is the day of the instant, in UTC, spelled as DuckDB spells a date + shown(byFormat(SQLTypes.Date, ts, "epoch_second")(Integer.valueOf(1706697000))) shouldBe + shown(ZonedDateTime.parse("2024-01-31T00:00:00Z")) + byFormat(SQLTypes.Date, SQLTypes.Varchar, "dd/MM/yyyy HH:mm")("31/01/2024 10:30") shouldBe + "2024-01-31" + byFormat(ts, SQLTypes.Varchar, "yyyy/MM/dd||epoch_millis")("2024/01/31") shouldBe + "2024-01-31 00:00:00" + // one converter reads every row of a leg: a value the response parser read as an ISO date is + // read by its shape once (Elasticsearch 6's strict_date_time wants a fraction, so it reads + // 10:30:00Z as 10:30:00.0Z), then each value of that shape is its own instant + val sdt6 = byFormat(ts, ts, "strict_date_time", Some("6.8.23")) + Seq("2024-01-31T10:30:00Z", "2024-02-01T11:00:00Z", "2024-01-31T12:30:00.123456+02:00") + .map(ZonedDateTime.parse) + .foreach(z => + shown(sdt6(z)) shouldBe + shown(z.withNano(z.getNano / 1000000 * 1000000).withZoneSameInstant(ZoneOffset.UTC)) + ) + intercept[IllegalArgumentException](sdt6(LocalDate.parse("2024-01-31"))) + // NULL, a one-element array + byFormat(ts, ts, "epoch_second")(null) shouldBe (null: Any) + shown(byFormat(ts, ts, "epoch_second")(List("1706697000"))) shouldBe shown(at1030) + } + + it should "fail the statement, naming the field and the format, rather than be guessed" in { + val ts = SQLTypes.Timestamp + def failure(f: Any => Any, value: Any): String = + intercept[IllegalArgumentException](f(value)).getMessage + // a value the format does not read + failure(byFormat(ts, ts, "yyyy/MM/dd"), "31-01-2024") shouldBe + "Conversion Error: Could not read '31-01-2024' as a TIMESTAMP when casting from source column " + + "k: the date format 'yyyy/MM/dd' of field obj.ts does not read it as Elasticsearch 8 does" + // Elasticsearch 7 reads no negative epoch_millis, Elasticsearch 8 does + shown(byFormat(ts, ts, "epoch_millis")("-2024")) shouldBe + shown(ZonedDateTime.parse("1969-12-31T23:59:57.976Z")) + failure(byFormat(ts, ts, "epoch_millis", Some("7.17.29")), "-2024") shouldBe + "Conversion Error: Could not read '-2024' as a TIMESTAMP when casting from source column k: " + + "the date format 'epoch_millis' of field obj.ts does not read it as Elasticsearch 7 does" + // a pattern letter no measurement covers + failure(byFormat(ts, ts, "yyyy-MM-dd B||epoch_millis"), "1706697000000") shouldBe + "Conversion Error: Could not read '1706697000000' as a TIMESTAMP when casting from source " + + "column k: core cannot read it with the date format 'yyyy-MM-dd B||epoch_millis' of field " + + "obj.ts as Elasticsearch 8 does: 'yyyy-MM-dd B' is not a date format core reads as " + + "Elasticsearch 8 reads it: the letter 'B' is not one core has measured" + // a name every major reads (MEASURED): a text it does not parse is the next alternative's + shown(byFormat(ts, ts, "week_date||epoch_millis")("1706697000000")) shouldBe shown(at1030) + shown(byFormat(ts, ts, "epoch_millis||week_date")("1706697000000")) shouldBe shown(at1030) + // a spelling the response parser erased, read differently by the format's alternatives + failure( + byFormat(ts, ts, "yyyy-dd-MM'T'HH:mm:ss||strict_date_optional_time"), + LocalDateTime.parse("2024-01-02T10:30") + ) shouldBe + "Conversion Error: Could not read '2024-01-02T10:30' as a TIMESTAMP when casting from source " + + "column k: core cannot read it with the date format " + + "'yyyy-dd-MM'T'HH:mm:ss||strict_date_optional_time' of field obj.ts as Elasticsearch 8 does: " + + "the format reads it differently depending on how it was spelled, which the response no " + + "longer says" + // an offset the response parser dropped, which the format reads + failure(byFormat(ts, ts, "yyyy-MM-dd[XXX]"), LocalDate.parse("2024-01-31")) should + endWith( + "the response gives it without the offset it may have been written with, which the format " + + "reads" + ) + // the Elasticsearch version not known + failure(byFormat(ts, ts, "epoch_second", None), "1706697000") shouldBe + "Conversion Error: Could not read '1706697000' as a TIMESTAMP when casting from source column " + + "k: the date format 'epoch_second' of field obj.ts reads it as the Elasticsearch version " + + "does, which is not known" + } + + // DuckDB 1.5.5: a BLOB beside text is a BLOB, each text value cast to bytes (MEASURED) + + private def utf8(s: String): Array[Byte] = s.getBytes(java.nio.charset.StandardCharsets.UTF_8) + + private def bytesOf(v: Any): Seq[Int] = v.asInstanceOf[Array[Byte]].toSeq.map(_ & 0xff) + + "A VARBINARY value reaching a VARBINARY column" should "be the bytes of the text Elasticsearch holds" in { + val f = converter(SQLTypes.VarBinary, SQLTypes.VarBinary) + // a `binary` field's base64, read as its UTF-8 bytes -- what arrow and the JDBC driver read + bytesOf(f("YWI=")) shouldBe utf8("YWI=").toSeq.map(_ & 0xff) + kind(f("YWI=")) shouldBe classOf[Array[Byte]] + val raw = Array[Byte](1, 2, -1) + f(raw) shouldBe raw + f(null) shouldBe (null: Any) + } + + "A text value reaching a VARBINARY column" should "be cast to bytes as DuckDB casts a VARCHAR to a BLOB" in { + val f = converter(SQLTypes.Varchar, SQLTypes.VarBinary) + bytesOf(f("x")) shouldBe Seq(0x78) + bytesOf(f("hello world")) shouldBe utf8("hello world").toSeq.map(_ & 0xff) + bytesOf(f("\\x41\\xAA")) shouldBe Seq(0x41, 0xaa) + bytesOf(f("a\\x41b")) shouldBe Seq(0x61, 0x41, 0x62) + bytesOf(f("\\xaA")) shouldBe Seq(0xaa) + bytesOf(f("\\x00")) shouldBe Seq(0x00) + bytesOf(f("\t\u007f")) shouldBe Seq(0x09, 0x7f) + bytesOf(f("")) shouldBe Nil + bytesOf(f(List("x"))) shouldBe Seq(0x78) + f(null) shouldBe (null: Any) + kind(f("x")) shouldBe classOf[Array[Byte]] + } + + it should "fail, as DuckDB's cast does, on a character outside ASCII or a broken escape" in { + val f = converter(SQLTypes.Varchar, SQLTypes.VarBinary) + def failure(text: String): String = + intercept[IllegalArgumentException](f(text)).getMessage + failure("\u00e9") shouldBe + "Conversion Error: Invalid byte encountered in STRING -> BLOB conversion of string \"\u00e9\". " + + "All non-ascii characters must be escaped with hex codes (e.g. \\xAA)" + failure("a\\b") shouldBe + "Conversion Error: Invalid hex escape code encountered in string -> blob conversion of string " + + "\"a\\b\": unterminated escape code at end of blob" + failure("\\x4") should endWith("\"\\x4\": unterminated escape code at end of blob") + failure("ab\\") should endWith("\"ab\\\": unterminated escape code at end of blob") + failure("\\xZZ") should endWith("\"\\xZZ\": \\xZZ") + failure("\\XAA") should endWith("\"\\XAA\": \\XAA") + failure("\\\\x41") should endWith("\"\\\\x41\": \\\\x4") + // left to right: the first fault met names the failure + failure("\\xZZ\u00e9") should endWith(": \\xZZ") + failure("\u00e9\\xZZ") should include("Invalid byte encountered") + failure("a\\\u00e9") should endWith("unterminated escape code at end of blob") + } + + // ── the per-leg plan ───────────────────────────────────────────────────────────────────── + + private val schema = Table( + "t", + columns = List( + Column("id", SQLTypes.Keyword), + Column("n", SQLTypes.Int), + Column("x", SQLTypes.Double), + Column("d", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("bin", SQLTypes.VarBinary), + Column("geo", SQLTypes.GeoPoint), + Column( + "obj", + SQLTypes.Struct, + multiFields = List(Column("a", SQLTypes.Int), Column("b", SQLTypes.Keyword)) + ), + Column( + "objl", + SQLTypes.Struct, + multiFields = List(Column("a", SQLTypes.BigInt), Column("b", SQLTypes.Keyword)) + ), + Column("oi", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Int))), + Column("ol", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.BigInt))), + Column("ox", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Double))), + Column("od", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Date))), + Column("ots", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Timestamp))), + Column("obin", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.VarBinary))), + Column("ov", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Keyword))), + Column( + "si", + SQLTypes.Struct, + multiFields = + List(Column("p", SQLTypes.Struct, multiFields = List(Column("c", SQLTypes.Int)))) + ), + Column( + "sl", + SQLTypes.Struct, + multiFields = + List(Column("p", SQLTypes.Struct, multiFields = List(Column("c", SQLTypes.BigInt)))) + ), + Column("ai", SQLTypes.Array(SQLTypes.Int)), + Column("av", SQLTypes.Array(SQLTypes.Keyword)), + Column( + "items", + SQLTypes.Array(SQLTypes.Struct), + multiFields = List(Column("p", SQLTypes.Keyword), Column("q", SQLTypes.Int)) + ), + Column( + "items2", + SQLTypes.Array(SQLTypes.Struct), + multiFields = List(Column("p", SQLTypes.Int)) + ) + ) + ) + + private def resolved(sql: String): MultiSearch = + Parser(sql) match { + case Right(m: MultiSearch) => m.copy(requests = m.requests.map(_.update(Some(schema)))) + case other => fail(s"[$sql] expected a set operation, got $other") + } + + "The per-leg plan" should "convert nothing when every column's branches agree" in { + SetOperationValues.converters( + resolved("SELECT id, n FROM t UNION ALL SELECT id, n FROM t"), + es8 + ) shouldBe Seq(None, None) + // a NULL literal beside an INT: nothing to convert either + SetOperationValues.converters( + resolved("SELECT n AS k FROM t UNION ALL SELECT NULL AS k FROM t"), + es8 + ) shouldBe Seq(None, None) + } + + it should "convert, in every leg, only the columns whose branches disagree" in { + val plan = SetOperationValues.converters( + resolved("SELECT id, n FROM t UNION ALL SELECT id, x FROM t"), + es8 + ) + plan.map(_.map(_.map(_ ne null).toSeq)) shouldBe + Seq(Some(Seq(false, true)), Some(Seq(false, true))) + plan.head.get.apply(1)(Integer.valueOf(3)) shouldBe 3.0 + } + + it should "fail a multi-valued value with DuckDB's own words, naming the column" in { + def failure(sql: String, leg: Int, value: Any): String = { + val plan = SetOperationValues.converters(resolved(sql), es8) + intercept[IllegalArgumentException](plan(leg).get.apply(0)(value)).getMessage + } + val tags = List("a", "b") + // MEASURED on DuckDB 1.5.5, the same shapes over a VARCHAR[] column + failure("SELECT id AS k FROM t UNION ALL SELECT n AS k FROM t", 0, tags) shouldBe + "Conversion Error: Unimplemented type for cast (INTEGER -> VARCHAR[]) when casting from " + + "source column k" + failure("SELECT x AS k FROM t UNION ALL SELECT id AS k FROM t", 1, tags) shouldBe + "Conversion Error: Unimplemented type for cast (DOUBLE -> VARCHAR[]) when casting from " + + "source column k" + failure("SELECT id AS k FROM t UNION ALL SELECT ts AS k FROM t", 0, tags) shouldBe + "Conversion Error: Unimplemented type for cast (TIMESTAMP -> VARCHAR[]) when casting from " + + "source column k" + failure("SELECT id AS k FROM t UNION ALL SELECT bin AS k FROM t", 0, tags) shouldBe + "Conversion Error: Unimplemented type for cast (BLOB -> VARCHAR[]) when casting from " + + "source column k" + // the first other branch is text: DuckDB names its value, which this branch cannot know + failure( + "SELECT id AS k FROM t UNION ALL SELECT 'c' AS k FROM t UNION ALL SELECT n AS k FROM t", + 0, + tags + ) shouldBe + "Conversion Error: Type VARCHAR can't be cast to the destination type VARCHAR[] when " + + "casting from source column k" + } + + it should "spell every leg as text in a VARCHAR column DuckDB gives no type to" in { + val plan = SetOperationValues.converters( + resolved("SELECT bin AS k FROM t UNION ALL SELECT n AS k FROM t"), + es8 + ) + plan.head.get.apply(0)("YWI=") shouldBe "YWI=" + plan(1).get.apply(0)(Integer.valueOf(3)) shouldBe "3" + // a binary field with several values: the column holds one value per row + intercept[IllegalArgumentException]( + plan.head.get.apply(0)(List("YWI=", "eA==")) + ).getMessage shouldBe + "Conversion Error: Unimplemented type for cast (INTEGER -> BLOB[]) when casting from source " + + "column k" + } + + it should "give every leg of a STRUCT column DuckDB's merged struct" in { + val plan = SetOperationValues.converters( + resolved("SELECT geo AS k FROM t UNION ALL SELECT obj AS k FROM t"), + es8 + ) + plan.head.get.apply(0)(List(2.35, 48.85)) shouldBe + ListMap("lat" -> 48.85, "lon" -> 2.35, "a" -> null, "b" -> null) + plan(1).get.apply(0)(Map("a" -> 1, "b" -> "x")) shouldBe + ListMap("lat" -> null, "lon" -> null, "a" -> 1, "b" -> "x") + // two points stored differently: both {lat, lon} + val points = SetOperationValues.converters( + resolved("SELECT geo AS k FROM t UNION ALL SELECT geo AS k FROM t"), + es8 + ) + points.head.get.apply(0)("48.85,2.35") shouldBe ListMap("lat" -> 48.85, "lon" -> 2.35) + points(1).get.apply(0)(Map("lat" -> 48.85, "lon" -> 2.35)) shouldBe + ListMap("lat" -> 48.85, "lon" -> 2.35) + } + + /** The value branch `leg` of `sql` answers at column 1 for `value`. */ + private def converted(sql: String, leg: Int, value: Any): Any = + SetOperationValues.converters(resolved(sql), es8)(leg) match { + case Some(functions) if functions(0) ne null => functions(0)(value) + case other => fail(s"[$sql] leg $leg converts nothing: $other") + } + + private def union(a: String, b: String): String = + s"SELECT $a AS k FROM t UNION ALL SELECT $b AS k FROM t" + + /** The field `a` branch `leg` of `sql` answers for the object `{a: value}`, with its class. */ + private def fieldA(sql: String, leg: Int, value: Any): (Any, Class[_]) = + converted(sql, leg, Map("a" -> value)) match { + case m: scala.collection.Map[_, _] => + shown(m.asInstanceOf[scala.collection.Map[String, Any]]("a")) + case other => fail(s"[$sql] leg $leg answered $other") + } + + it should "convert every branch of a converting field, the one already of its type too" in { + // a BIGINT field beside an INT one: a Long in BOTH branches, also where the JSON fitted an int + fieldA(union("ol", "oi"), 0, Integer.valueOf(1)) shouldBe shown(java.lang.Long.valueOf(1L)) + fieldA(union("ol", "oi"), 1, Integer.valueOf(2)) shouldBe shown(java.lang.Long.valueOf(2L)) + // a DOUBLE beside an INT: a Double in both + fieldA(union("ox", "oi"), 0, Integer.valueOf(2)) shouldBe shown(java.lang.Double.valueOf(2.0)) + // a VARBINARY beside a VARCHAR: bytes in both + fieldA(union("obin", "ov"), 0, "YWI=") shouldBe shown(utf8("YWI=")) + fieldA(union("obin", "ov"), 1, "x") shouldBe shown(utf8("x")) + // a TIMESTAMP beside a DATE: a ZonedDateTime in both, whatever form the timestamp was read in + val midnight = ZonedDateTime.parse("2024-01-31T00:00:00Z") + val at1030 = ZonedDateTime.parse("2024-01-31T10:30:00Z") + Seq( + LocalDate.parse("2024-01-31") -> midnight, + LocalDateTime.parse("2024-01-31T10:30:00") -> at1030, + java.lang.Long.valueOf(1706697000000L) -> at1030, + at1030 -> at1030 + ).foreach { case (value, want) => + fieldA(union("ots", "od"), 0, value) shouldBe shown(want) + } + fieldA(union("ots", "od"), 1, LocalDate.parse("2024-01-31")) shouldBe shown(midnight) + // a field whose branches agree keeps its value as read, inside a converting struct + converted(union("objl", "obj"), 1, Map("a" -> Integer.valueOf(1), "b" -> "x")) shouldBe + ListMap("a" -> java.lang.Long.valueOf(1L), "b" -> "x") + // a NULL literal beside a struct changes nothing; in a converting struct column it is NULL + SetOperationValues.converters(resolved(union("ol", "NULL")), es8) shouldBe Seq(None, None) + converted(union("ol", "oi") + " UNION ALL SELECT NULL AS k FROM t", 2, List(null)) shouldBe + (null: Any) + } + + it should "convert a struct inside a struct, and the elements of a list, the same way" in { + converted(union("sl", "si"), 1, Map("p" -> Map("c" -> Integer.valueOf(3)))) shouldBe + ListMap("p" -> ListMap("c" -> java.lang.Long.valueOf(3L))) + // lists of values: every element in the element's type; a list of one stored bare + converted(union("ai", "av"), 0, List(Integer.valueOf(1), Integer.valueOf(2))) shouldBe + List("1", "2") + converted(union("ai", "av"), 0, Integer.valueOf(3)) shouldBe List("3") + converted(union("ai", "av"), 1, List("a")) shouldBe List("a") + converted(union("ai", "av"), 0, null) shouldBe (null: Any) + SetOperationValues.converters(resolved(union("ai", "NULL")), es8) shouldBe Seq(None, None) + converted(union("ai", "av") + " UNION ALL SELECT NULL AS k FROM t", 2, List(null)) shouldBe + (null: Any) + // lists of objects: every element the merged struct (p VARCHAR, q INT) + converted(union("items", "items2"), 1, List(Map("p" -> Integer.valueOf(1)))) shouldBe + List(ListMap("p" -> "1", "q" -> null)) + converted(union("items", "items2"), 0, Map("p" -> "a", "q" -> Integer.valueOf(2))) shouldBe + List(ListMap("p" -> "a", "q" -> Integer.valueOf(2))) + } + + it should "convert nothing for a statement it refuses, or beside a SELECT * branch" in { + SetOperationValues.converters( + resolved("SELECT d AS k FROM t UNION ALL SELECT n AS k FROM t"), + es8 + ) shouldBe Seq(None, None) + SetOperationValues.converters( + resolved("SELECT n AS k FROM t UNION ALL SELECT * FROM t"), + es8 + ) shouldBe Seq(None, None) + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/TemporalLiteralSearchSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/TemporalLiteralSearchSpec.scala index 8ba5e8b3..f42e7a2d 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/TemporalLiteralSearchSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/TemporalLiteralSearchSpec.scala @@ -409,19 +409,21 @@ class TemporalLiteralSearchSpec extends AnyFlatSpec with Matchers with BeforeAnd * At parse time a bare column is `SQLTypes.Any` and every pair passes vacuously, so a parse-only * assertion pins the parse-time half TWICE and the seam not at all — which is exactly what the * first version of this story shipped: replacing the seam's `MultiSearch.branchTypes(resolved)` - * with `Right(())` left the whole core suite green. This stub seeds `events` with `id: KEYWORD` - * and `amount: INT`, so the two bare columns ARE typed here and disagree. + * with `Right(())` left the whole core suite green. This stub seeds `events` with `event_ts: + * DATE` and `amount: INT`, so the two bare columns ARE typed here and disagree -- a pair DuckDB + * 1.5.5 refuses too (`id: KEYWORD` beside `amount` is a VARCHAR column there, answered since the + * lead's rule of 2026-10-06). */ it should "reject a branch TYPE mismatch at the seam, once schemas are attached" in { val client = seeded() client.search( - SelectStatement("SELECT id FROM events UNION ALL SELECT amount FROM events") + SelectStatement("SELECT event_ts FROM events UNION ALL SELECT amount FROM events") ) match { case ElasticFailure(error) => error.statusCode shouldBe Some(400) error.operation shouldBe Some("search") error.message should include("compatible types at column 1") - error.message should include("KEYWORD") + error.message should include("DATE") error.message should include("INT") case ElasticSuccess(other) => fail(s"expected a rejection, got $other") } diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index f300e78e..1364953a 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -615,11 +615,11 @@ LIMIT 10 OFFSET 20; > Bare `UNION` (with row de-duplication) is **not supported** and is rejected at parse time — the > two are not synonyms. It used to be accepted silently, returning only the first leg's rows. -All SELECT statements in a UNION ALL must be **strictly compatible**: +All SELECT statements in a UNION ALL must project: -- **same number of columns** -- **same column names** (after alias resolution) -- **same or implicitly compatible types** +- the **same number of columns**, matched **by position** (the result takes the first SELECT's + column names); +- **types that combine**, column by column (see [Column types](#column-types) below). If these conditions are not met, the Gateway raises a validation error before executing the query. @@ -643,11 +643,189 @@ The SQL Gateway executes `UNION ALL` using **Elasticsearch Multi‑Search (`_mse ### Notes - `UNION ALL` does **not** sort or deduplicate results. -- Column names in the final output are taken from the **first SELECT**. -- All subsequent SELECTs must produce columns with the **same names**. -- A type mismatch between branches is refused before execution, once each branch's mapping is read - (`SELECT id FROM a UNION ALL SELECT amount FROM b`, a `keyword` against a number); with no - mapping to read, nothing is refused on types and Elasticsearch answers. +- Column names in the final output are taken from the **first SELECT**; the columns of the other + SELECTs are matched by position, whatever they are called. + +### Column types + +Since `0.24.0`, a set operation is typed by ONE rule, the same at every depth: a column, each field +of an object (a `STRUCT`, a `GEO_POINT`), each element of a list (an `ARRAY`), and so on inside +them. The rule gives the type DuckDB gives — DuckDB being the engine that runs `UNION`, +`INTERSECT` and `EXCEPT`, so a statement answers the same types on both paths — `VARCHAR` where +DuckDB gives none, and refuses only what the relational engine fails on when the statement runs. +At one position, over the branches of one node (below): + +| Branches | Type | Example | +|---|---|---| +| numbers, and `BOOLEAN` | the widest: `TINYINT` < `SMALLINT` < `INT` < `BIGINT` < `REAL` < `DOUBLE`; a `BOOLEAN` reads as `1` / `0` | `n` (`INT`) with `x` (`DOUBLE`) is `DOUBLE` | +| a `DATE` and a `TIMESTAMP` | `TIMESTAMP`, the date at midnight UTC | `d` with `ts` | +| a text branch (`KEYWORD`, `TEXT`, a string literal) with any other type | `VARCHAR`, each value spelled as DuckDB spells its type (below) | `id` with `n`, `1` with `'a'`, `ts` with `s` | +| a `VARBINARY` with text branches only | `VARBINARY` (bytes), DuckDB's `BLOB`: a binary value is the bytes of its base64 text, a text value is cast to bytes (below) | `payload` with `name` | +| a `NULL` literal | the other branches' type | `n` with `NULL` is `INT` | +| a `DATE` or a `TIMESTAMP` with a number or a `BOOLEAN`, and neither a text nor a `VARBINARY` branch | refused | `d` with `n` | +| any other pair DuckDB gives no type: a `VARBINARY` with a number, a temporal or a `BOOLEAN`; a `TIME` with a `DATE`, a `TIMESTAMP`, a number or a `BOOLEAN` | `VARCHAR`, each value spelled as text (below) | `payload` with `price`, `CAST(ts AS TIME)` with `d` | +| an `ARRAY`, a `STRUCT` or a `GEO_POINT` with a type of another family, text included | refused | `location` with `name` | +| lists | a list of the type this rule gives their elements | `ARRAY` with `ARRAY` is `ARRAY` | +| objects and points | DuckDB's merged `STRUCT`: every field of every branch, in the order it first appears, matched whatever its case; a field a branch does not have is `NULL`, and each field takes the type this rule gives it over the branches that have it; points alone stay a `GEO_POINT` | `location` with `address` | + +So a field `a` that is an `INT` in one object and a `TIME` in another is a `VARCHAR`, as two such +columns are, and an `INT` beside a `DATE` is refused in a field as in a column, the field named by +its path. A `VARBINARY` counts as text beside a date and a number — the relational engine reads a +`binary` field as its base64 text — so `payload`, `d` and `n` make a `VARCHAR` column. + +A `GEO_POINT` is DuckDB's `STRUCT(lat DOUBLE, lon DOUBLE)`: in a column of points, or of points and +objects, every point is a `{lat, lon}` map whatever way Elasticsearch stored it — an object, a +`"lat,lon"` text, an array `[lon, lat]`, a geohash (the south-west corner of its cell, as +Elasticsearch reads it), a WKT `POINT (lon lat)` or a GeoJSON point. A `STRUCT`'s fields are taken +by name, in name order. A value that cannot be read as its branch's point or object, or several of +them in one row, fails the statement. A column read without a type (`ANY`), as an `ARRAY` +column of a mapping is read today, refuses nothing. + +A whole number literal is an `INT` (`SELECT 1`), a `BIGINT` past the `INT` range. The type +compares every branch, so `d`, `n` and `'1'` make a `VARCHAR` column although `d` and `n` alone +are refused; and each branch's value goes to the column's type from its OWN type (`d`, `ts` and +`'1'` spell `d` as a date, not as a midnight timestamp). + +A chain that mixes operators is typed as a tree, as DuckDB types it: `INTERSECT` binds first, then +the operators apply left to right, `INTERSECT` and `EXCEPT` typing their two sides, and a run of +ONE union kind (`UNION ALL` after `UNION ALL`, or `UNION` after `UNION`) is typed as one node, +whatever its order; a `UNION` beside a `UNION ALL`, a derived table, are separate nodes. Each node +of the tree takes the rule above, at every depth, over all its branches at once — a subtree's type +being one of them — so their order never changes the type. So `n UNION d UNION s` and `payload +INTERSECT d INTERSECT n` are `VARCHAR` columns, while `n INTERSECT d INTERSECT s`, `d INTERSECT n +INTERSECT payload` and `n EXCEPT x UNION ALL d` are refused: `n INTERSECT d`, and `n EXCEPT x` +beside `d`, are a number beside a date with no text. The relational engine runs these operators; +core runs `UNION ALL` itself. + +Every value holds its position's type, at every depth: wherever the branches' types differ at a +position — a column, a field, an element — every branch's value there is converted, and the +position holds ONE class: a `DOUBLE` a `Double`, a `BIGINT` a `Long`, a `TIMESTAMP` a +`ZonedDateTime` (a date-only, zone-less or epoch-milliseconds value included), a `VARBINARY` +bytes, a `VARCHAR` a string, a list a list of its element's type, an object a map of its fields +in order — which is the type JDBC and Arrow report, since they read it from the first value. A +position whose branches already have one type is returned as each branch reads it, nothing +converted: a branch over an aggregate keeps the form an aggregate answers, a `Double`, so `SELECT +MAX(n) ... UNION ALL SELECT n ...` answers `10.0` beside `3` and `5`. + +Each value is first read as the type its field is declared with, as Elasticsearch indexed it, and +only then converted: a value Elasticsearch accepted as text is the value its queries see. In an +integer field `"5"`, `"5.0"`, `"5.5"` and `5.5` are `5` (truncated toward zero), `"5e2"` is `500`; +in a `DOUBLE` field `"5"` is `5.0` (spelled `5.0` in a `VARCHAR` position) and `"1e3"` is `1000.0`; +in a `BOOLEAN` field `"true"` is `true` and an empty text `false`; an empty text in a number field +is no value. + +At a position that converts, a date field's value is read with the field's mapping `format` — +its own, or Elasticsearch's default `strict_date_optional_time||epoch_millis` — as the Elasticsearch major the statement runs +on reads that format: Elasticsearch 6 parses with Joda-Time, Elasticsearch 7 and later with +`java.time`, and the same mapping reads some texts differently across them. A `TIMESTAMP` is the +instant that major indexed, to the millisecond, as Elasticsearch keeps it, and a `DATE` its day, in +UTC. So `1706697000` in an `epoch_second` field is `2024-01-31 10:30:00`, not a moment of 1970; +`"1706697000.5"` there keeps its half second on Elasticsearch 7 and 8 and is truncated to the +second on Elasticsearch 6; `"2024/01/31"` in a `yyyy/MM/dd` field is a date, not a text; a format's +`||` alternatives are tried in order, and the first one that parses a value decides; in the default +format `"2024"` and `2024` are the year 2024. A value computed by the statement (a function, a +`CAST`) has no format: there, digits are epoch milliseconds and an ISO text is its instant. + +A value is read only where its reading is the same on every Java version core runs on (8, 11, 17 +and 21) and equal to the instant the Elasticsearch major indexed, measured value by value on +Elasticsearch 6.8.23, 7.17.29 and 8.18.3. That covers the built-in formats of each major's +documentation (the epoch, ISO, ordinal, week, time and `basic_` formats; in camel case on +Elasticsearch 6 and 7; with a leading `8`), and custom patterns — the pattern letters at each +width, alone and in the combinations measured, with `java.time` on Elasticsearch 7 and later and +in Elasticsearch 6.8's `8`-prefixed mode, with Joda-Time on Elasticsearch 6 — with a zone written +as `Z`, an offset, `UTC`, `GMT` or `UT`, together with epochs, negative ones included, and JSON +numbers. Elasticsearch's own readings are kept, however odd: in a `YYYY-MM-dd` field (a week-based year) +Elasticsearch 8 indexes `2024-01-31` as `2023-12-31`, the first day of the week-based year's first +week, and Elasticsearch 7 as `2024-01-01`; a year with no month and an hour (`2024T10` in a +`strict_date_optional_time` field) is `1970-01-01 10:00:00`. + +A value its field's format does not read fails the statement, naming the value, the field and the +format, and so does a value core cannot read without guessing, rather than be read as a date it +may not be: + +- a pattern letter, a width or a combination of letters that no measurement covers — two zone + letters included (an offset beside a zone id, Joda-Time's `ZZ ZZZ`) — and a name of + Elasticsearch 6.8's `8`-prefixed mode; +- a zone id or a zone name other than `UTC`, `GMT` and `UT`, in every format, the default + included: a region id (`Europe/Paris` in a `VV` or a Joda-Time `ZZZ` letter, in the default + format, or in brackets after an offset, `+05:00[Europe/Paris]`) and an abbreviation or a long + name (`CET`, `PST`, `Pacific Time`). Its offset comes from zone data, and the zone data core's + Java version holds is not the one Elasticsearch indexed the value with: `2023-07-01T12:00:00 + America/Mexico_City` is `17:00:00Z` on Elasticsearch 8.18.3 and `18:00:00Z` with older zone data, + and `IST` is UTC on Elasticsearch 8 and Israel time on Elasticsearch 7; +- a value whose reading would depend on the Java version core runs on: a localized offset (`O`, + `OOOO`, `ZZZZ`: `GMT+1`), which Java 8 parses differently from later versions; a fraction right + after digits of a variable width (`yyyyMMddHHmmssSSS`); and, in Elasticsearch 6.8's `8`-prefixed + mode, an AM/PM with no hour (`8yyyy a`), which Java 16 and later set to 06:00 or 18:00, and a week + past the last of its week-based year (`2020-53-7` in a `8YYYY-ww-e` field), which Java 21 + rejects; +- a value the response already gives as a date or a time, without the offset or the digits its + format reads (`10:30:00Z` in a `time` field); +- a fractional JSON number (`1706697000.0`) whose digits, which Elasticsearch read, the number no + longer pins to the millisecond. + +```sql +-- ts is a TIMESTAMP field of format 'yyyy/MM/dd' holding '31-01-2024', s a KEYWORD +SELECT ts AS k FROM t UNION ALL SELECT s AS k FROM t; +-- Error: Conversion Error: Could not read '31-01-2024' as a TIMESTAMP when casting from source column k: the date format 'yyyy/MM/dd' of field ts does not read it as Elasticsearch 8 does +-- a zone name +-- Error: Conversion Error: Could not read '2024-01-31T10:30:00 CET' as a TIMESTAMP when casting from source column k: core cannot read it with the date format 'yyyy-MM-dd'T'HH:mm:ss z' of field ts as Elasticsearch 8 does: the zone 'CET' is a region id or a zone name, whose offset comes from the zone data of the JVM reading it, not Elasticsearch's: core reads only Z, an offset, UTC, GMT and UT +``` + +The readings were measured on Elasticsearch 6.8.23, 7.17.29 and 8.18.3. The other minor versions +of a major are read as its measured one, and Elasticsearch 9 as Elasticsearch 8. + +A value with several values — a multi-valued field — has no one value of its position's type, so +at a position that converts its values it fails the statement, in the words DuckDB uses for the +same failure: `Conversion Error: Unimplemented type for cast (INTEGER -> VARCHAR[]) when casting +from source column k` for a multi-valued keyword beside an `INT`. A position whose branches have +one type converts nothing, and answers such a value as its branch reads it. + +```sql +-- id is a KEYWORD, n an INT: one VARCHAR column ('r1', ..., '3', ...) +SELECT id AS k FROM t UNION ALL SELECT n AS k FROM t; +-- n is an INT, x a DOUBLE: one DOUBLE column (3.0, ..., 0.25, ...) +SELECT n AS k FROM t UNION ALL SELECT x AS k FROM t; +-- refused, by name, before anything runs +SELECT d AS k FROM t UNION ALL SELECT n AS k FROM t; +-- Error: Set operation branches must project compatible types at column 1: branch 1 'k' is DATE, branch 2 'k' is INT +-- two objects whose field a is an INT and a DATE: refused, naming the field +SELECT oi AS k FROM t UNION ALL SELECT od AS k FROM t; +-- Error: Set operation branches must project compatible types at column 1: branch 1 'k' field 'a' is INT, branch 2 'k' field 'a' is DATE +``` + +Inside a `VARCHAR` column, each value is spelled as DuckDB spells its own type when it casts it to +text, so both paths answer the same string: + +| Branch type | Text | Examples | +|---|---|---| +| a whole number | its digits | `3`, `-2`, `3000000000` | +| `DOUBLE`, `REAL` | the shortest digits, with a fractional part; an exponent below `1e-04` or from `1e+16` | `250000000000.0`, `-2.5`, `5.0`, `1e-05`, `1e+16` | +| `BOOLEAN` | `true` / `false` | | +| `DATE` | `YYYY-MM-DD` | `2024-01-31`, `0044-03-15 (BC)` | +| `TIMESTAMP` | `YYYY-MM-DD HH:MM:SS`, in UTC, the fraction of a second only when there is one | `2024-01-31 10:30:00`, `2023-12-25 23:59:59.999`, `2024-01-31 00:00:00` for a date-only value | +| `TIME` | `HH:MM:SS`, the same fraction | `10:30:00`, `06:00:00.5` | +| `VARBINARY` | its base64 text, as Elasticsearch holds a `binary` field | `YWI=` | + +```sql +-- ts is a TIMESTAMP, s a KEYWORD: one VARCHAR column ('2024-01-31 10:30:00', ..., 'a', ...) +SELECT ts AS k FROM t UNION ALL SELECT s AS k FROM t; +-- x is a DOUBLE: ('250000000000.0', '1e-05', ..., 'a') +SELECT x AS k FROM t UNION ALL SELECT 'a' AS k FROM t; +``` + +A text value becomes bytes in a `VARBINARY` column as DuckDB casts a `VARCHAR` to a `BLOB`: each +ASCII character is its byte and `\xHH` (two hexadecimal digits) is the byte `HH`; a character +outside ASCII, or a backslash not followed by `x` and two hexadecimal digits, fails the statement, +as it fails in DuckDB (`Conversion Error: Invalid byte encountered in STRING -> BLOB conversion`). + +A `DOUBLE` is spelled as DuckDB spells it, digit for digit, including the few values its algorithm +does not shorten (`7.168e25` is `7.1680000000000004e+25`). A decimal literal is a `DOUBLE` here +(`1.50` is `1.5`), where DuckDB keeps it a `DECIMAL` (`1.50`). A value Elasticsearch returns as +text, in a column of another type, stays that text. + +The types are compared once each branch's mapping is read; with no mapping to read, nothing is +refused on types and Elasticsearch answers. --- @@ -841,6 +1019,19 @@ ORDER BY COUNT(*) DESC; wrap a transform (`HAVING MAX(YEAR(birthdate)) > 1990`, `ORDER BY MAX(ABS(age)) DESC`). - Arithmetic over aggregates is computed per group (`MAX(price) - MIN(price) AS price_range`); the operands are computed as hidden aggregations of the group. +- With **no `GROUP BY`**, the whole table is one group and the statement answers **one row**, + calculations included (since `0.24.0`): `SELECT MAX(price) - MIN(price) AS price_range FROM t`, + `COUNT(*) * 2`, `MAX(created) + 1` (a `DATE`), `DATEDIFF(MAX(created), MIN(created))`, + `GREATEST(MAX(created), MAX(updated))`. A calculation is evaluated as it is per group, with the + same rules. Every `SELECT` item must then be an aggregate, a calculation over aggregates or a + constant. +- Over **no matching document**, with no `GROUP BY`: a `MAX`, a `MIN` or an `AVG` is NULL, and so + is a calculation that reads one; `COUNT(*)` is `0` (`COUNT(*) * 2` is `0`, and + `HAVING COUNT(*) = 0` keeps the one row). A calculation over `COUNT(column)`, + `COUNT(DISTINCT column)` or `SUM(column)` is evaluated too — `COUNT(n) * 2` is `0`, `SUM(x) * 2` + is `0.0`, `SUM` over no value being `0.0` — on Elasticsearch 7.17 and later. Elasticsearch 6.8 + cannot evaluate it (it refuses the `keep_values` gap policy), so such a calculation is NULL + there. - `HAVING` may reference a `SELECT` aggregate by its alias (`COUNT(*) AS cnt ... HAVING cnt > 1`), including the alias of an arithmetic expression over aggregates (`... AS price_range ... HAVING price_range > 10`); `BETWEEN`, `IN` and `NOT` apply to aggregates as to columns. diff --git a/documentation/sql/operators.md b/documentation/sql/operators.md index 0b3152d9..345ee2f4 100644 --- a/documentation/sql/operators.md +++ b/documentation/sql/operators.md @@ -462,8 +462,9 @@ SELECT 10 % -3 AS result; **Description:** Since `0.24.0`, `+` and `-` follow the rules SQL engines apply to dates, in every place a script -runs: a row `SELECT`, a `WHERE`, a per-group calculation over aggregates (`MAX(d) - MIN(d) AS x`), -a `HAVING` over its alias, a computed column (`SCRIPT AS`) and a materialized view. +runs: a row `SELECT`, a `WHERE`, a per-group calculation over aggregates (`MAX(d) - MIN(d) AS x`) +or one over the whole table with no `GROUP BY`, a `HAVING` over its alias, a computed column +(`SCRIPT AS`) and a materialized view. | Expression | Result | Type | |---|---|---| diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 7eacc633..5505f76d 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -65,7 +65,8 @@ import com.sksamuel.elastic4s.searches.aggs.{ NestedAggregation, StatsAggregation, TermsAggregation, - TermsOrder + TermsOrder, + ValueCountAggregation } import com.sksamuel.elastic4s.searches.aggs.pipeline.BucketScriptPipelineAgg import com.sksamuel.elastic4s.searches.sort.FieldSort @@ -726,11 +727,15 @@ object ElasticAggregation { val script = metricSelectorForBucket(criteria, None, references) if (script.isEmpty) None else { + val documentCounts = documentCountNames(aggs) val bucketSelector = bucketSelectorAggregation( "having_filter", now(Script(script)), - extractMetricsPathForBucket(criteria, None, references) + readingDocumentCount( + extractMetricsPathForBucket(criteria, None, references), + documentCounts + ) ) Some( KeyedFiltersAggregation( @@ -739,13 +744,91 @@ object ElasticAggregation { app.softnetwork.elastic.sql.query.SingleSearch.WholeTableHavingBucket -> matchAllQuery() ), - subaggs = aggs.map(_.agg) :+ bucketSelector + subaggs = aggs.map(agg => readingDocumentCount(agg.agg, documentCounts)) :+ + bucketSelector ) ) } case _ => None } + /** The names of the aggregations of the whole-table bucket that count its DOCUMENTS: `COUNT(*)` + * (and `COUNT(_id)`, `COUNT(DISTINCT *)`), the `value_count` on `_index` that + * [[countAggregation]] emits -- a field every document carries exactly once. + */ + private def documentCountNames(aggs: Seq[ElasticAggregation]): Set[String] = + aggs + .map(_.agg) + .collect { + case count: ValueCountAggregation + if count.field.contains("_index") && count.script.isEmpty => + count.name + } + .toSet + + /** A pipeline of the whole-table bucket reads `COUNT(*)` as the bucket's own document count, + * `_count` -- equal to it by construction -- because Elasticsearch never takes `_count` for a + * gap. Over ZERO documents every other `buckets_path` is one, so `COUNT(*) * 2` was skipped + * (NULL) and `HAVING COUNT(*) = 0` dropped the bucket (no row); SQL answers `0` and one row. + * MEASURED on 6.8.23, 7.17.29 and 8.18.3: `_count` is `0` there on every major. + */ + private def readingDocumentCount( + paths: Map[String, String], + documentCounts: Set[String] + ): Map[String, String] = + if (documentCounts.isEmpty) paths + else + paths.map { case (param, path) => + param -> (if (documentCounts.contains(path)) "_count" else path) + } + + private def readingDocumentCount( + agg: AbstractAggregation, + documentCounts: Set[String] + ): AbstractAggregation = + agg match { + case script: BucketScriptPipelineAgg + if script.bucketsPaths.values.exists(documentCounts.contains) => + script.copy(bucketsPaths = readingDocumentCount(script.bucketsPaths, documentCounts)) + case other => other + } + + /** The synthetic whole-table bucket for a calculation over aggregates with NO `GROUP BY` (issue + * #413, [[app.softnetwork.elastic.sql.query.SingleSearch.wholeTableCalculation]]). + * + * The calculation is a `bucket_script`, and Elasticsearch accepts one only inside a multi-bucket + * aggregation: at the top level of `aggs` the search was refused on every major (6.8: `Only + * sibling pipeline aggregations are allowed at the top level`; 7.17 and 8.18: `must be declared + * inside of another aggregation`). The root aggregations therefore move inside the keyed + * `filters` aggregation [[wholeTableHavingAggregation]] builds -- the same name, the same one + * `match_all` bucket, which `parseAggregations` already reads as transparent -- and the + * calculation is evaluated in that bucket exactly as it is in a `GROUP BY` bucket. + * + * Not under a `HAVING`: that statement is [[wholeTableHavingAggregation]] 's, and when that one + * declines (no condition of the predicate resolves at this level) wrapping here would hang the + * calculation WITHOUT the predicate -- a filter silently dropped. It keeps the emission it had. + */ + def wholeTableCalculationAggregation( + request: app.softnetwork.elastic.sql.query.SingleSearch, + aggs: Seq[ElasticAggregation] + ): Option[Aggregation] = + if ( + request.wholeTableCalculation && request.having.flatMap(_.criteria).isEmpty && + aggs.exists(_.aggType.isBucketScript) + ) { + val documentCounts = documentCountNames(aggs) + Some( + KeyedFiltersAggregation( + app.softnetwork.elastic.sql.query.SingleSearch.WholeTableHavingAgg, + Seq( + app.softnetwork.elastic.sql.query.SingleSearch.WholeTableHavingBucket -> + matchAllQuery() + ), + subaggs = aggs.map(agg => readingDocumentCount(agg.agg, documentCounts)) + ) + ) + } else None + /** Generates the bucket_selector script for a given bucket */ def metricSelectorForBucket( diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index daadf6e0..0e0f66b7 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -229,10 +229,13 @@ package object bridge { // No buckets. A `HAVING` with no `GROUP BY` filters the ONE implicit whole-table group, // so the metric aggregations move inside a synthetic single-bucket `filters` aggregation // that the `having_filter` bucket_selector can hang from; without it the predicate was - // silently discarded (see ElasticAggregation.wholeTableHavingAggregation). + // silently discarded (see ElasticAggregation.wholeTableHavingAggregation). A calculation + // over the aggregates needs the same parent, its `bucket_script` being refused at the top + // level (issue #413, ElasticAggregation.wholeTableCalculationAggregation). case Nil => ElasticAggregation .wholeTableHavingAggregation(request, aggs, notNestedReferences) + .orElse(ElasticAggregation.wholeTableCalculationAggregation(request, aggs)) .map(Seq(_)) .getOrElse(aggs.map(_.agg)) case aggs => diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/HavingFunctionEmissionSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/HavingFunctionEmissionSpec.scala index d50dcf8f..860aa0a0 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/HavingFunctionEmissionSpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/HavingFunctionEmissionSpec.scala @@ -453,11 +453,13 @@ class HavingFunctionEmissionSpec extends AnyFlatSpec with Matchers { // --------------------------------------------------------------------------------------------- "a whole-table HAVING over a function of an aggregate" should "hang the selector off the synthetic bucket" in { + // `COUNT(*)` is read as the bucket's own `_count` there (issue #413): over no document every + // other path is a gap, and `HAVING COUNT(*) = 0` dropped its one row queryOf("SELECT COUNT(*) AS c FROM t HAVING GREATEST(COUNT(*), 0) > 1") shouldBe Seq( """{"query":{"match_all":{}},"size":0,"_source":false,""", """"aggs":{"__whole_table_having__":{"filters":{"filters":{"_all":{"match_all":{}}}},""", """"aggs":{"c":{"value_count":{"field":"_index"}},""", - """"having_filter":{"bucket_selector":{"buckets_path":{"c":"c"},""", + """"having_filter":{"bucket_selector":{"buckets_path":{"c":"_count"},""", """"script":{"source":"(params.c == null ? 0 : Math.max(params.c, 0)) > 1"}}}}}}}""" ).mkString } diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/WholeTableCalculationSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/WholeTableCalculationSpec.scala new file mode 100644 index 00000000..81f0edfa --- /dev/null +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/WholeTableCalculationSpec.scala @@ -0,0 +1,148 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime +import scala.jdk.CollectionConverters._ + +/** Issue #413 -- a calculation over aggregates with NO `GROUP BY` is evaluated in ONE bucket, as + * rendered by the HAND-MAINTAINED es6 bridge. + * + * Its `bucket_script` used to sit at the top level of `aggs`, which Elasticsearch refuses on every + * major (6.8: `Only sibling pipeline aggregations are allowed at the top level`; 7.17 and 8.18: + * `must be declared inside of another aggregation`). It now hangs, with the metrics it reads, from + * the single-bucket `filters` aggregation a `HAVING` without `GROUP BY` already uses, and the + * statement is an aggregation (`size` 0), not a row query. Each shape below was EXECUTED on + * Elasticsearch 6.8.23, 7.17.29 and 8.18.3 and answered one row. + */ +class WholeTableCalculationSpec extends AnyFlatSpec with Matchers { + + import scala.language.implicitConversions + + implicit def timestamp: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) + case other => throw new IllegalArgumentException(s"Not a single search: $other") + } + + private val mapper = new ObjectMapper() + + private def bodyOf(sql: String): JsonNode = { + val select: ElasticSearchRequest = SelectStatement(sql) + mapper.readTree(select.query) + } + + private def names(node: JsonNode): Seq[String] = + Option(node).map(_.fieldNames().asScala.toSeq.sorted).getOrElse(Nil) + + private val wrapper = SingleSearch.WholeTableHavingAgg + + "a calculation over aggregates without GROUP BY" should "hang from the whole-table bucket" in { + Seq( + "SELECT MAX(n) - MIN(n) AS v FROM t" -> Seq("max_n", "min_n", "v"), + "SELECT COUNT(*) * 2 AS v FROM t" -> Seq("count_all", "v"), + "SELECT MAX(d) + 1 AS v FROM t" -> Seq("max_d", "v"), + "SELECT MAX(n) AS m, MAX(n) - MIN(n) AS v FROM t" -> Seq("m", "min_n", "v") + ).foreach { case (sql, inside) => + withClue(s"[$sql] ") { + val body = bodyOf(sql) + // an aggregation, not a row query: no document is fetched + body.get("size").asInt() shouldBe 0 + names(body.get("aggs")) shouldBe Seq(wrapper) + val bucket = body.get("aggs").get(wrapper) + bucket + .get("filters") + .get("filters") + .get(SingleSearch.WholeTableHavingBucket) + .has( + "match_all" + ) shouldBe true + names(bucket.get("aggs")) shouldBe inside + bucket.get("aggs").get("v").has("bucket_script") shouldBe true + } + } + } + + it should "keep the HAVING's selector beside it, in the same bucket" in { + val bucket = + bodyOf("SELECT COUNT(*) * 2 AS v FROM t HAVING COUNT(*) > 1").get("aggs").get(wrapper) + names(bucket.get("aggs")) shouldBe Seq("count_all", "having_filter", "v") + } + + it should "leave a statement it does not cover as it was" in { + // a column beside the calculation has no value for the whole table: unchanged, still refused + // by Elasticsearch -- not answered with a NULL calculation per document + val mixed = bodyOf("SELECT id, MAX(n) - MIN(n) AS v FROM t") + names(mixed.get("aggs")) shouldBe Seq("max_n", "min_n", "v") + // per group: the GROUP BY bucket is the parent, as before + names(bodyOf("SELECT g, MAX(n) - MIN(n) AS v FROM t GROUP BY g").get("aggs")) shouldBe Seq("g") + // a plain aggregate needs no parent + names(bodyOf("SELECT MAX(n) AS m FROM t").get("aggs")) shouldBe Seq("m") + } + + private def calculation(sql: String): JsonNode = + bodyOf(sql).get("aggs").get(wrapper).get("aggs").get("v").get("bucket_script") + + private def gapPolicyOf(sql: String): Option[String] = + Option(calculation(sql).get("gap_policy")).map(_.asText()) + + "over zero documents, the whole-table bucket" should "read COUNT(*) as its own _count" in { + // Elasticsearch never takes `_count` for a gap: `COUNT(*) * 2` is 0 and `HAVING COUNT(*) = 0` + // keeps the bucket over no document, on every major (MEASURED on 6.8.23, 7.17.29, 8.18.3) + calculation("SELECT COUNT(*) * 2 AS v FROM t") + .get("buckets_path") + .get("count_all") + .asText() shouldBe + "_count" + val selector = bodyOf("SELECT COUNT(*) AS c FROM t HAVING COUNT(*) = 0") + .get("aggs") + .get(wrapper) + .get("aggs") + .get("having_filter") + .get("bucket_selector") + selector.get("buckets_path").elements().asScala.map(_.asText()).toSeq shouldBe Seq("_count") + // the COUNT(*) column itself is still the count it was + names( + bodyOf("SELECT COUNT(*) AS c FROM t HAVING COUNT(*) = 0").get("aggs").get(wrapper).get("aggs") + ) shouldBe Seq("c", "having_filter") + // a count of a column is not the bucket's document count + calculation("SELECT COUNT(n) * 2 AS v FROM t") + .get("buckets_path") + .get("count_n") + .asText() shouldBe + "count_n" + } + + it should "send no gap policy: Elasticsearch 6.8 refuses keep_values" in { + // `Invalid gap policy: [keep_values], accepted values: [insert_zeros, skip]` (MEASURED on + // 6.8.23), so COUNT(x) and COUNT(DISTINCT x) of a calculation are NULL over no document there + Seq( + "SELECT COUNT(n) * 2 AS v FROM t", + "SELECT SUM(x) * 2 AS v FROM t", + "SELECT COUNT(DISTINCT g) * 10 AS v FROM t", + "SELECT MAX(n) - MIN(n) AS v FROM t" + ).foreach(sql => withClue(s"[$sql] ")(gapPolicyOf(sql) shouldBe None)) + } +} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientAggregateWithoutGroupBySpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientAggregateWithoutGroupBySpec.scala new file mode 100644 index 00000000..134ac829 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientAggregateWithoutGroupBySpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientAggregateWithoutGroupBySpec extends AggregateWithoutGroupBySpec diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSetOperationTypeSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSetOperationTypeSpec.scala new file mode 100644 index 00000000..2ff48200 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSetOperationTypeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientSetOperationTypeSpec extends SetOperationTypeSpec diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientAggregateWithoutGroupBySpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientAggregateWithoutGroupBySpec.scala new file mode 100644 index 00000000..6b6bdcf5 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientAggregateWithoutGroupBySpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientAggregateWithoutGroupBySpec extends AggregateWithoutGroupBySpec diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSetOperationTypeSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSetOperationTypeSpec.scala new file mode 100644 index 00000000..d22311b7 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSetOperationTypeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientSetOperationTypeSpec extends SetOperationTypeSpec diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientAggregateWithoutGroupBySpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientAggregateWithoutGroupBySpec.scala new file mode 100644 index 00000000..6b6bdcf5 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientAggregateWithoutGroupBySpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientAggregateWithoutGroupBySpec extends AggregateWithoutGroupBySpec diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSetOperationTypeSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSetOperationTypeSpec.scala new file mode 100644 index 00000000..d22311b7 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSetOperationTypeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientSetOperationTypeSpec extends SetOperationTypeSpec diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientAggregateWithoutGroupBySpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientAggregateWithoutGroupBySpec.scala new file mode 100644 index 00000000..ce8b7e51 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientAggregateWithoutGroupBySpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientAggregateWithoutGroupBySpec extends AggregateWithoutGroupBySpec diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSetOperationTypeSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSetOperationTypeSpec.scala new file mode 100644 index 00000000..7fccc12b --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSetOperationTypeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientSetOperationTypeSpec extends SetOperationTypeSpec diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientAggregateWithoutGroupBySpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientAggregateWithoutGroupBySpec.scala new file mode 100644 index 00000000..ce8b7e51 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientAggregateWithoutGroupBySpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientAggregateWithoutGroupBySpec extends AggregateWithoutGroupBySpec diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSetOperationTypeSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSetOperationTypeSpec.scala new file mode 100644 index 00000000..7fccc12b --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSetOperationTypeSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientSetOperationTypeSpec extends SetOperationTypeSpec diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index e4613985..a8839e5e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -49,7 +49,8 @@ package object convert { //override def nullable: Boolean = value.nullable override def painless(context: Option[PainlessContext] = None): String = - SQLTypeUtils.coerce(value, targetType, context) + if (perGroupTemporal(context)) perGroupTemporalOf(value.painless(context)) + else SQLTypeUtils.coerce(value, targetType, context) /** A conversion renders as an expression -- EXCEPT the one case `toPainless` below folds: an * un-typed (no schema) identifier narrowed to DATE or TIME gets `.toLocalDate()` / @@ -115,6 +116,7 @@ package object convert { } case _ => // do nothing } + if (perGroupTemporal(context)) return perGroupTemporalOf(base) val ret = SQLTypeUtils.coerce(base, sourceType, targetType, sourceNullable, context) val bloc = ret.startsWith("{") && ret.endsWith("}") val retWithBrackets = if (bloc) ret else s"{ $ret }" @@ -176,6 +178,29 @@ package object convert { */ private def sourceType: SQLType = arithmetic.map(_.out).getOrElse(value.baseType) + /** A temporal converted PER GROUP to a `DATE` or a `TIMESTAMP`, as in `(MAX(d) + 1)::DATE` or + * `GREATEST(MAX(d), MAX(d2))::TIMESTAMP`. + * + * Per group (no context: a `bucket_script`, a `bucket_selector`, and the per-group calculation + * a view's pivot runs, `SingleSearch.transformBucketScripts`, which is the same rendering) + * there is no document, and a temporal is the number of milliseconds Elasticsearch stores a + * `date` in: the metric it computed (`MAX(d)`) or a date calculation over it, handed back in + * milliseconds (`ArithmeticExpression.dateArithmeticPainless`), which is also what a + * `bucket_script` must return. The document-level conversion renders a `java.time` value + * instead (`.toInstant()…toLocalDate()`), and Elasticsearch failed the search on the number: + * `member method [double, toInstant/0] not found`. Converted in milliseconds, a `DATE` is the + * start of its UTC day (DuckDB's `CAST(TIMESTAMP AS DATE)` keeps the date part) and a + * `TIMESTAMP` the instant itself (a `DATE` is already its day's start). + */ + private def perGroupTemporal(context: Option[PainlessContext]): Boolean = + context.isEmpty && value.hasAggregation && sourceType.isTemporal && + (targetType == SQLTypes.Date || targetType == SQLTypes.Timestamp || + targetType == SQLTypes.DateTime) + + /** [[perGroupTemporal]]'s conversion of the per-group value `base`, in milliseconds. */ + private def perGroupTemporalOf(base: String): String = + if (targetType == SQLTypes.Date) s"(Math.floor($base / 86400000.0) * 86400000.0)" else base + /** Can the value converted be NULL? Asked of the chain's innermost PRODUCER: the nameless * identifier around an arithmetic answers `false` whatever its operands hold, so the * conversion was emitted unguarded -- `((long) )` failed with `Cannot cast null to a diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 537667c4..e7a9de0c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -16,8 +16,8 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} -import app.softnetwork.elastic.sql.operator.{AND, OR, SetOperator, UNION} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.operator.{AND, OR, SetOperator, UNION, UNION_DISTINCT} import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.schema.{ sqlConfig, @@ -838,6 +838,10 @@ package object query { .filterNot(_.isScriptField) .filterNot(_.nested) .filterNot(_.isAggregation) + // a calculation over aggregates (`MAX(n) - MIN(n)`) is a metric of the whole table, never + // a document field: listing it made the statement fetch documents and page through them + // as a ROW query (issue #413) -- see `wholeTableCalculation` + .filterNot(_.isBucketScript) .map(_.sourceField) .filterNot(f => excludes.contains(f)) .distinct @@ -977,6 +981,30 @@ package object query { lazy val wholeTableHaving: Boolean = groupBy.isEmpty && having.flatMap(_.criteria).isDefined && !returnsRows + /** A calculation over aggregates with NO `GROUP BY` -- `SELECT MAX(n) - MIN(n) FROM t`, + * `COUNT(*) * 2`, `MAX(d) + 1`, `DATEDIFF(MAX(d), MIN(d))` (issue #413): the whole table is + * ONE implicit group, and the statement answers ONE row. + * + * The calculation is a `bucket_script`, which Elasticsearch accepts only inside a multi-bucket + * aggregation -- at the top level of `aggs` it refused the search on every major (MEASURED on + * 6.8.23, 7.17.29 and 8.18.3). So the bridge gives it the parent [[wholeTableHaving]] already + * uses: the root aggregations move inside the single-bucket keyed `filters` aggregation named + * [[SingleSearch.WholeTableHavingAgg]], whose one bucket is the whole table. The calculation + * then means exactly what it means per group -- same script, same `buckets_path`, same NULL + * contract + * -- by construction rather than by a second, client-side evaluation of the expression. + * + * Only when EVERY SELECT item is an aggregate, a calculation over aggregates or a + * row-invariant literal: an item naming a column has no value for the whole table (`SELECT id, + * MAX(n) - MIN(n) FROM t`), and such a statement keeps the answer it had. + */ + lazy val wholeTableCalculation: Boolean = + groupBy.isEmpty && !windowFunctions.exists(_.isWindowing) && + select.fields.exists(_.isBucketScript) && + select.fields.forall(f => + f.hasAggregation || SingleSearch.isRowInvariantLiteral(f.identifier) + ) + private lazy val selectAggs: Seq[Field] = select.fieldsWithComputedAliases .filter(f => f.isAggregation || f.isBucketScript) @@ -2646,43 +2674,620 @@ package object query { } } - /** Positional type compatibility with the FIRST declaring branch, using the ONE pairwise - * predicate the cast/coercion layer already trusts (`SQLTypeUtils.matches`: same family, or - * either side `Any`/`Null`). + /** The type of ONE position of a set operation -- a column, a field of a struct, the element of + * a list -- as the relational engine types it: ONE rule at every depth ([[typeNode]], the + * lead's rules of 2026-10-07). * - * At parse time a bare column is `Any` and passes; once `SearchApi.resolveWithSchema` has - * attached each branch's schema the SAME method rejects a `keyword UNION long` pair before any - * request is sent — which is why this is `private[elastic]` and not `private[query]`. - * - * 🔴 Deliberately NOT `leastCommonSuperType`, which answers `Varchar` for `{BigInt, Varchar}` - * — a super type, not a verdict — and would let `SELECT 1 … UNION SELECT 'a' …` through. And - * this is the ONLY type guard: DuckDB performs implicit casting across set-operation branches - * (MEASURED on 1.5.5.1: `BIGINT UNION VARCHAR` succeeds and yields a VARCHAR column), so the - * engine is never a backstop. + * @param sqlType + * the position's type, the families DuckDB does not tell apart folded (`KEYWORD`, `TEXT` and + * `CHAR` are `VARCHAR`, a `DATETIME` a `TIMESTAMP`); a list's is `ARRAY`; `ANY` where it is not known (no schema), `NULL` for a `NULL` literal, `NUMERIC` / + * `TEMPORAL` for a number / a temporal of unknown width + * @param fields + * a `STRUCT`'s or a `GEO_POINT`'s fields, in DuckDB's order ([[StructField]]) -- a + * `GEO_POINT` is `STRUCT(lat DOUBLE, lon DOUBLE)`; empty for any other type, and for a + * `STRUCT` whose fields are not known, whose values are then answered as read + * @param element + * a list's element type -- a list of objects' carrying their fields; `None` for any other + * type + * @param format + * a BRANCH's date FIELD's mapping `format` -- its own (`yyyy/MM/dd`, `epoch_second`, ...), + * else Elasticsearch's default ([[TemporalLiterals.DefaultDateFormat]]) -- by which core + * reads its values where they convert, as the Elasticsearch major reads them; `None` for a + * computed value, for any other type, and on a column's [[SetOperationColumn.result]], which + * no field decides + */ + final case class SetOperationType( + sqlType: SQLType, + fields: Seq[StructField] = Nil, + element: Option[SetOperationType] = None, + format: Option[String] = None + ) + + object SetOperationType { + + /** A `GEO_POINT` to the relational engine: `STRUCT(lat DOUBLE, lon DOUBLE)`. */ + lazy val GeoPoint: SetOperationType = + SetOperationType(SQLTypes.GeoPoint, StructField.GeoPoint) + } + + /** One field of a `STRUCT` or a `GEO_POINT` position: its name and its type, at every depth. */ + final case class StructField(name: String, valueType: SetOperationType) { + def sqlType: SQLType = valueType.sqlType + def fields: Seq[StructField] = valueType.fields + } + + object StructField { + + /** A field of a scalar type. */ + def apply(name: String, sqlType: SQLType): StructField = + StructField(name, SetOperationType(sqlType)) + + /** A `GEO_POINT`'s fields: `lat` and `lon`, two `DOUBLE`s. */ + lazy val GeoPoint: Seq[StructField] = + Seq(StructField("lat", SQLTypes.Double), StructField("lon", SQLTypes.Double)) + } + + /** One column of a set operation, typed by the lead's rules of 2026-10-07 ([[typeNode]]) -- the + * type DuckDB 1.5.5 gives it where DuckDB gives one, the relational engine that runs `UNION`, + * `INTERSECT` and `EXCEPT` being DuckDB; `VARCHAR` where it gives none; a refusal only where + * the relational engine fails -- at the column and at every depth inside it. The contract read + * by core's `UNION ALL` and by the relational engine (arrow): * - * Column NAMES are NOT checked: SQL takes the first branch's. + * @param result + * the column's type, at every depth ([[SetOperationType]]) + * @param branches + * each branch's type at this column, by branch index, at every depth; `None` for a branch + * that projects `*`, which declares no projection */ - private[elastic] def branchTypes(requests: Seq[SingleSearch]): Either[String, Unit] = { - val known = requests.zipWithIndex.flatMap { case (r, i) => declared(r).map(f => (i, f)) } - known.headOption match { - case None => Right(()) - case Some((i0, f0)) => - val offending = for { - (i, f) <- known.tail - ((a, b), pos) <- f0.zip(f).zipWithIndex - if !SQLTypeUtils.matches(a.identifier.out, b.identifier.out) - } yield (i, pos, a, b) - offending.headOption match { - case Some((i, pos, a, b)) => - Left( - s"Set operation branches must project compatible types at column ${pos + 1}: " + - s"branch ${i0 + 1} '${a.outputName}' is ${a.identifier.out.typeId}, branch " + - s"${i + 1} '${b.outputName}' is ${b.identifier.out.typeId}" + final case class SetOperationColumn( + result: SetOperationType, + branches: Seq[Option[SetOperationType]] + ) { + + def resultType: SQLType = result.sqlType + + def branchTypes: Seq[Option[SQLType]] = branches.map(_.map(_.sqlType)) + + def fields: Seq[StructField] = result.fields + + /** Whether the branches' values change class to reach [[result]]: when a branch's type + * differs from it at the column or anywhere inside ([[SetOperationColumn.converts]]) -- + * never for a column whose branches already agree (a `NULL` literal aside), the + * overwhelmingly common case, which pays nothing; never for a type that is not known, nor + * beside a `SELECT *` branch, whose values are not typed here. A column of points always + * converts: every point, stored as an object, a text, an array, a geohash or a WKT `POINT`, + * becomes `{lat, lon}`. + */ + lazy val converts: Boolean = + branches.forall(_.isDefined) && (result.sqlType == SQLTypes.GeoPoint || + SetOperationColumn.converts(result, branches.flatten)) + } + + object SetOperationColumn { + + /** The scalar types a value is converted to: every one a position can REACH from branches of + * different types. `DATE`, `TIME` and `BOOLEAN` are reached only by branches of that same + * type (a `NULL` aside), so a value never needs converting to them. + */ + private[query] val Converted: Set[SQLType] = Set( + SQLTypes.Varchar, + SQLTypes.Double, + SQLTypes.Real, + SQLTypes.BigInt, + SQLTypes.Int, + SQLTypes.SmallInt, + SQLTypes.TinyInt, + SQLTypes.Timestamp, + SQLTypes.VarBinary + ) + + /** Whether the values at ONE position -- a column, a field, an element -- convert: when a + * branch's type there differs from `result`, at that position or inside it. Then EVERY + * branch's value there is converted, recursively; a position whose branches all agree keeps + * its values as read. A point is a point: two `GEO_POINT`s agree. + */ + def converts(result: SetOperationType, branches: Seq[SetOperationType]): Boolean = + branches.exists(differs(_, result)) + + private def differs(branch: SetOperationType, result: SetOperationType): Boolean = + if (branch.sqlType == SQLTypes.Null) false + else + result.sqlType match { + case SQLTypes.Any | SQLTypes.GeoPoint => false + case SQLTypes.Struct => + result.fields.nonEmpty && (branch.sqlType == SQLTypes.GeoPoint || + branch.fields.map(_.name) != result.fields.map(_.name) || + branch.fields.zip(result.fields).exists { case (b, r) => + differs(b.valueType, r.valueType) + }) + case SQLTypes.Array(_) => + (branch.element, result.element) match { + case (Some(b), Some(r)) => differs(b, r) + case _ => false + } + case scalar => branch.sqlType != scalar && Converted.contains(scalar) + } + + /** Each branch's type at the field `name` of a struct position, matched whatever its case -- + * `None` for a branch that has no such field, whose value there is NULL. + */ + def fieldTypes( + branches: Seq[Option[SetOperationType]], + name: String + ): Seq[Option[SetOperationType]] = + branches.map(_.flatMap(_.fields.find(_.name.equalsIgnoreCase(name)).map(_.valueType))) + + /** Each branch's element type at a list position. */ + def elementTypes(branches: Seq[Option[SetOperationType]]): Seq[Option[SetOperationType]] = + branches.map(_.flatMap(_.element)) + } + + /** DuckDB's numeric lattice over core's types: a `BOOLEAN` reads as 1 / 0 beside any number, + * and the wider type wins (`INTEGER` with `BIGINT` is `BIGINT`, with `REAL` is `REAL`, with + * `DOUBLE` is `DOUBLE`). MEASURED on DuckDB 1.5.5, every pair. + */ + private val numberRank: Map[SQLType, Int] = Map( + SQLTypes.Boolean -> 0, + SQLTypes.TinyInt -> 1, + SQLTypes.SmallInt -> 2, + SQLTypes.Int -> 3, + SQLTypes.BigInt -> 4, + SQLTypes.Real -> 5, + SQLTypes.Double -> 6 + ) + + /** A `DATE` or a `TIMESTAMP`: the types rule 1 refuses beside a number ([[typeNode]]). */ + private def dateCarrying(t: SQLType): Boolean = t == SQLTypes.Date || t == SQLTypes.Timestamp + + /** A number of any width -- `NUMERIC`, a number of unknown width, included -- or a `BOOLEAN`, + * the bottom of DuckDB's numeric lattice. + */ + private def numberOrBoolean(t: SQLType): Boolean = + t == SQLTypes.Numeric || numberRank.contains(t) + + /** An `ARRAY`, a `STRUCT` or a `GEO_POINT` (a `STRUCT` to the relational engine): a type no + * other type is cast to ([[typeNode]]). + */ + private def nested(t: SQLType): Boolean = + t match { + case SQLTypes.Array(_) | SQLTypes.Struct | SQLTypes.GeoPoint => true + case _ => false + } + + /** The type a branch's column has for a set operation, before the text types fold to `VARCHAR`: + * the type the item is DECLARED with (`ComparisonRule.declaredTypeOf`: a `DATE` column is a + * `DATE` although Painless reads a `TIMESTAMP`), with DuckDB's reading of a whole number + * literal -- an `INTEGER` when it fits one, as `SELECT 1` is. + */ + private def declaredOperandType(field: Field): SQLType = { + val id = field.identifier + if (id.name.isEmpty) + id.functions match { + case (l: LongValue) :: Nil if l.value.isValidInt => SQLTypes.Int + case _ => ComparisonRule.declaredTypeOf(id) + } + else ComparisonRule.declaredTypeOf(id) + } + + /** A scalar declared type with the type families DuckDB does not tell apart folded: `KEYWORD`, + * `TEXT` and `CHAR` are `VARCHAR`, a `DATETIME` is a `TIMESTAMP`. + */ + private def unionOperandType(declared: SQLType): SQLType = + declared match { + case t if t.isText => SQLTypes.Varchar + case SQLTypes.Char => SQLTypes.Varchar + case SQLTypes.DateTime => SQLTypes.Timestamp + case other => other + } + + /** The type DuckDB 1.5.5 gives two non-NULL, non-text, non-nested branch types together, or + * `None` when it gives them none: a temporal beside a number or a `BOOLEAN`, a `TIME` beside a + * `DATE` or a `TIMESTAMP`, a `VARBINARY` beside another type. `NUMERIC` and `TEMPORAL` (a + * number or a temporal of unknown width) join their own family. + */ + private def joinTypes(a: SQLType, b: SQLType): Option[SQLType] = + if (a == b) Some(a) + else + (numberRank.get(a), numberRank.get(b)) match { + case (Some(x), Some(y)) => Some(if (x >= y) a else b) + case _ => + if (numberOrBoolean(a) && numberOrBoolean(b)) + // a number of unknown width is a DOUBLE beside a DOUBLE, the top of the lattice + Some( + if (a == SQLTypes.Double || b == SQLTypes.Double) SQLTypes.Double + else SQLTypes.Numeric ) - case None => Right(()) + else if ( + (a == SQLTypes.Temporal && b.isTemporal) || (b == SQLTypes.Temporal && a.isTemporal) + ) + // a temporal of unknown kind is never refused beside another temporal + Some(SQLTypes.Temporal) + else if (dateCarrying(a) && dateCarrying(b)) Some(SQLTypes.Timestamp) + else None + } + + /** One operand of a node at one position: a branch, or a subtree of the operator tree -- the + * branch that decides its type, the one a refusal names, with the item and the type it is + * declared with -- and its type THERE: the column's, or, inside, a field's or an element's. + */ + private final case class Operand( + branch: Int, + field: Field, + declared: SQLType, + valueType: SetOperationType + ) { + def folded: SQLType = valueType.sqlType + def at(t: SetOperationType): Operand = copy(valueType = t) + } + + private def operandOf(branch: Int, field: Field): Operand = { + val declaredType = declaredOperandType(field) + val column = field.identifier match { + case id: GenericIdentifier if id.functions.isEmpty => id.col + case _ => None + } + Operand(branch, field, declaredType, positionType(declaredType, column)) + } + + /** A declared type as a set operation types it, built once at every depth: a scalar folded + * ([[unionOperandType]]); a list with its element's type; a `STRUCT` with its fields -- its + * column's, read from the schema, BY NAME: the schema read from a mapping holds an object's + * fields in no reliable order -- a list of objects' element too; a `GEO_POINT` as `{lat, + * lon}`. A `STRUCT` whose column is not known has no fields. + */ + private def positionType(declared: SQLType, column: Option[Column]): SetOperationType = + declared match { + case SQLTypes.GeoPoint => SetOperationType.GeoPoint + case SQLTypes.Struct => SetOperationType(SQLTypes.Struct, fieldsOf(column)) + case SQLTypes.Array(element) => + val e = positionType(element, column) + SetOperationType(SQLTypes.Array(e.sqlType), element = Some(e)) + case scalar => + SetOperationType( + unionOperandType(scalar), + format = + if (scalar.isTemporal) + column.map( + app.softnetwork.elastic.sql.query.TemporalLiterals.FieldFormat.of(_).spec + ) + else None + ) + } + + /** A type as a column's RESULT holds it: no field decides it, so no date format. */ + private def resultOf(t: SetOperationType): SetOperationType = + t.copy( + fields = t.fields.map(f => f.copy(valueType = resultOf(f.valueType))), + element = t.element.map(resultOf), + format = None + ) + + private def fieldsOf(column: Option[Column]): Seq[StructField] = + column.toSeq.flatMap(_.multiFields).sortBy(_.name).map { c => + StructField(c.name, positionType(c.dataType, Some(c))) + } + + /** Two operands a node refuses, in branch order, at one position: `path` names it -- empty for + * the column, the field's path inside it (`p.c` for the field `c` of the field `p`). + */ + private final case class Conflict(a: Operand, b: Operand, path: Seq[String]) + + private def conflict(a: Operand, b: Operand, path: Seq[String]): Conflict = + if (a.branch <= b.branch) Conflict(a, b, path) else Conflict(b, a, path) + + /** Nested operands -- lists, structs, points -- typed together, every operand of a node at + * once, by the SAME rule as a column ([[typeNode]]): lists by their elements; structs and + * points by their fields, merged as DuckDB 1.5.5 merges them (MEASURED): every field of every + * operand in the order it first appears (`STRUCT(lat, lon)` beside `STRUCT(a, b)` is + * `STRUCT(lat, lon, a, b)`), a field several of them have, matched whatever its case, keeping + * its first name and typed over all of them at once; points alone stay a `GEO_POINT`. A list + * beside a struct is refused, as a list's element or a field the rule refuses is -- an element + * named by its lists. + */ + private def nestedNode(operands: Seq[Operand], path: Seq[String]): Either[Conflict, Operand] = { + val (lists, structs) = operands.partition(o => !structLike(o.folded)) + if (lists.nonEmpty && structs.nonEmpty) Left(conflict(lists.head, structs.head, path)) + else if (structs.isEmpty) { + val elements = + lists.map(o => o.at(o.valueType.element.getOrElse(SetOperationType(SQLTypes.Any)))) + typeNode(elements, path) match { + case Right(e) => + Right( + lists.head.at(SetOperationType(SQLTypes.Array(e.folded), element = Some(e.valueType))) + ) + case Left(Conflict(a, b, at)) if at == path => + // a list's elements refused: the lists are named, with their types + def list(o: Operand) = lists.find(_.branch == o.branch).getOrElse(o) + Left(conflict(list(a), list(b), path)) + case Left(refused) => Left(refused) + } + } else if (structs.forall(_.folded == SQLTypes.GeoPoint)) Right(structs.head) + // a STRUCT whose fields are not known has nothing to merge: the position's fields are not + // known either + else if (structs.exists(_.valueType.fields.isEmpty)) + Right(structs.head.at(SetOperationType(SQLTypes.Struct))) + else { + val names = structs.flatMap(_.valueType.fields.map(_.name)).foldLeft(Vector.empty[String]) { + (seen, name) => if (seen.exists(_.equalsIgnoreCase(name))) seen else seen :+ name + } + names + .foldLeft[Either[Conflict, Vector[StructField]]](Right(Vector.empty)) { + case (Right(merged), name) => + val having = structs.flatMap { o => + o.valueType.fields.find(_.name.equalsIgnoreCase(name)).map(f => o.at(f.valueType)) + } + typeNode(having, path :+ name).map(t => merged :+ StructField(name, t.valueType)) + case (refused, _) => refused } + .map(fields => structs.head.at(SetOperationType(SQLTypes.Struct, fields))) } } + + private def structLike(t: SQLType): Boolean = t == SQLTypes.Struct || t == SQLTypes.GeoPoint + + /** The type of ONE node at ONE position -- every operand of a `UNION ALL`, of a run of one + * union kind, or the two sides of an `INTERSECT` or an `EXCEPT`, at a column, a struct's field + * or a list's element alike: ONE rule at every depth -- as the operand that decides it, its + * type the node's; or, `Left`, the two operands the node refuses, at `path`. + * + * The lead's rules of 2026-10-07, which refuse only what fails when the statement runs + * (MEASURED through arrow, the relational engine, statement by statement): + * + * - REFUSED, a NESTED type beside another type: an `ARRAY`, a `STRUCT` or a `GEO_POINT` + * beside a type other than its own -- text included, which the relational engine casts to + * a list or a struct only when it spells one. Lists are typed by their elements, structs + * and points by their fields, each by this same rule ([[nestedNode]]). + * - REFUSED, a DATE beside a NUMBER: a `DATE` or a `TIMESTAMP` beside a `BOOLEAN` or a + * number of any width, in either order (DuckDB's `Unimplemented type for cast (INTEGER -> + * DATE)`), in a node with neither a text nor a `VARBINARY` operand: either makes the node + * text, a `VARBINARY` being its base64 text to the relational engine. + * - ANSWERED, everything else: the type DuckDB gives the node where it gives one (the widest + * number, a `TIMESTAMP` for a `DATE` beside a `TIMESTAMP`, `VARCHAR` beside text, + * `VARBINARY` for a `VARBINARY` beside text only, the type every operand shares), and + * `VARCHAR` where it gives none -- a `VARBINARY` beside a number, a temporal or a + * `BOOLEAN`, a `TIME` beside a date or a number -- every value then spelled as text. + * + * The type does not depend on the operands' order. A node whose type is not known (`ANY`) + * refuses nothing; a `NULL` literal takes the others' type. + */ + private def typeNode(operands: Seq[Operand], path: Seq[String]): Either[Conflict, Operand] = + operands.find(_.folded == SQLTypes.Any) match { + case Some(unknown) => Right(unknown) + case None => + val valued = operands.filterNot(_.folded == SQLTypes.Null) + valued.headOption match { + case None => Right(operands.head) + case Some(first) => + valued.find(o => nested(o.folded)) match { + case Some(list) => + // a nested type beside its own family only + valued.find(o => !nested(o.folded)) match { + case Some(other) => Left(conflict(list, other, path)) + case None => nestedNode(valued, path) + } + case None => + valued.find(_.folded == SQLTypes.Varchar) match { + case Some(text) => + // DuckDB: a BLOB beside text is a BLOB, each text value cast to bytes; text + // beside any other mix is VARCHAR + val others = valued.filterNot(_.folded == SQLTypes.Varchar) + if (others.nonEmpty && others.forall(_.folded == SQLTypes.VarBinary)) + Right(others.head) + else Right(text) + case None => + val binary = valued.exists(_.folded == SQLTypes.VarBinary) + ( + if (binary) None + else commonType(valued.filter(o => dateCarrying(o.folded))), + commonType(valued.filter(o => numberOrBoolean(o.folded))) + ) match { + case (Some(date), Some(number)) => Left(conflict(date, number, path)) + case _ => + // every pair must have a common type, so the order cannot change it + val joined = + if ( + valued + .combinations(2) + .forall(pair => + joinTypes(pair.head.folded, pair(1).folded).isDefined + ) + ) commonType(valued) + else None + Right(joined.getOrElse(first.at(SetOperationType(SQLTypes.Varchar)))) + } + } + } + } + } + + /** DuckDB's common type of non-NULL, non-text `operands`, as the operand that decides it -- the + * first one with the widest type -- or `None` when two of them have none ([[joinTypes]]). + */ + private def commonType(operands: Seq[Operand]): Option[Operand] = + operands.headOption.flatMap { first => + operands.tail.foldLeft[Option[Operand]](Some(first)) { + case (Some(acc), e) => + joinTypes(acc.folded, e.folded).map(joined => + (if (joined == acc.folded) acc else e).at(SetOperationType(joined)) + ) + case (none, _) => none + } + } + + /** The refusal of column `position`, naming the two operands a node refuses there: at the + * column, by the type each is declared with; inside it, by the field's path and its type + * there. + */ + private def refusal(position: Int, refused: Conflict): String = { + val Conflict(a, b, path) = refused + def side(o: Operand): String = + if (path.isEmpty) s"branch ${o.branch + 1} '${o.field.outputName}' is ${o.declared.typeId}" + else + s"branch ${o.branch + 1} '${o.field.outputName}' field '${path.mkString(".")}' is " + + o.folded.typeId + s"Set operation branches must project compatible types at column ${position + 1}: " + + side(a) + ", " + side(b) + } + + /** The type of each column of a `UNION ALL`, position by position over the declaring branches + * -- every branch one node ([[typeNode]]): `columnTypes(requests, operators)` with `UNION ALL` + * everywhere -- or, `Left`, the column refused and why, by name. + * + * DuckDB's types, MEASURED pair by pair over core's types: a column takes the WIDEST type of + * its branches in the numeric lattice (`BOOLEAN` < `TINYINT` < `SMALLINT` < `INTEGER` < + * `BIGINT` < `REAL` < `DOUBLE`); a `DATE` beside a `TIMESTAMP` is a `TIMESTAMP`; a text branch + * makes the column `VARCHAR` whatever the others are, even two the column could not otherwise + * combine (`d UNION ALL n UNION ALL '1'`); a `VARBINARY` beside text makes a `VARBINARY` + * column (DuckDB's `BLOB`); a `NULL` literal takes the others' type. Where DuckDB gives no + * type, the column is `VARCHAR` -- and a date beside a number, or a nested type beside + * another, is refused ([[typeNode]]). Each branch's value is then converted from ITS type to + * the column's, never through an intermediate one: `d UNION ALL ts UNION ALL 'x'` spells `d` + * as a date, not as a midnight timestamp. + */ + private[elastic] def columnTypes( + requests: Seq[SingleSearch] + ): Either[String, Seq[SetOperationColumn]] = + columnTypes(requests, Nil) + + /** The operator tree as DuckDB types it: [[precedenceTree]] -- `INTERSECT [ALL]` first, then + * left to right -- with every run of ONE union kind (`UNION ALL` after `UNION ALL`, or `UNION` + * after `UNION`) typed as one node, and every other operator typing its two sides. + */ + private sealed trait TypingNode + private final case class TypingLeaf(branch: Int) extends TypingNode + private final case class TypingRun(children: Seq[TypingNode]) extends TypingNode + + private def typingTree(node: SetOpNode): TypingNode = + node match { + case SetOpLeaf(branch) => TypingLeaf(branch) + case SetOpBranch(left, operator, right) => + def flattened(child: SetOpNode): Seq[TypingNode] = + child match { + case SetOpBranch(l, inner, r) + if inner == operator && (operator == UNION || operator == UNION_DISTINCT) => + flattened(l) ++ flattened(r) + case other => Seq(typingTree(other)) + } + TypingRun(flattened(left) ++ flattened(right)) + } + + /** The operand a node of the tree folds to at one column -- `None` when no branch under it + * declares one (`SELECT *`) -- or the two operands refused under it. + */ + private def typeOf( + node: TypingNode, + operand: Int => Option[Operand] + ): Either[Conflict, Option[Operand]] = + node match { + case TypingLeaf(branch) => Right(operand(branch)) + case TypingRun(children) => + children + .foldLeft[Either[Conflict, Vector[Operand]]](Right(Vector.empty)) { + case (Right(typed), child) => typeOf(child, operand).map(typed ++ _) + case (conflict, _) => conflict + } + .flatMap(typed => if (typed.isEmpty) Right(None) else typeNode(typed, Nil).map(Some(_))) + } + + /** The type of each column of a set operation whose branches are joined by `operators`, or the + * column refused and why, by name. See `branchTypes(requests, operators)` for the contract. + * + * The statement is typed as a TREE, as DuckDB types it (MEASURED on 18,000 chains of two to + * five branches over the six operators): SQL precedence (`INTERSECT [ALL]` binds first, then + * left to right), a run of ONE union kind typed as one node -- so `n UNION d UNION s` is + * `VARCHAR` -- and every other node typed over its two sides -- so `n EXCEPT d UNION ALL s` is + * refused, `n` and `d` being one node. EACH node is typed by the rules of [[typeNode]], and a + * subtree is an operand of its parent with the subtree's type. Each column comes with the type + * each branch projects there and, for a `STRUCT` or a `GEO_POINT` column, its fields and each + * branch's ([[SetOperationColumn]]). + */ + private[elastic] def columnTypes( + requests: Seq[SingleSearch], + operators: Seq[SetOperator] + ): Either[String, Seq[SetOperationColumn]] = { + val joins = + if (operators.isEmpty) Seq.fill(math.max(requests.size - 1, 0))(UNION) else operators + val known = requests.map(declared).zipWithIndex.collect { case (Some(f), i) => (i, f) } + if (known.isEmpty) Right(Nil) + else if (joins.size != requests.size - 1) + Left( + s"A set operation over ${requests.size} branches needs ${requests.size - 1} operators, " + + s"got ${joins.size}" + ) + else { + val tree = typingTree(precedenceTree(requests.indices.toList, joins.toList)) + val byBranch = known.toMap + val width = known.map(_._2.size).min + (0 until width).foldLeft[Either[String, Vector[SetOperationColumn]]](Right(Vector.empty)) { + case (Right(columns), pos) => + val leaves = requests.indices.map(branch => + byBranch.get(branch).map(f => operandOf(branch, f(pos))) + ) + typeOf(tree, leaves) match { + case Right(result) => + Right( + columns :+ SetOperationColumn( + result.fold(SetOperationType(SQLTypes.Any))(r => resultOf(r.valueType)), + leaves.map(_.map(_.valueType)) + ) + ) + case Left(refused) => Left(refusal(pos, refused)) + } + case (refused, _) => refused + } + } + } + + /** Whether the branches of a set operation, joined by `operators`, may run, column by column, + * once each branch's schema is attached -- `Right(())` -- or the refusal, by name. + * + * THE CONTRACT, shared by core (`SearchApi.resolveWithSchema(MultiSearch)`, for `UNION ALL`) + * and softclient4es-arrow (`app.softnetwork.elastic.arrow`, for the operators it runs on + * DuckDB), which is why it is `private[elastic]`: + * + * - INPUT: the branches, in text order, each with its schema ATTACHED + * (`SingleSearch.update(Some(schema))`), and the operators between them, in text order + * (`MultiSearch.resolvedOperators`; `Nil` is `UNION ALL` everywhere). A bare column of a + * branch without a schema is `ANY` and passes -- nothing is refused before the column + * types are known (the lead's ruling of 2026-10-05) -- and a `SELECT *` branch declares + * nothing and is skipped. + * - TYPING: the operator tree (`columnTypes(requests, operators)`): `INTERSECT [ALL]` binds + * first, a run of one union kind is typed as one node, every other operator types its two + * sides; each node, at every depth, by the lead's rules of 2026-10-07 ([[typeNode]]). + * - REFUSED, and only these, each because the relational engine fails on it when the + * statement runs: an `ARRAY`, a `STRUCT` or a `GEO_POINT` beside a type of another family + * (`ARRAY`s, structs or points as DuckDB 1.5.5 types them, every element and every field + * over all the operands of the node at once); a `DATE` or a `TIMESTAMP` beside a `BOOLEAN` + * or a number, in one node with neither a text nor a `VARBINARY` operand. `Left(message)`: + * `Set operation branches must project compatible types at column : branch '' + * is , branch '' is ` -- the two operands (a subtree is named by the + * branch that decides its type), `` being what the item is declared as (`DATE` for a + * `DATE` column, `INT` for a whole number literal); for two structs that collide on a + * field, `branch '' field '' is , branch '' field '' + * is `, `` the field (`a`, or `p.c` inside a field `p`) and `` its type + * as DuckDB reads it (`INT`, `DATE`, `VARCHAR` for a `KEYWORD`); or `A set operation over + * branches needs operators, got `. + * - `Right(())`: the statement may run. Every other node answers: the type DuckDB 1.5.5 + * gives it -- a `STRUCT` beside a `STRUCT` or a `GEO_POINT` its merged struct, a + * `GEO_POINT` beside a `GEO_POINT` a `GEO_POINT` -- or `VARCHAR` where DuckDB gives none + * (a `VARBINARY` beside a number, a temporal or a `BOOLEAN`; a `TIME` beside a date or a + * number), every value spelled as text. + * + * The branch WIDTH is `branchArity`'s rule, asked by `validate()`; column NAMES are not + * checked: SQL takes the first branch's. + */ + private[elastic] def branchTypes( + requests: Seq[SingleSearch], + operators: Seq[SetOperator] + ): Either[String, Unit] = + columnTypes(requests, operators).map(_ => ()) + + /** `branchTypes(requests, operators)` with `UNION ALL` between every branch: one run, typed as + * `columnTypes(requests)` types it. + */ + private[elastic] def branchTypes(requests: Seq[SingleSearch]): Either[String, Unit] = + branchTypes(requests, Nil) } /** FROM-less SELECT of constant scalar expressions — the connection/health idiom of the diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala index 6c73e51e..90a4e036 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala @@ -237,17 +237,25 @@ class SetOperatorSpec extends AnyFlatSpec with Matchers { "Branch types" should "be validated where both sides are typeable" in { // A TYPE rule, asked once each branch's schema is attached (`SearchApi.resolveWithSchema`), - // never when the statement is parsed (the lead's ruling of 2026-10-05). + // never when the statement is parsed (the lead's ruling of 2026-10-05). It answers what + // DuckDB 1.5.5 answers (the lead's rule of 2026-10-06; `SetOperationTypesSpec` has the table). def branchTypes(sql: String): Either[String, Unit] = MultiSearch.branchTypes(parse(sql).requests) + // a temporal beside a number: DuckDB refuses the pair Seq( - "SELECT 1 AS n FROM t UNION SELECT 'a' AS n FROM u", - "SELECT CAST(x AS BIGINT) AS v FROM t UNION ALL SELECT CAST(y AS VARCHAR) AS v FROM u" + "SELECT CURRENT_DATE AS n FROM t UNION SELECT 1 AS n FROM u", + "SELECT CAST(x AS DATE) AS v FROM t UNION ALL SELECT CAST(y AS BIGINT) AS v FROM u" ).foreach { sql => withClue(s"[$sql] ") { branchTypes(sql).swap.getOrElse("") should include("compatible types at column 1") } } + // a text branch makes the column VARCHAR -- MEASURED on DuckDB 1.5.5: `SELECT 1 UNION ALL + // SELECT 'a'` is VARCHAR '1', 'a' (core refused both pairs before) + branchTypes("SELECT 1 AS n FROM t UNION SELECT 'a' AS n FROM u") shouldBe Right(()) + branchTypes( + "SELECT CAST(x AS BIGINT) AS v FROM t UNION ALL SELECT CAST(y AS VARCHAR) AS v FROM u" + ) shouldBe Right(()) // numeric family branchTypes("SELECT 1 AS n FROM t UNION SELECT 2.5 AS n FROM u") shouldBe Right(()) // a bare column is `Any` until its schema is attached — the core seam checks with schemas diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala index ac1559c0..ff299e85 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala @@ -432,4 +432,34 @@ class DateArithmeticTypeSpec extends AnyFlatSpec with Matchers { "and INT for n" ) } + + /** Per group a date calculation is the number of milliseconds Elasticsearch computed, so a `DATE` + * or `TIMESTAMP` conversion over it is arithmetic on that number -- the start of its UTC day, or + * the number itself -- never a `java.time` call (`(MAX(d) + 1)::DATE` rendered + * `….toInstant()…toLocalDate()` on a double, and Elasticsearch failed the search). The view's + * pivot computes the same rendering (`SingleSearch.transformBucketScripts`). + */ + "a DATE or TIMESTAMP conversion over a per-group date calculation" should + "be computed on the milliseconds the group answers" in { + def rendered(expression: String): String = { + val (resolved, item) = resolvedItem(s"SELECT g, $expression AS x FROM ev GROUP BY g") + resolved.validateResolved() shouldBe Right(()) + val script = item.identifier.painless(None) + // the view's pivot computes the same rendering + resolved.transformBucketScripts.get("x").map(_.script) shouldBe Some(script) + script + } + Seq("(MAX(d) + 1)", "(MAX(ts) - 0.5)", "GREATEST(MAX(d), MAX(d2))").foreach { calc => + val timestamp = rendered(s"$calc::TIMESTAMP") + val date = rendered(s"$calc::DATE") + withClue(s"[$calc] TIMESTAMP $timestamp DATE $date ") { + Seq(timestamp, date).foreach { script => + script should not include "toInstant" + script should not include "toLocalDate" + } + // the TIMESTAMP is the milliseconds themselves; the DATE, their UTC day's start + date shouldBe s"(Math.floor($timestamp / 86400000.0) * 86400000.0)" + } + } + } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/SetOperationTypeRuleSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/SetOperationTypeRuleSpec.scala new file mode 100644 index 00000000..b653391a --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/SetOperationTypeRuleSpec.scala @@ -0,0 +1,374 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.MultiSearch.SetOperationType +import app.softnetwork.elastic.sql.schema.{Column, Table} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** ONE rule types a set operation at every depth -- a column, a field of a struct, the element of a + * list, a field of a struct inside a struct -- and this spec checks it against ONE table, written + * here, independently of the code: for every set of two or three types, the rule's answer, and + * beside it DuckDB 1.5.5's (MEASURED: the same at every depth, whatever the order). Every pair and + * every triple, in every order, under `UNION ALL`, `UNION`, `INTERSECT` and `EXCEPT`, at every + * depth: 13,120 statements. + */ +class SetOperationTypeRuleSpec extends AnyFlatSpec with Matchers { + + private val types: Seq[String] = + Seq("INT", "BIGINT", "DOUBLE", "BOOLEAN", "DATE", "TIMESTAMP", "TIME", "VARBINARY", "VARCHAR") + + private val declared: Map[String, SQLType] = Map( + "INT" -> SQLTypes.Int, + "BIGINT" -> SQLTypes.BigInt, + "DOUBLE" -> SQLTypes.Double, + "BOOLEAN" -> SQLTypes.Boolean, + "DATE" -> SQLTypes.Date, + "TIMESTAMP" -> SQLTypes.Timestamp, + "TIME" -> SQLTypes.Time, + "VARBINARY" -> SQLTypes.VarBinary, + "VARCHAR" -> SQLTypes.Keyword + ) + + /** Each type at each depth: `c_` a column, `f_` a struct's field `a`, `e_` a + * list's element, `s_` the field `c` of a struct field `p`; `NULL` a NULL literal. + */ + private val schema = Table( + "t", + columns = types.toList.flatMap { name => + val t = declared(name) + List( + Column(s"c_$name", t), + Column(s"f_$name", SQLTypes.Struct, multiFields = List(Column("a", t))), + Column(s"e_$name", SQLTypes.Array(t)), + Column( + s"s_$name", + SQLTypes.Struct, + multiFields = List(Column("p", SQLTypes.Struct, multiFields = List(Column("c", t)))) + ) + ) + } + ) + + private val depths: Seq[(String, String)] = + Seq("column" -> "c", "field" -> "f", "element" -> "e", "nested" -> "s") + + /** THE TABLE: a set of types, the rule's answer for one node holding them (REFUSED, or the type), + * and DuckDB 1.5.5's answer for the same node (FAILS, or the type), MEASURED at every depth. + */ + private val table: Map[Set[String], (String, String)] = + """ + |INT+INT INT INT + |INT+BIGINT BIGINT BIGINT + |INT+DOUBLE DOUBLE DOUBLE + |INT+BOOLEAN INT INT + |INT+DATE REFUSED FAILS + |INT+TIMESTAMP REFUSED FAILS + |INT+TIME VARCHAR FAILS + |INT+VARBINARY VARCHAR FAILS + |INT+VARCHAR VARCHAR VARCHAR + |INT+NULL INT INT + |BIGINT+BIGINT BIGINT BIGINT + |BIGINT+DOUBLE DOUBLE DOUBLE + |BIGINT+BOOLEAN BIGINT BIGINT + |BIGINT+DATE REFUSED FAILS + |BIGINT+TIMESTAMP REFUSED FAILS + |BIGINT+TIME VARCHAR FAILS + |BIGINT+VARBINARY VARCHAR FAILS + |BIGINT+VARCHAR VARCHAR VARCHAR + |BIGINT+NULL BIGINT BIGINT + |DOUBLE+DOUBLE DOUBLE DOUBLE + |DOUBLE+BOOLEAN DOUBLE DOUBLE + |DOUBLE+DATE REFUSED FAILS + |DOUBLE+TIMESTAMP REFUSED FAILS + |DOUBLE+TIME VARCHAR FAILS + |DOUBLE+VARBINARY VARCHAR FAILS + |DOUBLE+VARCHAR VARCHAR VARCHAR + |DOUBLE+NULL DOUBLE DOUBLE + |BOOLEAN+BOOLEAN BOOLEAN BOOLEAN + |BOOLEAN+DATE REFUSED FAILS + |BOOLEAN+TIMESTAMP REFUSED FAILS + |BOOLEAN+TIME VARCHAR FAILS + |BOOLEAN+VARBINARY VARCHAR FAILS + |BOOLEAN+VARCHAR VARCHAR VARCHAR + |BOOLEAN+NULL BOOLEAN BOOLEAN + |DATE+DATE DATE DATE + |DATE+TIMESTAMP TIMESTAMP TIMESTAMP + |DATE+TIME VARCHAR FAILS + |DATE+VARBINARY VARCHAR FAILS + |DATE+VARCHAR VARCHAR VARCHAR + |DATE+NULL DATE DATE + |TIMESTAMP+TIMESTAMP TIMESTAMP TIMESTAMP + |TIMESTAMP+TIME VARCHAR FAILS + |TIMESTAMP+VARBINARY VARCHAR FAILS + |TIMESTAMP+VARCHAR VARCHAR VARCHAR + |TIMESTAMP+NULL TIMESTAMP TIMESTAMP + |TIME+TIME TIME TIME + |TIME+VARBINARY VARCHAR FAILS + |TIME+VARCHAR VARCHAR VARCHAR + |TIME+NULL TIME TIME + |VARBINARY+VARBINARY VARBINARY VARBINARY + |VARBINARY+VARCHAR VARBINARY VARBINARY + |VARBINARY+NULL VARBINARY VARBINARY + |VARCHAR+VARCHAR VARCHAR VARCHAR + |VARCHAR+NULL VARCHAR VARCHAR + |NULL+NULL NULL INT + |INT+BIGINT+DOUBLE DOUBLE DOUBLE + |INT+BIGINT+BOOLEAN BIGINT BIGINT + |INT+BIGINT+DATE REFUSED FAILS + |INT+BIGINT+TIMESTAMP REFUSED FAILS + |INT+BIGINT+TIME VARCHAR FAILS + |INT+BIGINT+VARBINARY VARCHAR FAILS + |INT+BIGINT+VARCHAR VARCHAR VARCHAR + |INT+BIGINT+NULL BIGINT BIGINT + |INT+DOUBLE+BOOLEAN DOUBLE DOUBLE + |INT+DOUBLE+DATE REFUSED FAILS + |INT+DOUBLE+TIMESTAMP REFUSED FAILS + |INT+DOUBLE+TIME VARCHAR FAILS + |INT+DOUBLE+VARBINARY VARCHAR FAILS + |INT+DOUBLE+VARCHAR VARCHAR VARCHAR + |INT+DOUBLE+NULL DOUBLE DOUBLE + |INT+BOOLEAN+DATE REFUSED FAILS + |INT+BOOLEAN+TIMESTAMP REFUSED FAILS + |INT+BOOLEAN+TIME VARCHAR FAILS + |INT+BOOLEAN+VARBINARY VARCHAR FAILS + |INT+BOOLEAN+VARCHAR VARCHAR VARCHAR + |INT+BOOLEAN+NULL INT INT + |INT+DATE+TIMESTAMP REFUSED FAILS + |INT+DATE+TIME REFUSED FAILS + |INT+DATE+VARBINARY VARCHAR FAILS + |INT+DATE+VARCHAR VARCHAR VARCHAR + |INT+DATE+NULL REFUSED FAILS + |INT+TIMESTAMP+TIME REFUSED FAILS + |INT+TIMESTAMP+VARBINARY VARCHAR FAILS + |INT+TIMESTAMP+VARCHAR VARCHAR VARCHAR + |INT+TIMESTAMP+NULL REFUSED FAILS + |INT+TIME+VARBINARY VARCHAR FAILS + |INT+TIME+VARCHAR VARCHAR VARCHAR + |INT+TIME+NULL VARCHAR FAILS + |INT+VARBINARY+VARCHAR VARCHAR FAILS + |INT+VARBINARY+NULL VARCHAR FAILS + |INT+VARCHAR+NULL VARCHAR VARCHAR + |BIGINT+DOUBLE+BOOLEAN DOUBLE DOUBLE + |BIGINT+DOUBLE+DATE REFUSED FAILS + |BIGINT+DOUBLE+TIMESTAMP REFUSED FAILS + |BIGINT+DOUBLE+TIME VARCHAR FAILS + |BIGINT+DOUBLE+VARBINARY VARCHAR FAILS + |BIGINT+DOUBLE+VARCHAR VARCHAR VARCHAR + |BIGINT+DOUBLE+NULL DOUBLE DOUBLE + |BIGINT+BOOLEAN+DATE REFUSED FAILS + |BIGINT+BOOLEAN+TIMESTAMP REFUSED FAILS + |BIGINT+BOOLEAN+TIME VARCHAR FAILS + |BIGINT+BOOLEAN+VARBINARY VARCHAR FAILS + |BIGINT+BOOLEAN+VARCHAR VARCHAR VARCHAR + |BIGINT+BOOLEAN+NULL BIGINT BIGINT + |BIGINT+DATE+TIMESTAMP REFUSED FAILS + |BIGINT+DATE+TIME REFUSED FAILS + |BIGINT+DATE+VARBINARY VARCHAR FAILS + |BIGINT+DATE+VARCHAR VARCHAR VARCHAR + |BIGINT+DATE+NULL REFUSED FAILS + |BIGINT+TIMESTAMP+TIME REFUSED FAILS + |BIGINT+TIMESTAMP+VARBINARY VARCHAR FAILS + |BIGINT+TIMESTAMP+VARCHAR VARCHAR VARCHAR + |BIGINT+TIMESTAMP+NULL REFUSED FAILS + |BIGINT+TIME+VARBINARY VARCHAR FAILS + |BIGINT+TIME+VARCHAR VARCHAR VARCHAR + |BIGINT+TIME+NULL VARCHAR FAILS + |BIGINT+VARBINARY+VARCHAR VARCHAR FAILS + |BIGINT+VARBINARY+NULL VARCHAR FAILS + |BIGINT+VARCHAR+NULL VARCHAR VARCHAR + |DOUBLE+BOOLEAN+DATE REFUSED FAILS + |DOUBLE+BOOLEAN+TIMESTAMP REFUSED FAILS + |DOUBLE+BOOLEAN+TIME VARCHAR FAILS + |DOUBLE+BOOLEAN+VARBINARY VARCHAR FAILS + |DOUBLE+BOOLEAN+VARCHAR VARCHAR VARCHAR + |DOUBLE+BOOLEAN+NULL DOUBLE DOUBLE + |DOUBLE+DATE+TIMESTAMP REFUSED FAILS + |DOUBLE+DATE+TIME REFUSED FAILS + |DOUBLE+DATE+VARBINARY VARCHAR FAILS + |DOUBLE+DATE+VARCHAR VARCHAR VARCHAR + |DOUBLE+DATE+NULL REFUSED FAILS + |DOUBLE+TIMESTAMP+TIME REFUSED FAILS + |DOUBLE+TIMESTAMP+VARBINARY VARCHAR FAILS + |DOUBLE+TIMESTAMP+VARCHAR VARCHAR VARCHAR + |DOUBLE+TIMESTAMP+NULL REFUSED FAILS + |DOUBLE+TIME+VARBINARY VARCHAR FAILS + |DOUBLE+TIME+VARCHAR VARCHAR VARCHAR + |DOUBLE+TIME+NULL VARCHAR FAILS + |DOUBLE+VARBINARY+VARCHAR VARCHAR FAILS + |DOUBLE+VARBINARY+NULL VARCHAR FAILS + |DOUBLE+VARCHAR+NULL VARCHAR VARCHAR + |BOOLEAN+DATE+TIMESTAMP REFUSED FAILS + |BOOLEAN+DATE+TIME REFUSED FAILS + |BOOLEAN+DATE+VARBINARY VARCHAR FAILS + |BOOLEAN+DATE+VARCHAR VARCHAR VARCHAR + |BOOLEAN+DATE+NULL REFUSED FAILS + |BOOLEAN+TIMESTAMP+TIME REFUSED FAILS + |BOOLEAN+TIMESTAMP+VARBINARY VARCHAR FAILS + |BOOLEAN+TIMESTAMP+VARCHAR VARCHAR VARCHAR + |BOOLEAN+TIMESTAMP+NULL REFUSED FAILS + |BOOLEAN+TIME+VARBINARY VARCHAR FAILS + |BOOLEAN+TIME+VARCHAR VARCHAR VARCHAR + |BOOLEAN+TIME+NULL VARCHAR FAILS + |BOOLEAN+VARBINARY+VARCHAR VARCHAR FAILS + |BOOLEAN+VARBINARY+NULL VARCHAR FAILS + |BOOLEAN+VARCHAR+NULL VARCHAR VARCHAR + |DATE+TIMESTAMP+TIME VARCHAR FAILS + |DATE+TIMESTAMP+VARBINARY VARCHAR FAILS + |DATE+TIMESTAMP+VARCHAR VARCHAR VARCHAR + |DATE+TIMESTAMP+NULL TIMESTAMP TIMESTAMP + |DATE+TIME+VARBINARY VARCHAR FAILS + |DATE+TIME+VARCHAR VARCHAR VARCHAR + |DATE+TIME+NULL VARCHAR FAILS + |DATE+VARBINARY+VARCHAR VARCHAR FAILS + |DATE+VARBINARY+NULL VARCHAR FAILS + |DATE+VARCHAR+NULL VARCHAR VARCHAR + |TIMESTAMP+TIME+VARBINARY VARCHAR FAILS + |TIMESTAMP+TIME+VARCHAR VARCHAR VARCHAR + |TIMESTAMP+TIME+NULL VARCHAR FAILS + |TIMESTAMP+VARBINARY+VARCHAR VARCHAR FAILS + |TIMESTAMP+VARBINARY+NULL VARCHAR FAILS + |TIMESTAMP+VARCHAR+NULL VARCHAR VARCHAR + |TIME+VARBINARY+VARCHAR VARCHAR FAILS + |TIME+VARBINARY+NULL VARCHAR FAILS + |TIME+VARCHAR+NULL VARCHAR VARCHAR + |VARBINARY+VARCHAR+NULL VARBINARY VARBINARY + |""".stripMargin.linesIterator + .map(_.trim) + .filter(_.nonEmpty) + .map { line => + val Array(set, rule, duckdb) = line.split("\\s+") + set.split("\\+").toSet -> (rule -> duckdb) + } + .toMap + + private def rule(set: Set[String]): String = + table.getOrElse(set, fail(s"no row for ${set.mkString("+")}"))._1 + + /** The rule's answer for a statement: one node for a run of one union kind; binary nodes, left to + * right, for `INTERSECT` and `EXCEPT`. + */ + private def expected(operator: String, items: Seq[String]): String = + if (operator.startsWith("UNION")) rule(items.toSet) + else + items.tail.foldLeft(items.head) { (acc, item) => + if (acc == "REFUSED") acc else rule(Set(acc, item)) + } + + private def name(t: SQLType): String = + t match { + case SQLTypes.Int => "INT" + case SQLTypes.BigInt => "BIGINT" + case SQLTypes.Double => "DOUBLE" + case SQLTypes.Boolean => "BOOLEAN" + case SQLTypes.Date => "DATE" + case SQLTypes.Timestamp => "TIMESTAMP" + case SQLTypes.Time => "TIME" + case SQLTypes.VarBinary => "VARBINARY" + case SQLTypes.Varchar => "VARCHAR" + case SQLTypes.Null => "NULL" + case other => other.typeId + } + + /** The type the code answers at `depth`, or REFUSED. */ + private def answered(sql: String, depth: String): Either[String, String] = + Parser(sql) match { + case Right(m: MultiSearch) => + MultiSearch.columnTypes(m.requests.map(_.update(Some(schema))), m.resolvedOperators) match { + case Left(refusal) => Left(refusal) + case Right(columns) => + val result: SetOperationType = columns.head.result + def inner(t: SetOperationType): Option[SQLType] = + depth match { + case "column" => Some(t.sqlType) + case "field" => t.fields.find(_.name == "a").map(_.sqlType) + case "element" => t.element.map(_.sqlType) + case _ => + t.fields.find(_.name == "p").flatMap(_.fields.find(_.name == "c")).map(_.sqlType) + } + Right( + if (result.sqlType == SQLTypes.Null) "NULL" + else inner(result).fold(s"? ${result}")(name) + ) + } + case other => fail(s"[$sql] expected a set operation, got $other") + } + + "The table" should "equal DuckDB wherever DuckDB answers, refuse or say VARCHAR elsewhere" in { + // rule 1: a date beside a number or a BOOLEAN, with neither text nor a VARBINARY + def ruleOne(set: Set[String]): Boolean = + !set("VARCHAR") && !set("VARBINARY") && (set("DATE") || set("TIMESTAMP")) && + Seq("INT", "BIGINT", "DOUBLE", "BOOLEAN").exists(set) + // where DuckDB gives no type and rule 1 does not refuse: VARCHAR (rule 2), listed here + val ruleTwo: Set[Set[String]] = table.collect { case (set, ("VARCHAR", "FAILS")) => + set + }.toSet + val wrong = table.toSeq.flatMap { case (set, (r, duckdb)) => + val key = set.toSeq.sorted.mkString("+") + if (duckdb == "FAILS") { + val want = if (ruleOne(set)) "REFUSED" else "VARCHAR" + if (r == want) None else Some(s"$key: the rule answers $r where DuckDB fails, not $want") + } else if (r == duckdb) None + // a NULL-only node: DuckDB types its NULL literals INTEGER, core NULL + else if (set == Set("NULL") && duckdb == "INT") None + else Some(s"$key: the rule answers $r, DuckDB $duckdb") + } + wrong shouldBe empty + table should have size 175 + ruleTwo.size shouldBe 61 + ruleTwo.forall(set => !ruleOne(set)) shouldBe true + } + + "Every set operation" should "be typed by the table at every depth, order and operator" in { + val sets = (types :+ "NULL").map(t => Seq(t, t)) ++ + (types :+ "NULL").combinations(2).flatMap(_.permutations) ++ + (types :+ "NULL").combinations(3).flatMap(_.permutations) + val statements = for { + (depth, prefix) <- depths + items <- sets + operator <- Seq("UNION ALL", "UNION", "INTERSECT", "EXCEPT") + } yield ( + depth, + operator, + items, + items + .map(i => s"SELECT ${if (i == "NULL") "NULL" else s"${prefix}_$i"} AS k FROM t") + .mkString(s" $operator ") + ) + statements.size shouldBe 13120 + val wrong = statements.flatMap { case (depth, operator, items, sql) => + val want = expected(operator, items) + (answered(sql, depth), want) match { + case (Left(refusal), "REFUSED") => + // a field is named by its path; a list's element by its lists + val named = depth match { + case "field" => refusal.contains("field 'a' is") + case "nested" => refusal.contains("field 'p.c' is") + case "element" => refusal.contains("is ARRAY<") + case _ => !refusal.contains("field '") + } + if (named) None else Some(s"[$depth] [$sql] refused as $refusal") + case (Right(got), w) if got == w => None + case (got, w) => Some(s"[$depth] [$sql] answered $got instead of $w") + } + } + wrong shouldBe empty + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/SetOperationTypesSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/SetOperationTypesSpec.scala new file mode 100644 index 00000000..973ca237 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/SetOperationTypesSpec.scala @@ -0,0 +1,797 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** A set operation's column types follow the lead's rules of 2026-10-07, once each branch's schema + * is attached: the type DuckDB 1.5.5 gives the column where it gives one -- the relational engine + * that runs `UNION`, `INTERSECT` and `EXCEPT` is DuckDB -- `VARCHAR` where it gives none, and a + * refusal ONLY for a date beside a number, the one pair the relational engine fails on when the + * statement runs. + * + * Every DuckDB type below was MEASURED on DuckDB 1.5.5 over the same columns (`id`, `s` and `txt` + * text, `n` INTEGER, `x` DOUBLE, `f` FLOAT, `bg` BIGINT, `sm` SMALLINT, `d` / `d2` DATE, `ts` + * TIMESTAMP, `b` BOOLEAN), statement by statement; every verdict on a literal-free statement was + * MEASURED through arrow (its runtime matrix: 1,404 statements). PostgreSQL 16 refuses most of the + * text pairs (`UNION types text and integer cannot be matched`); DuckDB comes first. + */ +class SetOperationTypesSpec extends AnyFlatSpec with Matchers { + + private val schema = Table( + "t", + columns = List( + Column("id", SQLTypes.Keyword), + Column("s", SQLTypes.Keyword), + Column("txt", SQLTypes.Text), + Column("n", SQLTypes.Int), + Column("x", SQLTypes.Double), + Column("f", SQLTypes.Real), + Column("bg", SQLTypes.BigInt), + Column("sm", SQLTypes.SmallInt), + Column("d", SQLTypes.Date), + Column("d2", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("b", SQLTypes.Boolean), + Column("geo", SQLTypes.GeoPoint), + Column("bin", SQLTypes.VarBinary), + Column("tags", SQLTypes.Array(SQLTypes.Keyword)), + Column("arr", SQLTypes.Array(SQLTypes.Int)), + Column("arrb", SQLTypes.Array(SQLTypes.BigInt)), + Column("arrd", SQLTypes.Array(SQLTypes.Date)), + Column( + "obj", + SQLTypes.Struct, + multiFields = List(Column("a", SQLTypes.Int), Column("b", SQLTypes.Keyword)) + ), + Column( + "o2", + SQLTypes.Struct, + multiFields = List(Column("b", SQLTypes.Keyword), Column("c", SQLTypes.Boolean)) + ), + Column("oa", SQLTypes.Struct, multiFields = List(Column("A", SQLTypes.Keyword))), + Column("oad", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Date))), + Column( + "olat", + SQLTypes.Struct, + multiFields = List(Column("lat", SQLTypes.Keyword), Column("z", SQLTypes.Int)) + ), + Column( + "onest", + SQLTypes.Struct, + multiFields = List(Column("p", SQLTypes.GeoPoint), Column("q", SQLTypes.Int)) + ), + Column("geo2", SQLTypes.GeoPoint), + // one field `a` of several types, and lists of three of them + Column("oi", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Int))), + Column("od", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Date))), + Column("ov", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Keyword))), + Column("ob", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.Boolean))), + Column("obl", SQLTypes.Struct, multiFields = List(Column("a", SQLTypes.VarBinary))), + Column("ai", SQLTypes.Array(SQLTypes.Int)), + Column("ad", SQLTypes.Array(SQLTypes.Date)), + Column("av", SQLTypes.Array(SQLTypes.Keyword)), + // a field `p`: an object whose field `c` has one of three types, an INT, or a KEYWORD + Column("si", SQLTypes.Struct, multiFields = List(inner("p", SQLTypes.Int))), + Column("sd", SQLTypes.Struct, multiFields = List(inner("p", SQLTypes.Date))), + Column("sv", SQLTypes.Struct, multiFields = List(inner("p", SQLTypes.Keyword))), + Column("pn", SQLTypes.Struct, multiFields = List(Column("p", SQLTypes.Int))), + Column("pv", SQLTypes.Struct, multiFields = List(Column("p", SQLTypes.Keyword))) + ) + ) + + /** A field `name` that is an object with one field `c` of type `c`. */ + private def inner(name: String, c: SQLType): Column = + Column(name, SQLTypes.Struct, multiFields = List(Column("c", c))) + + private def branches(sql: String, withSchema: Boolean = true): Seq[SingleSearch] = + Parser(sql) match { + case Right(m: MultiSearch) => + if (withSchema) m.requests.map(_.update(Some(schema))) else m.requests + case other => fail(s"[$sql] expected a set operation, got $other") + } + + private def columns(sql: String): Either[String, Seq[MultiSearch.SetOperationColumn]] = + MultiSearch.columnTypes(branches(sql)) + + private def typeOf(sql: String): Either[String, SQLType] = + columns(sql).map(_.head.resultType) + + /** `SELECT AS k FROM t UNION ALL SELECT AS k FROM t ...` */ + private def union(items: String*): String = + items.map(i => s"SELECT $i AS k FROM t").mkString(" UNION ALL ") + + "A set operation's column" should "take the type DuckDB 1.5.5 gives it, every pair" in { + val expected: Seq[(String, SQLType)] = Seq( + // the brief's pairs + union("d", "CURRENT_DATE") -> SQLTypes.Date, + union("n", "'1'") -> SQLTypes.Varchar, + union("d", "'2024-01-31'") -> SQLTypes.Varchar, + union("id", "n") -> SQLTypes.Varchar, + union("1", "'a'") -> SQLTypes.Varchar, + union("d", "ts") -> SQLTypes.Timestamp, + union("ts", "CAST(d AS DATE)") -> SQLTypes.Timestamp, + union("d", "d2") -> SQLTypes.Date, + union("n", "x") -> SQLTypes.Double, + union("n", "CAST(n AS BIGINT)") -> SQLTypes.BigInt, + union("n", "2") -> SQLTypes.Int, + union("s", "id") -> SQLTypes.Varchar, + // the same pairs the other way round + union("CURRENT_DATE", "d") -> SQLTypes.Date, + union("'1'", "n") -> SQLTypes.Varchar, + union("'2024-01-31'", "d") -> SQLTypes.Varchar, + union("n", "id") -> SQLTypes.Varchar, + union("'a'", "1") -> SQLTypes.Varchar, + union("ts", "d") -> SQLTypes.Timestamp, + union("x", "n") -> SQLTypes.Double, + union("CAST(n AS BIGINT)", "n") -> SQLTypes.BigInt, + union("2", "n") -> SQLTypes.Int, + // a NULL literal takes the other branch's type + union("NULL", "n") -> SQLTypes.Int, + union("n", "NULL") -> SQLTypes.Int, + union("NULL", "d") -> SQLTypes.Date, + union("NULL", "'a'") -> SQLTypes.Varchar, + union("x", "NULL") -> SQLTypes.Double, + union("b", "NULL") -> SQLTypes.Boolean, + // three branches: the widest, and a text branch makes the column text whatever the others + union("n", "x", "CAST(n AS BIGINT)") -> SQLTypes.Double, + union("d", "ts", "d2") -> SQLTypes.Timestamp, + union("d", "n", "'1'") -> SQLTypes.Varchar, + union("n", "NULL", "x") -> SQLTypes.Double, + union("1", "2", "'a'") -> SQLTypes.Varchar, + union("s", "n", "d") -> SQLTypes.Varchar, + union("CAST(n AS BIGINT)", "n", "n") -> SQLTypes.BigInt, + union("d", "d2", "CURRENT_DATE") -> SQLTypes.Date, + // a KEYWORD and a TEXT column are both text + union("id", "txt") -> SQLTypes.Varchar, + union("txt", "n") -> SQLTypes.Varchar, + // a TIMESTAMP, a DOUBLE, a REAL or a TIME beside text is text too, spelled as DuckDB spells + // it (the lead's ruling of 2026-10-06) + union("ts", "s") -> SQLTypes.Varchar, + union("s", "ts") -> SQLTypes.Varchar, + union("ts", "'x'") -> SQLTypes.Varchar, + union("x", "s") -> SQLTypes.Varchar, + union("'a'", "x") -> SQLTypes.Varchar, + union("f", "txt") -> SQLTypes.Varchar, + union("2.5", "'a'") -> SQLTypes.Varchar, + union("CAST(ts AS TIME)", "s") -> SQLTypes.Varchar, + union("d", "ts", "s") -> SQLTypes.Varchar, + union("x", "ts", "'1'") -> SQLTypes.Varchar, + // a TIME beside a TIMESTAMP, with text: VARCHAR (DuckDB) + union("CAST(ts AS TIME)", "ts", "'x'") -> SQLTypes.Varchar, + // a VARBINARY beside text is a VARBINARY, DuckDB's BLOB (the lead's ruling of 2026-10-07) + union("bin", "s") -> SQLTypes.VarBinary, + union("s", "bin") -> SQLTypes.VarBinary, + union("bin", "'x'") -> SQLTypes.VarBinary, + union("bin", "txt", "s") -> SQLTypes.VarBinary, + union("bin", "NULL", "s") -> SQLTypes.VarBinary, + union("bin", "NULL") -> SQLTypes.VarBinary, + union("bin", "bin") -> SQLTypes.VarBinary, + // a BOOLEAN reads as 1 / 0 beside a number + union("b", "n") -> SQLTypes.Int, + union("n", "b") -> SQLTypes.Int, + union("b", "x") -> SQLTypes.Double, + union("b", "1") -> SQLTypes.Int, + union("TRUE", "n") -> SQLTypes.Int, + // the numeric lattice + union("n", "f") -> SQLTypes.Real, + union("bg", "f") -> SQLTypes.Real, + union("sm", "n") -> SQLTypes.Int, + union("bg", "x") -> SQLTypes.Double, + union("f", "x") -> SQLTypes.Double, + union("sm", "bg") -> SQLTypes.BigInt, + union("n", "bg") -> SQLTypes.BigInt, + // the other set operators: the same rule (the relational engine runs them) + "SELECT n AS k FROM t INTERSECT SELECT x AS k FROM t" -> SQLTypes.Double, + "SELECT id AS k FROM t EXCEPT SELECT n AS k FROM t" -> SQLTypes.Varchar, + "SELECT n AS k FROM t UNION SELECT '1' AS k FROM t" -> SQLTypes.Varchar, + "SELECT d AS k FROM t INTERSECT SELECT ts AS k FROM t" -> SQLTypes.Timestamp, + "SELECT b AS k FROM t EXCEPT SELECT n AS k FROM t" -> SQLTypes.Int + ) + val wrong = expected.flatMap { case (sql, want) => + typeOf(sql) match { + case Right(got) if got == want => None + case other => Some(s"[$sql] answered $other instead of $want") + } + } + wrong shouldBe empty + } + + it should "be refused by name where a date meets a number, with no text and no VARBINARY" in { + // a DATE or a TIMESTAMP beside a BOOLEAN or a number of any width, in one node with neither a + // text nor a VARBINARY operand: the pair the relational engine fails on (MEASURED through + // arrow: DuckDB's `Unimplemented type for cast (INTEGER -> DATE)` when a value flows) + val refused: Seq[(String, String)] = Seq( + union("d", "n") -> "column 1: branch 1 'k' is DATE, branch 2 'k' is INT", + union("n", "d") -> "column 1: branch 1 'k' is INT, branch 2 'k' is DATE", + union("b", "d") -> "column 1: branch 1 'k' is BOOLEAN, branch 2 'k' is DATE", + union("ts", "n") -> "column 1: branch 1 'k' is TIMESTAMP, branch 2 'k' is INT", + union("d", "f") -> "column 1: branch 1 'k' is DATE, branch 2 'k' is REAL", + union("sm", "ts") -> "column 1: branch 1 'k' is SMALLINT, branch 2 'k' is TIMESTAMP", + union("x", "d") -> "column 1: branch 1 'k' is DOUBLE, branch 2 'k' is DATE", + union("ts", "bg") -> "column 1: branch 1 'k' is TIMESTAMP, branch 2 'k' is BIGINT", + // a whole number LITERAL is an INTEGER, as DuckDB reads it + union("CURRENT_DATE", "1") -> "column 1: branch 1 'k' is DATE, branch 2 'k' is INT", + // each side is named by the branch that decides its type: the widest number, the TIMESTAMP + union("b", "n", "d") -> "column 1: branch 2 'k' is INT, branch 3 'k' is DATE", + union("d", "ts", "n") -> "column 1: branch 2 'k' is TIMESTAMP, branch 3 'k' is INT", + union("NULL", "d", "n") -> "column 1: branch 2 'k' is DATE, branch 3 'k' is INT", + // a TIME, which DuckDB has no common type for, does not hide the pair + union("CAST(ts AS TIME)", "d", "n") -> "column 1: branch 2 'k' is DATE, branch 3 'k' is INT", + "SELECT id, d FROM t UNION ALL SELECT g, n FROM t" -> + "column 2: branch 1 'd' is DATE, branch 2 'n' is INT", + "SELECT d AS k FROM t EXCEPT SELECT n AS k FROM t" -> + "column 1: branch 1 'k' is DATE, branch 2 'k' is INT" + ) + val wrong = refused.flatMap { case (sql, reason) => + MultiSearch.branchTypes(branches(sql)) match { + case Left(msg) + if msg == s"Set operation branches must project compatible types at $reason" => + None + case other => Some(s"[$sql] answered $other instead of refusing at $reason") + } + } + wrong shouldBe empty + } + + it should "answer VARCHAR where DuckDB gives the column no type" in { + // every value spelled as text; MEASURED through arrow for the VARBINARY nodes (arrow reads a + // binary field as its base64 text, so a VARBINARY is text to it, the date beside a number + // included), the lead's rule for the TIME pairs, whose computed branches arrow cannot read + // yet (issue #418) + val answered: Seq[(String, SQLType)] = Seq( + // a VARBINARY beside a number, a temporal or a BOOLEAN: DuckDB has no cast to BLOB + union("bin", "n") -> SQLTypes.Varchar, + union("x", "bin") -> SQLTypes.Varchar, + union("bin", "d") -> SQLTypes.Varchar, + union("ts", "bin") -> SQLTypes.Varchar, + union("b", "bin") -> SQLTypes.Varchar, + union("bin", "CAST(ts AS TIME)") -> SQLTypes.Varchar, + // ... with text in the column too: not a BLOB, text + union("bin", "s", "d") -> SQLTypes.Varchar, + union("b", "s", "bin") -> SQLTypes.Varchar, + // ... and beside a date AND a number, in any order (arrow's T, P2 and Q shapes) + union("bin", "d", "n") -> SQLTypes.Varchar, + union("d", "bin", "n") -> SQLTypes.Varchar, + union("n", "d", "bin") -> SQLTypes.Varchar, + union("bin", "ts", "x") -> SQLTypes.Varchar, + union("x", "ts", "bin") -> SQLTypes.Varchar, + union("bin", "d", "b") -> SQLTypes.Varchar, + union("b", "d", "bin") -> SQLTypes.Varchar, + union("s", "n", "d", "bin") -> SQLTypes.Varchar, + union("d", "bin", "n", "s") -> SQLTypes.Varchar, + // a TIME beside a date, a number or a BOOLEAN + union("CAST(ts AS TIME)", "ts") -> SQLTypes.Varchar, + union("d", "CAST(ts AS TIME)") -> SQLTypes.Varchar, + union("CAST(ts AS TIME)", "n") -> SQLTypes.Varchar, + union("x", "CAST(ts AS TIME)") -> SQLTypes.Varchar, + union("CAST(ts AS TIME)", "b") -> SQLTypes.Varchar, + // a text branch makes the column text even beside a date and a number (DuckDB) + union("d", "n", "s") -> SQLTypes.Varchar, + union("b", "ts", "txt") -> SQLTypes.Varchar + ) + val wrong = answered.flatMap { case (sql, want) => + typeOf(sql) match { + case Right(got) if got == want => None + case other => Some(s"[$sql] answered $other instead of $want") + } + } + wrong shouldBe empty + } + + it should "refuse an ARRAY, a STRUCT or a GEO_POINT beside another type, text included" in { + // MEASURED through arrow and on DuckDB 1.5.5: no date, number, BLOB or ordinary text is cast + // to a list or a struct (`Unimplemented type for cast (INTEGER -> INTEGER[])`, `Type VARCHAR + // with value 'a' can't be cast to the destination type STRUCT(lat DOUBLE, lon DOUBLE)`) + val refused: Seq[(String, String)] = Seq( + union("geo", "s") -> "column 1: branch 1 'k' is GEO_POINT, branch 2 'k' is KEYWORD", + union("geo", "n") -> "column 1: branch 1 'k' is GEO_POINT, branch 2 'k' is INT", + union("d", "geo") -> "column 1: branch 1 'k' is DATE, branch 2 'k' is GEO_POINT", + union("geo", "bin") -> "column 1: branch 1 'k' is GEO_POINT, branch 2 'k' is VARBINARY", + union("tags", "n") -> "column 1: branch 1 'k' is ARRAY, branch 2 'k' is INT", + union("s", "arr") -> "column 1: branch 1 'k' is KEYWORD, branch 2 'k' is ARRAY", + union("obj", "txt") -> "column 1: branch 1 'k' is STRUCT, branch 2 'k' is TEXT", + union("tags", "geo") -> "column 1: branch 1 'k' is ARRAY, branch 2 'k' is GEO_POINT", + // two structs whose fields DuckDB cannot merge: an INTEGER and a DATE (MEASURED), named + union("obj", "oad") -> + "column 1: branch 1 'k' field 'a' is INT, branch 2 'k' field 'a' is DATE", + union("geo", "tags") -> "column 1: branch 1 'k' is GEO_POINT, branch 2 'k' is ARRAY", + // the nested type is refused before text can make the node text + union("geo", "s", "n") -> "column 1: branch 1 'k' is GEO_POINT, branch 2 'k' is KEYWORD", + // two lists whose elements DuckDB cannot combine (`Unimplemented type for cast (INTEGER -> + // DATE)`, MEASURED) + union("arr", "arrd") -> "column 1: branch 1 'k' is ARRAY, branch 2 'k' is ARRAY" + ) + val wrong = refused.flatMap { case (sql, reason) => + MultiSearch.branchTypes(branches(sql)) match { + case Left(msg) + if msg == s"Set operation branches must project compatible types at $reason" => + None + case other => Some(s"[$sql] answered $other instead of refusing at $reason") + } + } + wrong shouldBe empty + } + + it should "answer a STRUCT beside a STRUCT or a GEO_POINT as DuckDB's merged STRUCT" in { + // MEASURED on DuckDB 1.5.5, and through arrow for a point stored as an object: every field of + // every branch, in the order it first appears; a field both have, matched whatever its case, + // takes DuckDB's type for the two; a GEO_POINT is STRUCT(lat DOUBLE, lon DOUBLE) + def fields(sql: String): Either[String, (SQLType, Seq[(String, String)])] = + columns(sql).map { cs => + def render(f: Seq[MultiSearch.StructField]): Seq[(String, String)] = + f.map { x => + val nested = render(x.fields).map(p => s"${p._1} ${p._2}") + x.name -> (if (x.fields.nonEmpty) nested.mkString("(", ", ", ")") else x.sqlType.typeId) + } + cs.head.resultType -> render(cs.head.fields) + } + val point = Seq("lat" -> "DOUBLE", "lon" -> "DOUBLE") + val expected: Seq[(String, (SQLType, Seq[(String, String)]))] = Seq( + union("geo", "obj") -> (SQLTypes.Struct, point ++ Seq("a" -> "INT", "b" -> "VARCHAR")), + union("obj", "geo") -> (SQLTypes.Struct, Seq("a" -> "INT", "b" -> "VARCHAR") ++ point), + union("geo", "geo2") -> (SQLTypes.GeoPoint, point), + union("geo", "NULL") -> (SQLTypes.GeoPoint, point), + union("obj", "obj") -> (SQLTypes.Struct, Seq("a" -> "INT", "b" -> "VARCHAR")), + union("obj", "o2") -> (SQLTypes.Struct, Seq( + "a" -> "INT", + "b" -> "VARCHAR", + "c" -> "BOOLEAN" + )), + union("o2", "obj") -> (SQLTypes.Struct, Seq( + "b" -> "VARCHAR", + "c" -> "BOOLEAN", + "a" -> "INT" + )), + // a field both have: DuckDB's type for the two, the first name + union("obj", "oa") -> (SQLTypes.Struct, Seq("a" -> "VARCHAR", "b" -> "VARCHAR")), + union("geo", "olat") -> (SQLTypes.Struct, Seq( + "lat" -> "VARCHAR", + "lon" -> "DOUBLE", + "z" -> "INT" + )), + // a point inside an object, beside a point + union("onest", "geo") -> + (SQLTypes.Struct, Seq("p" -> "(lat DOUBLE, lon DOUBLE)", "q" -> "INT") ++ point), + union("geo", "obj", "o2") -> + (SQLTypes.Struct, point ++ Seq("a" -> "INT", "b" -> "VARCHAR", "c" -> "BOOLEAN")) + ) + val wrong = expected.flatMap { case (sql, want) => + fields(sql) match { + case Right(got) if got == want => None + case other => Some(s"[$sql] answered $other instead of $want") + } + } + wrong shouldBe empty + } + + it should "type a struct's field and a list's element over every branch of the node at once" in { + // MEASURED on DuckDB 1.5.5, every order, UNION ALL and UNION alike: a field `a` that is an + // INTEGER, a DATE and a VARCHAR is a VARCHAR ({'1'}, {'2024-01-31'}, {'x'}); so are the + // elements of INTEGER[], DATE[] and VARCHAR[], and the field `c` of a field `p` -- whatever + // the order, although an INTEGER beside a DATE alone fails. Core typed them two by two, from + // the left, and refused the orders that met the INTEGER and the DATE first. + def typed(sql: String): Either[String, (SQLType, Seq[String])] = + Parser(sql) match { + case Right(m: MultiSearch) => + def render(f: Seq[MultiSearch.StructField]): Seq[String] = + f.map { x => + if (x.fields.nonEmpty) s"${x.name} (${render(x.fields).mkString(", ")})" + else s"${x.name} ${x.sqlType.typeId}" + } + MultiSearch + .columnTypes(m.requests.map(_.update(Some(schema))), m.resolvedOperators) + .map(cs => cs.head.resultType -> render(cs.head.fields)) + case other => fail(s"[$sql] expected a set operation, got $other") + } + val cases: Seq[(Seq[String], (SQLType, Seq[String]))] = Seq( + Seq("oi", "od", "ov") -> (SQLTypes.Struct, Seq("a VARCHAR")), + Seq("ai", "ad", "av") -> (SQLTypes.Array(SQLTypes.Varchar), Nil), + Seq("si", "sd", "sv") -> (SQLTypes.Struct, Seq("p (c VARCHAR)")), + // a BOOLEAN joins an INTEGER, and text makes the field text + Seq("ob", "oi", "ov") -> (SQLTypes.Struct, Seq("a VARCHAR")), + // a VARBINARY beside text and another type is text, as in a column: the relational engine + // reads a binary as its base64 text (DuckDB alone, typing a BLOB, fails on it) + Seq("obl", "ov", "oi") -> (SQLTypes.Struct, Seq("a VARCHAR")) + ) + val statements = for { + (items, want) <- cases + order <- items.permutations.toSeq + operator <- Seq("UNION ALL", "UNION") + } yield order.map(i => s"SELECT $i AS k FROM t").mkString(s" $operator ") -> want + statements should have size 60 + val wrong = statements.flatMap { case (sql, want) => + typed(sql) match { + case Right(got) if got == want => None + case other => Some(s"[$sql] answered $other instead of $want") + } + } + wrong shouldBe empty + // a VARBINARY beside text only stays a VARBINARY (MEASURED: STRUCT(a BLOB)) + typed(union("obl", "ov")) shouldBe Right((SQLTypes.Struct, Seq("a VARBINARY"))) + } + + it should "name the field two structs collide on, and its type in each" in { + // MEASURED on DuckDB 1.5.5: an INTEGER beside a DATE, with no text, fails in every order + // (`Unimplemented type for cast (INTEGER -> DATE)`), inside a field as inside a list; so does + // an object beside any other type, text included + val refused: Seq[(String, String)] = Seq( + union("oi", "od") -> "branch 1 'k' field 'a' is INT, branch 2 'k' field 'a' is DATE", + union("od", "oi") -> "branch 1 'k' field 'a' is DATE, branch 2 'k' field 'a' is INT", + // the first two the field fails on: the BOOLEAN joins the INTEGER, not the DATE + union("oi", "ob", "od") -> "branch 1 'k' field 'a' is INT, branch 3 'k' field 'a' is DATE", + union("si", "sd") -> "branch 1 'k' field 'p.c' is INT, branch 2 'k' field 'p.c' is DATE", + union("si", "pn") -> "branch 1 'k' field 'p' is STRUCT, branch 2 'k' field 'p' is INT", + union("pv", "si", "sv") -> + "branch 1 'k' field 'p' is VARCHAR, branch 2 'k' field 'p' is STRUCT", + // a list keeps naming the two lists + union("ai", "ad") -> "branch 1 'k' is ARRAY, branch 2 'k' is ARRAY", + // a node of the tree: the subtree is named by the branch that decides its type + chain("oi UNION ALL oi EXCEPT od") -> + "branch 1 'k' field 'a' is INT, branch 3 'k' field 'a' is DATE" + ) + val wrong = refused.flatMap { case (sql, reason) => + Parser(sql) match { + case Right(m: MultiSearch) => + MultiSearch.branchTypes( + m.requests.map(_.update(Some(schema))), + m.resolvedOperators + ) match { + case Left(msg) + if msg == + s"Set operation branches must project compatible types at column 1: $reason" => + None + case other => Some(s"[$sql] answered $other instead of refusing with $reason") + } + case other => fail(s"[$sql] expected a set operation, got $other") + } + } + wrong shouldBe empty + } + + it should "answer an ARRAY, a STRUCT or a GEO_POINT beside its own type, NULL, or a list" in { + // two lists of different element types take DuckDB 1.5.5's answer (MEASURED): INTEGER[] with + // VARCHAR[] is VARCHAR[], with BIGINT[] BIGINT[]; VARCHAR[] with DATE[] is VARCHAR[] + val answered: Seq[(String, SQLType)] = Seq( + union("geo", "geo") -> SQLTypes.GeoPoint, + union("geo", "NULL") -> SQLTypes.GeoPoint, + union("obj", "obj") -> SQLTypes.Struct, + union("tags", "tags") -> SQLTypes.Array(SQLTypes.Varchar), + union("NULL", "tags") -> SQLTypes.Array(SQLTypes.Varchar), + union("arr", "tags") -> SQLTypes.Array(SQLTypes.Varchar), + union("tags", "arr") -> SQLTypes.Array(SQLTypes.Varchar), + union("arr", "arrb") -> SQLTypes.Array(SQLTypes.BigInt), + union("tags", "arrd") -> SQLTypes.Array(SQLTypes.Varchar) + ) + val wrong = answered.flatMap { case (sql, want) => + typeOf(sql) match { + case Right(got) if got == want => None + case other => Some(s"[$sql] answered $other instead of $want") + } + } + wrong shouldBe empty + } + + it should "refuse nothing on types before the column types are known" in { + // with no schema a bare column is ANY: Elasticsearch answers, as for a single statement + MultiSearch + .columnTypes(branches("SELECT d AS k FROM t UNION ALL SELECT n AS k FROM t", false)) + .map(_.map(_.resultType)) shouldBe Right(Seq(SQLTypes.Any)) + MultiSearch.branchTypes(branches("SELECT a FROM t UNION SELECT 'x' FROM u", false)) shouldBe + Right(()) + } + + it should "carry each date field's mapping format to its position, at every depth" in { + def format(f: String) = + scala.collection.immutable.ListMap[String, app.softnetwork.elastic.sql.Value[_]]( + "format" -> app.softnetwork.elastic.sql.StringValue(f) + ) + val formatted = Table( + "t", + columns = List( + Column("ts", SQLTypes.Timestamp, options = format("epoch_second")), + Column("d", SQLTypes.Date), + Column("n", SQLTypes.Int), + Column( + "obj", + SQLTypes.Struct, + multiFields = List( + Column("a", SQLTypes.Timestamp, options = format("yyyy/MM/dd")), + Column( + "in", + SQLTypes.Struct, + multiFields = List(Column("b", SQLTypes.Date, options = format("basic_date"))) + ) + ) + ), + Column( + "od", + SQLTypes.Struct, + multiFields = List( + Column("a", SQLTypes.Date), + Column("in", SQLTypes.Struct, multiFields = List(Column("b", SQLTypes.Timestamp))) + ) + ), + Column( + "items", + SQLTypes.Array(SQLTypes.Struct), + multiFields = List(Column("a", SQLTypes.Date, options = format("dd/MM/yyyy"))) + ), + Column( + "items2", + SQLTypes.Array(SQLTypes.Struct), + multiFields = List(Column("a", SQLTypes.Timestamp)) + ) + ) + ) + def typed(sql: String): MultiSearch.SetOperationColumn = + Parser(sql) match { + case Right(m: MultiSearch) => + MultiSearch.columnTypes(m.requests.map(_.update(Some(formatted)))) match { + case Right(columns) => columns.head + case Left(refusal) => fail(refusal) + } + case other => fail(s"[$sql] expected a set operation, got $other") + } + val default = Some(TemporalLiterals.DefaultDateFormat) + // the column: its own format, else Elasticsearch's default + typed(union("ts", "d")).branches.map(_.flatMap(_.format)) shouldBe + Seq(Some("epoch_second"), default) + // a computed value reads no field: no format + typed(union("ts", "CURRENT_DATE")).branches.map(_.flatMap(_.format)) shouldBe + Seq(Some("epoch_second"), None) + // an object's field, and a field of an object inside it + def field(t: Option[MultiSearch.SetOperationType], path: String*): Option[String] = + path + .foldLeft(t)((acc, name) => acc.flatMap(_.fields.find(_.name == name)).map(_.valueType)) + .flatMap(_.format) + val objects = typed(union("obj", "od")) + objects.branches.map(field(_, "a")) shouldBe Seq(Some("yyyy/MM/dd"), default) + objects.branches.map(field(_, "in", "b")) shouldBe Seq(Some("basic_date"), default) + // a list of objects' field + typed(union("items", "items2")).branches.map(b => field(b.flatMap(_.element), "a")) shouldBe + Seq(Some("dd/MM/yyyy"), default) + // a column's result holds no format: no field decides it + def formats(t: MultiSearch.SetOperationType): Seq[String] = + t.format.toSeq ++ t.fields.flatMap(f => formats(f.valueType)) ++ t.element.toSeq.flatMap( + formats + ) + Seq(union("ts", "d"), union("obj", "od"), union("items", "items2")).foreach { sql => + withClue(sql)(formats(typed(sql).result) shouldBe empty) + } + } + + it should "say which columns convert their values: only those whose branches disagree" in { + def converts(sql: String): Seq[Boolean] = columns(sql).map(_.map(_.converts)).getOrElse(Nil) + converts(union("n", "x")) shouldBe Seq(true) + converts(union("d", "ts")) shouldBe Seq(true) + converts(union("id", "n")) shouldBe Seq(true) + converts(union("ts", "s")) shouldBe Seq(true) + converts(union("x", "'a'")) shouldBe Seq(true) + converts(union("bin", "s")) shouldBe Seq(true) + // a VARCHAR column DuckDB gives no type to: every value spelled as text + converts(union("bin", "n")) shouldBe Seq(true) + converts(union("bin", "d", "n")) shouldBe Seq(true) + converts(union("CAST(ts AS TIME)", "d")) shouldBe Seq(true) + // two lists of different element types: every element converted (the lead's ruling of + // 2026-10-07: one converter at every depth); the same list, an object beside an object of the + // same fields: nothing to convert + converts(union("arr", "tags")) shouldBe Seq(true) + converts(union("tags", "tags")) shouldBe Seq(false) + converts(union("obj", "obj")) shouldBe Seq(false) + // a field whose branches disagree converts its struct, a field that agrees does not + converts(union("obj", "oa")) shouldBe Seq(true) + converts(union("onest", "onest")) shouldBe Seq(false) + // every point becomes {lat, lon}, every object the column's fields + converts(union("geo", "geo")) shouldBe Seq(true) + converts(union("geo", "obj")) shouldBe Seq(true) + converts(union("obj", "o2")) shouldBe Seq(true) + // a VARBINARY beside VARBINARY or a NULL literal only: nothing to convert + converts(union("bin", "bin")) shouldBe Seq(false) + converts(union("bin", "NULL")) shouldBe Seq(false) + // branches that agree pay nothing + converts(union("n", "2")) shouldBe Seq(false) + converts(union("id", "txt")) shouldBe Seq(false) + converts(union("d", "CURRENT_DATE")) shouldBe Seq(false) + // a NULL literal has nothing to convert + converts(union("n", "NULL")) shouldBe Seq(false) + // a SELECT * branch is not typed here, so nothing is converted beside it + converts("SELECT n AS k FROM t UNION ALL SELECT * FROM t") shouldBe Seq(false) + // position by position + converts("SELECT id, n FROM t UNION ALL SELECT g, x FROM t") shouldBe Seq(false, true) + } + + // ── the operator tree (a chain of set operators is typed as a TREE, as DuckDB 1.5.5 types it) ── + + /** `SELECT AS k FROM t SELECT AS k FROM t ...` from `a op b op c`. */ + private def chain(spec: String): String = { + val operators = "(UNION ALL|UNION DISTINCT|UNION|INTERSECT ALL|INTERSECT|EXCEPT ALL|EXCEPT)" + val parts = spec.split(s"\\s+$operators\\s+") + val ops = s"\\s+$operators\\s+".r.findAllMatchIn(spec).map(_.group(1)).toList + (s"SELECT ${parts.head} AS k FROM t" :: ops.zip(parts.tail).map { case (op, item) => + s"$op SELECT $item AS k FROM t" + }).mkString(" ") + } + + private def treeTypes(sql: String): Either[String, Seq[SQLType]] = + Parser(sql) match { + case Right(m: MultiSearch) => + MultiSearch + .columnTypes(m.requests.map(_.update(Some(schema))), m.resolvedOperators) + .map(_.map(_.resultType)) + case other => fail(s"[$sql] expected a set operation, got $other") + } + + "A chain of set operators" should "be typed over its operator tree, each node by the rules" in { + // the tree MEASURED on DuckDB 1.5.5 (over 18,000 random chains, and again parenthesised): a + // run of ONE union kind is one node, whatever its order or parentheses; INTERSECT binds first, + // INTERSECT and EXCEPT are binary, left to right; two union kinds are two nodes. Each node by + // the lead's rules of 2026-10-07. The literal-free chains are MEASURED through arrow (N.., U.., + // F.., X.., and the T / Q shapes of its second matrix) + val answered: Seq[((String, String), SQLType)] = Seq( + "X05" -> chain("n UNION d UNION 'a'") -> SQLTypes.Varchar, + "X09" -> chain("n UNION ALL d INTERSECT 'a'") -> SQLTypes.Varchar, + "X11" -> chain("n UNION DISTINCT d UNION DISTINCT 'a'") -> SQLTypes.Varchar, + "X12" -> chain("'a' EXCEPT n UNION ALL d") -> SQLTypes.Varchar, + "Y01" -> chain("'a' INTERSECT n INTERSECT d") -> SQLTypes.Varchar, + "Y02" -> chain("n INTERSECT 'a' INTERSECT d") -> SQLTypes.Varchar, + "Z01" -> chain("n INTERSECT x INTERSECT 'a'") -> SQLTypes.Varchar, + "Z02" -> chain("d INTERSECT ts UNION ALL 'a'") -> SQLTypes.Varchar, + "Z03" -> chain("n EXCEPT x UNION ALL 'a'") -> SQLTypes.Varchar, + "Z04" -> chain("n UNION ALL x INTERSECT bg") -> SQLTypes.Double, + "Z05" -> chain("d EXCEPT ts") -> SQLTypes.Timestamp, + "Z06" -> chain("n INTERSECT ALL x") -> SQLTypes.Double, + "Z07" -> chain("s EXCEPT ALL bin") -> SQLTypes.VarBinary, + "Z10" -> chain("n UNION ALL d UNION ALL 'a' EXCEPT x") -> SQLTypes.Varchar, + "Z12" -> chain("'a' UNION ALL n UNION d") -> SQLTypes.Varchar, + "Z13" -> chain("b INTERSECT n EXCEPT ALL x") -> SQLTypes.Double, + "Z14" -> chain("ts INTERSECT ALL d INTERSECT ALL 'x'") -> SQLTypes.Varchar, + "Z16" -> chain("n UNION ALL 'a' INTERSECT d") -> SQLTypes.Varchar, + "Z18" -> chain("n UNION ALL d UNION ALL 'a'") -> SQLTypes.Varchar, + "Z20" -> chain("s UNION ALL n EXCEPT ALL d") -> SQLTypes.Varchar, + // a node DuckDB gives no type to is VARCHAR (rule 2), the BLOB beside text and a number too + "X14" -> chain("n UNION ALL bin UNION ALL s") -> SQLTypes.Varchar, + "Z08" -> chain("bin UNION s UNION n") -> SQLTypes.Varchar, + "X14i" -> chain("n INTERSECT bin INTERSECT s") -> SQLTypes.Varchar, + // ... and a VARCHAR subtree is text to its parent + "W01" -> chain("bin INTERSECT n UNION ALL d") -> SQLTypes.Varchar, + // a text operand makes the node text before a date meets a number + "N07u" -> chain("n UNION d UNION s") -> SQLTypes.Varchar, + "N07a" -> chain("n UNION ALL d UNION ALL s") -> SQLTypes.Varchar, + "U55i" -> chain("s INTERSECT n INTERSECT d") -> SQLTypes.Varchar, + "U55e" -> chain("s EXCEPT n EXCEPT d") -> SQLTypes.Varchar, + "F12i" -> chain("b INTERSECT x INTERSECT s") -> SQLTypes.Varchar, + "N03" -> chain("n UNION ALL x INTERSECT s") -> SQLTypes.Varchar, + "N05" -> chain("n UNION x UNION ALL bg") -> SQLTypes.Double, + // a VARBINARY is text to the date beside the number (arrow's T and Q shapes): every union + // run holding it, every INTERSECT / EXCEPT whose first node holds it + "Tu" -> chain("d UNION n UNION bin") -> SQLTypes.Varchar, + "Ta" -> chain("bin UNION ALL d UNION ALL n") -> SQLTypes.Varchar, + "Ti" -> chain("bin INTERSECT d INTERSECT n") -> SQLTypes.Varchar, + "Te" -> chain("d EXCEPT bin EXCEPT n") -> SQLTypes.Varchar, + "Qi" -> chain("ts INTERSECT bin INTERSECT x INTERSECT s") -> SQLTypes.Varchar, + "Qe" -> chain("bin EXCEPT d EXCEPT n EXCEPT s") -> SQLTypes.Varchar, + // a nested type beside its own type, a subtree included + "G1" -> chain("geo INTERSECT geo UNION ALL geo") -> SQLTypes.GeoPoint, + "G2" -> chain("arr UNION ALL tags EXCEPT arrb") -> SQLTypes.Array(SQLTypes.Varchar), + // a point beside an object, in any node: DuckDB's merged struct + "G6" -> chain("obj UNION ALL geo INTERSECT obj") -> SQLTypes.Struct + ) + val refused: Seq[(String, String)] = Seq( + "X01" -> chain("n EXCEPT d UNION ALL 'a'"), + "X02" -> chain("d INTERSECT n UNION ALL 'a'"), + "X03" -> chain("n UNION d UNION ALL 'a'"), + "X04" -> chain("n UNION ALL d UNION 'a'"), + "X06" -> chain("n EXCEPT d EXCEPT 'a'"), + "X07" -> chain("n INTERSECT d INTERSECT 'a'"), + "X08" -> chain("'a' UNION ALL n INTERSECT d"), + "X10" -> chain("n EXCEPT x UNION ALL d"), + "X13" -> chain("n UNION ALL d EXCEPT 'a'"), + "X15" -> chain("s UNION ALL b INTERSECT d"), + "Y03" -> chain("n INTERSECT d"), + "Y04" -> chain("n EXCEPT d"), + "Y05" -> chain("n INTERSECT d INTERSECT 'a'"), + "Z15" -> chain("x UNION ALL n UNION ALL d"), + "Z17" -> chain("d UNION d2 EXCEPT n"), + "Z19" -> chain("n INTERSECT ALL d2 UNION ALL 'a'"), + // MEASURED through arrow: the date meets the number in a node with no text + "N07i" -> chain("n INTERSECT d INTERSECT s"), + "N07e" -> chain("n EXCEPT d EXCEPT s"), + // MEASURED through arrow (T and Q shapes): the first node meets the date and the number + // before the VARBINARY can make it text + "Ti2" -> chain("d INTERSECT n INTERSECT bin"), + "Te2" -> chain("x EXCEPT ts EXCEPT bin"), + "Qi2" -> chain("b INTERSECT d INTERSECT s INTERSECT bin"), + // a TIME does not make the node text + "W02" -> chain("CAST(ts AS TIME) UNION ALL d UNION ALL n"), + // a nested type beside another type, in any node + "G3" -> chain("geo INTERSECT geo UNION ALL s"), + "G4" -> chain("s UNION bin UNION geo"), + "G5" -> chain("arr EXCEPT arrd") + ) + val wrong = + answered.flatMap { case ((id, sql), want) => + treeTypes(sql) match { + case Right(Seq(got)) if got == want => None + case other => Some(s"$id [$sql] answered $other instead of $want") + } + } ++ refused.flatMap { case (id, sql) => + treeTypes(sql) match { + case Left(_) => None + case other => Some(s"$id [$sql] answered $other instead of refusing") + } + } + wrong shouldBe empty + } + + it should "name, in a refusal, the branch that decides each side's type" in { + def refusal(spec: String): Either[String, Unit] = + Parser(chain(spec)) match { + case Right(m: MultiSearch) => + MultiSearch.branchTypes(m.requests.map(_.update(Some(schema))), m.resolvedOperators) + case other => fail(s"[$spec] expected a set operation, got $other") + } + val prefix = "Set operation branches must project compatible types at column 1: " + // the INTERSECT / EXCEPT side is one node: `n` meets `d` there + refusal("n EXCEPT d UNION ALL 'a'") shouldBe + Left(prefix + "branch 1 'k' is INT, branch 2 'k' is DATE") + // a subtree is named by the branch that decides its type: `n EXCEPT x` is DOUBLE, x's + refusal("n EXCEPT x UNION ALL d") shouldBe + Left(prefix + "branch 2 'k' is DOUBLE, branch 3 'k' is DATE") + refusal("s UNION ALL b INTERSECT d") shouldBe + Left(prefix + "branch 2 'k' is BOOLEAN, branch 3 'k' is DATE") + refusal("n INTERSECT d INTERSECT s") shouldBe + Left(prefix + "branch 1 'k' is INT, branch 2 'k' is DATE") + refusal("d INTERSECT n INTERSECT bin") shouldBe + Left(prefix + "branch 1 'k' is DATE, branch 2 'k' is INT") + // a nested subtree is named by its first branch, beside the first operand of another type + refusal("geo INTERSECT geo UNION ALL s") shouldBe + Left(prefix + "branch 1 'k' is GEO_POINT, branch 3 'k' is KEYWORD") + } + + it should "answer core's UNION ALL exactly as the single run it always was" in { + // every UNION ALL above, through the operator tree and through the flat fold: one answer + val unionAll = Seq( + union("d", "CURRENT_DATE"), + union("n", "'1'"), + union("d", "ts", "d2"), + union("d", "n", "'1'"), + union("n", "NULL", "x"), + union("b", "n"), + union("bin", "s"), + union("bin", "NULL", "s"), + union("d", "n"), + union("b", "n", "d"), + union("bin", "s", "d"), + union("bin", "n"), + union("geo", "s"), + union("bin", "d", "n"), + union("d", "n", "s"), + "SELECT id, d FROM t UNION ALL SELECT g, n FROM t" + ) + val wrong = unionAll.flatMap { sql => + val flat = MultiSearch.columnTypes(branches(sql)).map(_.map(_.resultType)) + val tree = treeTypes(sql) + if (flat == tree) None else Some(s"[$sql] tree $tree, flat $flat") + } + wrong shouldBe empty + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/AggregateWithoutGroupBySpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/AggregateWithoutGroupBySpec.scala new file mode 100644 index 00000000..387f53ba --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/AggregateWithoutGroupBySpec.scala @@ -0,0 +1,336 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.{Sink, Source} +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ + ElasticFailure, + ElasticSuccess, + QueryRows, + QueryStream, + QueryStructured +} +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.persistence.generateUUID +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.time.temporal.ChronoUnit +import java.time.{Instant, LocalDate, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.language.implicitConversions +import scala.util.Try + +/** Issue #413 -- a calculation over aggregates with NO `GROUP BY` answers ONE row, the whole table + * being one implicit group: `SELECT MAX(n) - MIN(n) FROM t`, `COUNT(*) * 2`, `MAX(d) + 1`, + * `GREATEST(MAX(d), MAX(d2))`, `DATEDIFF(MAX(d), MIN(d))`. Elasticsearch refused every one of them + * on every major (a `bucket_script` at the top level of `aggs`). + * + * Every expected value is computed HERE, from the fixture, as DuckDB 1.5.5 and PostgreSQL 16 + * compute it (MEASURED over the same rows), with the date rules of the engine (a DATE moved by + * whole days is a DATE, DATE - DATE is whole days). The value is read as the per-group calculation + * of the same expression is -- a date as the instant its day starts at, which a calculation + * answers as epoch milliseconds -- since the whole table is computed exactly as a group is. + */ +trait AggregateWithoutGroupBySpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + private val table = "aggregate_without_group_by" + + private final case class Doc( + id: String, + g: String, + n: Option[Int], + x: Option[Double], + d: Option[LocalDate], + d2: Option[LocalDate], + ts: Option[Instant] + ) + + private def day(s: String): Option[LocalDate] = Some(LocalDate.parse(s)) + private def at(s: String): Option[Instant] = Some(Instant.parse(s)) + + private val docs: Seq[Doc] = Seq( + Doc( + "r1", + "g1", + Some(3), + Some(0.25), + day("2024-01-31"), + day("2024-02-01"), + at("2024-01-31T10:30:00Z") + ), + Doc( + "r2", + "g1", + Some(5), + Some(1.5), + day("2024-03-01"), + day("2024-02-28"), + at("2024-03-01T06:00:00Z") + ), + Doc("r3", "g2", Some(-2), None, day("2023-12-25"), None, at("2023-12-25T23:59:59.999Z")), + Doc("r4", "g3", None, Some(2.0), None, day("2024-01-31"), None), + Doc( + "r5", + "g3", + Some(10), + Some(-1.25), + day("2024-02-29"), + day("2024-02-29"), + at("2024-02-29T00:00:00Z") + ) + ) + + private def json(d: Doc): String = { + def q(s: String) = "\"" + s + "\"" + Seq( + Some("id" -> q(d.id)), + Some("g" -> q(d.g)), + d.n.map(v => "n" -> v.toString), + d.x.map(v => "x" -> v.toString), + d.d.map(v => "d" -> q(v.toString)), + d.d2.map(v => "d2" -> q(v.toString)), + d.ts.map(v => "ts" -> q(v.toString)) + ).flatten.map { case (k, v) => s""""$k":$v""" }.mkString("{", ",", "}") + } + + override def beforeAll(): Unit = { + super.beforeAll() + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $table"), 60.seconds)) + Await.result( + client.run( + s"CREATE TABLE $table (id KEYWORD, g KEYWORD, n INT, x DOUBLE, d DATE, d2 DATE, " + + "ts TIMESTAMP, PRIMARY KEY (id))" + ), + 60.seconds + ) match { + case ElasticSuccess(_) => + case ElasticFailure(error) => fail(s"CREATE TABLE failed: ${error.message}") + } + implicit val bulkOptions: BulkOptions = BulkOptions(defaultIndex = table, logEvery = 100) + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + client.bulk[String](docs.map(json).toList, identity, idKey = Some(Set("id"))) match { + case ElasticSuccess(r) if r.failedCount == 0 => client.refresh(table) + case other => fail(s"bulk failed: $other") + } + () + } + + override def afterAll(): Unit = { + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $table"), 60.seconds)) + super.afterAll() + } + + // ------------------------------------------------------------------------------------------- + // The oracle + // ------------------------------------------------------------------------------------------- + + private implicit val dayOrdering: Ordering[LocalDate] = Ordering.fromLessThan(_ isBefore _) + + private def max[T](values: Seq[Option[T]])(implicit o: Ordering[T]): Option[T] = + values.flatten.reduceOption((a, b) => if (o.gteq(a, b)) a else b) + private def min[T](values: Seq[Option[T]])(implicit o: Ordering[T]): Option[T] = + values.flatten.reduceOption((a, b) => if (o.lteq(a, b)) a else b) + + private def where(p: Doc => Boolean): Seq[Doc] = docs.filter(p) + + private def start(d: LocalDate): Instant = d.atStartOfDay(ZoneOffset.UTC).toInstant + + /** An expected cell: a number, a day (the instant it starts at), or NULL. */ + private sealed trait Cell + private final case class Num(value: Double) extends Cell + private final case class Day(value: LocalDate) extends Cell + private case object Null extends Cell + + private def num(v: Option[Double]): Cell = v.map(Num).getOrElse(Null) + private def date(v: Option[LocalDate]): Cell = v.map(Day).getOrElse(Null) + + private val all = docs + private val ns = all.map(_.n) + private val ds = all.map(_.d) + + /** statement -> the ONE row it answers, column by column (`None`: no row, a false HAVING) */ + private val expected: Seq[(String, Option[Seq[(String, Cell)]])] = Seq( + // the issue's five statements + s"SELECT MAX(n) - MIN(n) AS v FROM $table" -> + Some(Seq("v" -> num(for (a <- max(ns); b <- min(ns)) yield (a - b).toDouble))), + s"SELECT COUNT(*) * 2 AS v FROM $table" -> Some(Seq("v" -> Num(all.size * 2.0))), + s"SELECT MAX(d) + 1 AS v FROM $table" -> Some(Seq("v" -> date(max(ds).map(_.plusDays(1))))), + s"SELECT GREATEST(MAX(d), MAX(d2)) AS v FROM $table" -> + Some(Seq("v" -> date(max(Seq(max(ds), max(all.map(_.d2))))))), + s"SELECT DATEDIFF(MAX(d), MIN(d)) AS v FROM $table" -> + Some( + Seq("v" -> num(for (a <- max(ds); b <- min(ds)) yield ChronoUnit.DAYS.between(b, a).toDouble)) + ), + // unaliased, beside a plain aggregate, over a filter, with LIMIT and ORDER BY + s"SELECT MAX(n) - MIN(n) FROM $table" -> + Some(Seq("__c1" -> num(for (a <- max(ns); b <- min(ns)) yield (a - b).toDouble))), + s"SELECT MAX(n) AS m, MAX(n) - MIN(n) AS v FROM $table" -> + Some( + Seq( + "m" -> num(max(ns).map(_.toDouble)), + "v" -> num(for (a <- max(ns); b <- min(ns)) yield (a - b).toDouble) + ) + ), + s"SELECT MAX(n) - MIN(n) AS v FROM $table WHERE g = 'g1'" -> + Some(Seq("v" -> { + val g1 = where(_.g == "g1").map(_.n) + num(for (a <- max(g1); b <- min(g1)) yield (a - b).toDouble) + })), + s"SELECT MAX(n) - MIN(n) AS v FROM $table LIMIT 5" -> + Some(Seq("v" -> num(for (a <- max(ns); b <- min(ns)) yield (a - b).toDouble))), + s"SELECT MAX(n) - MIN(n) AS v FROM $table ORDER BY v" -> + Some(Seq("v" -> num(for (a <- max(ns); b <- min(ns)) yield (a - b).toDouble))), + // a fractional result, a DATE - DATE, a DATE moved back, a LEAST + s"SELECT MAX(x) - MIN(x) AS v FROM $table" -> + Some(Seq("v" -> num(for (a <- max(all.map(_.x)); b <- min(all.map(_.x))) yield a - b))), + s"SELECT AVG(n) * 2 AS v FROM $table" -> + Some(Seq("v" -> Num(ns.flatten.sum.toDouble / ns.flatten.size * 2))), + s"SELECT MAX(d) - MIN(d) AS v FROM $table" -> + Some( + Seq("v" -> num(for (a <- max(ds); b <- min(ds)) yield ChronoUnit.DAYS.between(b, a).toDouble)) + ), + s"SELECT MIN(d) - 1 AS v FROM $table" -> Some(Seq("v" -> date(min(ds).map(_.minusDays(1))))), + s"SELECT LEAST(MIN(d), MIN(d2)) AS v FROM $table" -> + Some(Seq("v" -> date(min(Seq(min(ds), min(all.map(_.d2))))))), + s"SELECT COUNT(DISTINCT g) * 10 AS v FROM $table" -> + Some(Seq("v" -> Num(all.map(_.g).distinct.size * 10.0))), + // over no document: a MAX is NULL, and so is the calculation + s"SELECT MAX(n) - MIN(n) AS v FROM $table WHERE g = 'none'" -> Some(Seq("v" -> Null)), + s"SELECT MAX(d) + 1 AS v FROM $table WHERE g = 'none'" -> Some(Seq("v" -> Null)), + // with a HAVING: one row when it holds, none when it does not + s"SELECT COUNT(*) * 2 AS v FROM $table HAVING COUNT(*) > 1" -> + Some(Seq("v" -> Num(all.size * 2.0))), + s"SELECT MAX(n) - MIN(n) AS v FROM $table HAVING MAX(n) - MIN(n) > 100" -> None + ) + + private def asNumber(v: Any): Option[Double] = v match { + case n: java.lang.Number => Some(n.doubleValue()) + case _ => None + } + + private def asInstant(v: Any): Option[Instant] = v match { + case n: java.lang.Number => Some(Instant.ofEpochMilli(Math.round(n.doubleValue()))) + case l: LocalDate => Some(start(l)) + case z: ZonedDateTime => Some(z.toInstant) + case i: Instant => Some(i) + case _ => None + } + + private def differs(cell: Cell, actual: Any): Option[String] = cell match { + case Null => if (actual == null) None else Some(s"$actual instead of NULL") + case Num(want) => + asNumber(actual) match { + case Some(got) if Math.abs(got - want) <= 1e-9 * Math.max(1.0, Math.abs(want)) => None + case _ => Some(s"$actual instead of $want") + } + case Day(want) => + asInstant(actual) match { + case Some(got) if got == start(want) => None + case _ => Some(s"$actual instead of $want") + } + } + + private def rows(sql: String): Either[String, Seq[ListMap[String, Any]]] = + Try { + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) + case ElasticSuccess(QueryStream(stream, _)) => + Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) + case ElasticSuccess(other) => Left(s"unexpected result $other") + case ElasticFailure(error) => Left(error.message) + } + }.toEither.left.map(t => s"${t.getClass.getSimpleName}: ${t.getMessage}").joinRight + + /** The Elasticsearch major the client talks to. */ + private lazy val esMajor: Int = + client.version match { + case ElasticSuccess(v) => v.takeWhile(_ != '.').toInt + case ElasticFailure(_) => 0 + } + + /** Over ZERO documents (the lead's ruling of 2026-10-06): `COUNT(*)` is 0 on every major (the + * bucket's own `_count`); `COUNT(x)` and `COUNT(DISTINCT x)` are 0 from Elasticsearch 7.17 + * (`keep_values`), and NULL on 6.8, which refuses that gap policy -- an Elasticsearch + * limitation; a `SUM` is core's 0.0 there too; a `MIN`, a `MAX` or an `AVG` is NULL, and so is a + * calculation reading one, as DuckDB 1.5.5 and PostgreSQL 16 answer (MEASURED). `HAVING COUNT(*) + * = 0` keeps the one row. + */ + private def overNothing: Seq[(String, Option[Seq[(String, Cell)]])] = { + val none = s"FROM $table WHERE g = 'none'" + def countOfValues(v: Double): Cell = if (esMajor >= 7) Num(v) else Null + Seq( + s"SELECT COUNT(*) * 2 AS v $none" -> Some(Seq("v" -> Num(0))), + s"SELECT COUNT(*) AS c, COUNT(*) * 2 AS v $none" -> Some(Seq("c" -> Num(0), "v" -> Num(0))), + s"SELECT COUNT(n) * 2 AS v $none" -> Some(Seq("v" -> countOfValues(0))), + s"SELECT COUNT(*) + COUNT(n) AS v $none" -> Some(Seq("v" -> countOfValues(0))), + s"SELECT COUNT(DISTINCT g) * 10 AS v $none" -> Some(Seq("v" -> countOfValues(0))), + s"SELECT SUM(x) * 2 AS v $none" -> Some(Seq("v" -> countOfValues(0))), + s"SELECT COUNT(*) + MAX(n) AS v $none" -> Some(Seq("v" -> Null)), + s"SELECT AVG(n) * 2 AS v $none" -> Some(Seq("v" -> Null)), + s"SELECT DATEDIFF(MAX(d), MIN(d)) AS v $none" -> Some(Seq("v" -> Null)), + s"SELECT GREATEST(MAX(d), MAX(d2)) AS v $none" -> Some(Seq("v" -> Null)), + s"SELECT COUNT(*) AS c $none HAVING COUNT(*) = 0" -> Some(Seq("c" -> Num(0))), + s"SELECT COUNT(*) AS c $none HAVING COUNT(*) > 0" -> None, + s"SELECT COUNT(*) * 2 AS v $none HAVING COUNT(*) = 0" -> Some(Seq("v" -> Num(0))), + // one document without the column: its count is 0, its MAX NULL, on every major + s"SELECT COUNT(x) * 2 AS v FROM $table WHERE id = 'r3'" -> Some(Seq("v" -> Num(0))), + s"SELECT MAX(x) - MIN(x) AS v FROM $table WHERE id = 'r3'" -> Some(Seq("v" -> Null)) + ) + } + + private def mismatches(statements: Seq[(String, Option[Seq[(String, Cell)]])]): Seq[String] = + statements.flatMap { case (sql, want) => + rows(sql) match { + case Left(error) => Seq(s"[$sql] failed: $error") + case Right(rs) => + want match { + case None => + if (rs.isEmpty) Nil else Seq(s"[$sql] answered ${rs.size} rows instead of none") + case Some(cells) => + if (rs.size != 1) Seq(s"[$sql] answered ${rs.size} rows instead of one: $rs") + else + cells.flatMap { case (column, cell) => + differs(cell, rs.head.getOrElse(column, null)).map(e => s"[$sql] $column: $e") + } + } + } + } + + "A calculation over aggregates without GROUP BY" should "answer ONE row, with the values SQL engines answer" in { + mismatches(expected) shouldBe empty + } + + it should "answer the counts of no document, and NULL for a MIN or a MAX of none" in { + mismatches(overNothing) shouldBe empty + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala index f39e830c..0c4dedd5 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala @@ -1112,6 +1112,57 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with verdict("per-group SELECT", groupVerdicts(groupForms)) } + /** A `DATE` or `TIMESTAMP` conversion over a per-group date calculation, as DuckDB converts it: a + * `DATE` is the calculation's UTC day, a `TIMESTAMP` the instant itself. Per group the value is + * the number of milliseconds a calculation answers there, and the conversion used to call a + * `java.time` method on that number: Elasticsearch failed the search (`member method [double, + * toInstant/0] not found`). + */ + private val castGroupForms: Seq[GroupForm] = Seq( + GroupForm("(MAX(d) + 1)::DATE", g => expect(g.maxD.map(a => At(start(a.plusDays(1)))))), + GroupForm("(MAX(d) + 1)::TIMESTAMP", g => expect(g.maxD.map(a => At(start(a.plusDays(1)))))), + GroupForm("(MAX(d) - 1)::DATE", g => expect(g.maxD.map(a => At(start(a.minusDays(1)))))), + GroupForm( + "(MAX(d) + 1.5)::DATE", + g => expect(g.maxD.map(a => At(start(utcDay(start(a).plusMillis(DayMillis * 3 / 2)))))) + ), + GroupForm("(MAX(d) + 1.5)::TIMESTAMP", g => expect(g.maxD.map(a => moved(start(a), 1.5)))), + GroupForm( + "(MAX(ts) - 0.5)::DATE", + g => expect(g.maxTs.map(t => At(start(utcDay(t.minusMillis(DayMillis / 2)))))) + ), + GroupForm("(MAX(ts) - 0.5)::TIMESTAMP", g => expect(g.maxTs.map(t => moved(t, -0.5)))), + GroupForm( + "(MAX(ts) + 1)::DATE", + g => expect(g.maxTs.map(t => At(start(utcDay(t.plusMillis(DayMillis)))))) + ), + GroupForm( + "GREATEST(MAX(d), MAX(d2))::DATE", + g => expect(for (a <- g.maxD; b <- g.maxD2) yield At(start(if (a.isAfter(b)) a else b))) + ), + GroupForm( + "GREATEST(MAX(d), MAX(d2))::TIMESTAMP", + g => expect(for (a <- g.maxD; b <- g.maxD2) yield At(start(if (a.isAfter(b)) a else b))) + ) + ) + + "a DATE or TIMESTAMP conversion over a per-group date calculation" should "answer the java.time oracle in every group, in a HAVING over its alias, and over the whole table" in { + verdict("per-group conversion", groupVerdicts(castGroupForms)) + // a HAVING over the converted alias keeps the groups whose day is after February 1 + val having = s"SELECT g, (MAX(d) + 1)::DATE AS x FROM $table GROUP BY g HAVING x > '2024-02-01'" + val want = groups + .filter(_.maxD.exists(a => a.plusDays(1).isAfter(on("2024-02-01")))) + .map(_.g) + .toSet + want should not be empty + gatewayRows(having).map(_.map(_.getOrElse("g", "?").toString).toSet) shouldBe Right(want) + // with no GROUP BY, the whole table is one group + val maxD = rows.flatMap(_.d).reduce((a, b) => if (a.isAfter(b)) a else b) + gatewayRows(s"SELECT (MAX(d) + 1)::DATE AS x FROM $table").map(rs => + rs.map(r => differs(At(start(maxD.plusDays(1))), r.getOrElse("x", null))) + ) shouldBe Right(Seq(None)) + } + "a HAVING over date arithmetic" should "keep exactly the groups the java.time oracle keeps" in { havingForms.exists(f => groups.exists(g => f.keeps(g).contains(true))) shouldBe true havingForms.exists(f => groups.exists(g => !f.keeps(g).contains(true))) shouldBe true @@ -2667,7 +2718,9 @@ trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with "LEAST requires numeric arguments, or date and timestamp arguments, but got VARCHAR for 'a'" ), s"SELECT id, 1 + 'abc' AS x FROM $table" -> Seq("input 'VARCHAR'"), - s"SELECT id FROM $table UNION ALL SELECT n FROM $table" -> Seq( + // a DATE beside an INT, which DuckDB refuses too (a KEYWORD beside an INT is a VARCHAR column + // there, answered since the lead's rule of 2026-10-06) + s"SELECT d FROM $table UNION ALL SELECT n FROM $table" -> Seq( "compatible types at column 1" ) ) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala new file mode 100644 index 00000000..2e476721 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SetOperationTypeSpec.scala @@ -0,0 +1,1643 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.{Sink, Source} +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ + ElasticFailure, + ElasticSuccess, + QueryRows, + QueryStream, + QueryStructured +} +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.elastic.sql.query.SelectStatement +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes, ValueCoercion} +import app.softnetwork.persistence.generateUUID +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.time.{Instant, LocalDate, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.language.implicitConversions +import scala.util.Try + +/** A `UNION ALL` answers what DuckDB 1.5.5 answers where DuckDB types the column (the relational + * engine that runs `UNION`, `INTERSECT` and `EXCEPT` is DuckDB, so its answer is the reference), + * `VARCHAR` where DuckDB gives the column no type, and refuses only a date beside a number -- the + * lead's rules of 2026-10-07. + * + * For every statement: the VERDICT (answered, or refused by name), the VALUES, the CLASS of every + * row's value -- ONE class per column: every row of a `DOUBLE` column is a `Double`, of a `BIGINT` + * one a `Long`, of a `VARCHAR` one a `String` -- and the type JDBC and Arrow REPORT, which they + * read from the column's first value. Each statement runs through the three routes a `UNION ALL` + * takes: the gateway (`run`), the client's `search` (un-LIMITed legs, executed leg by leg) and + * branches bounded by a `LIMIT` (one `_msearch`). + * + * Every expectation is computed HERE, from the fixture, as DuckDB 1.5.5 computes it (MEASURED + * statement by statement over the same rows) -- never read from the engine. PostgreSQL 16 refuses + * most of the text pairs (`UNION types text and integer cannot be matched`); DuckDB comes first. + * + * A computed branch (a literal, `CURRENT_DATE`, a `CAST`) reaches the client as Elasticsearch's + * one-element `script_fields` array, on every path and in a single statement alike; a column whose + * branches already project one type is not converted, so the value is read here through that array + * (`scalar`), as every other spec of this kit reads a computed column. + */ +trait SetOperationTypeSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + private val table = "set_operation_types" + + /** One document: `None` is a column it does not carry. */ + private final case class Doc( + id: String, + s: Option[String], + txt: Option[String], + n: Option[Int], + x: Option[Double], + f: Option[Float], + bg: Option[Long], + sm: Option[Short], + d: Option[LocalDate], + d2: Option[LocalDate], + ts: Option[Instant], + b: Option[Boolean] + ) + + private def day(s: String): Option[LocalDate] = Some(LocalDate.parse(s)) + private def at(s: String): Option[Instant] = Some(Instant.parse(s)) + + private val docs: Seq[Doc] = Seq( + Doc( + "r1", + Some("1"), + Some("hello one"), + Some(3), + Some(0.25), + Some(0.5f), + Some(3000000000L), + Some(3.toShort), + day("2024-01-31"), + day("2024-02-01"), + at("2024-01-31T10:30:00Z"), + Some(true) + ), + Doc( + "r2", + Some("5"), + Some("hello two"), + Some(5), + Some(1.5), + Some(1.25f), + Some(5L), + Some(5.toShort), + day("2024-03-01"), + day("2024-02-28"), + at("2024-03-01T06:00:00Z"), + Some(false) + ), + Doc( + "r3", + Some("12"), + None, + Some(-2), + None, + None, + Some(-2L), + Some(-2.toShort), + day("2023-12-25"), + None, + at("2023-12-25T23:59:59.999Z"), + None + ), + Doc( + "r4", + None, + Some("hello four"), + None, + Some(2.0), + Some(2.0f), + None, + None, + None, + day("2024-01-31"), + None, + Some(true) + ), + Doc( + "r5", + Some("3"), + Some("hello five"), + Some(10), + Some(-1.25), + Some(-0.75f), + Some(10L), + Some(10.toShort), + day("2024-02-29"), + day("2024-02-29"), + at("2024-02-29T00:00:00Z"), + Some(false) + ) + ) + + private def json(d: Doc): String = { + def q(s: String) = "\"" + s + "\"" + val fields = Seq( + Some("id" -> q(d.id)), + d.s.map(v => "s" -> q(v)), + d.txt.map(v => "txt" -> q(v)), + d.n.map(v => "n" -> v.toString), + d.x.map(v => "x" -> v.toString), + d.f.map(v => "f" -> v.toString), + d.bg.map(v => "bg" -> v.toString), + d.sm.map(v => "sm" -> v.toString), + d.d.map(v => "d" -> q(v.toString)), + d.d2.map(v => "d2" -> q(v.toString)), + d.ts.map(v => "ts" -> q(v.toString)), + d.b.map(v => "b" -> v.toString) + ).flatten + fields.map { case (k, v) => s""""$k":$v""" }.mkString("{", ",", "}") + } + + /** The values a `VARCHAR` column spells: each as its JSON, and DuckDB 1.5.5's text for it -- + * `CAST( AS VARCHAR)`, MEASURED value by value, the timestamp read in UTC as the + * relational engine loads it. `ts` holds a date-only value, one before the epoch, fractional + * seconds and an offset; `x` a whole number written without a fraction. + */ + private final case class Spelled(json: String, text: String) + + private final case class TextDoc( + id: String, + s: Option[String], + x: Option[Spelled], + f: Option[Spelled], + ts: Option[Spelled], + time: Option[String], + date: Option[String] + ) + + private val textTable = "set_operation_text" + + private def sp(json: String, text: String): Option[Spelled] = Some(Spelled(json, text)) + + private val textDocs: Seq[TextDoc] = Seq( + TextDoc( + "t1", + Some("a"), + sp("2.5E11", "250000000000.0"), + sp("0.1", "0.1"), + sp("\"2024-01-31T10:30:00Z\"", "2024-01-31 10:30:00"), + Some("10:30:00"), + Some("2024-01-31") + ), + TextDoc( + "t2", + Some("b"), + sp("1.0E-5", "1e-05"), + sp("-0.75", "-0.75"), + sp("\"2023-12-25T23:59:59.999Z\"", "2023-12-25 23:59:59.999"), + Some("23:59:59.999"), + Some("2023-12-25") + ), + TextDoc( + "t3", + Some("c"), + sp("1.0E16", "1e+16"), + sp("2.5E11", "250000000000.0"), + sp("\"2024-01-31\"", "2024-01-31 00:00:00"), + Some("00:00:00"), + Some("2024-01-31") + ), + TextDoc( + "t4", + None, + sp("-2.5", "-2.5"), + sp("4073260.75", "4073260.75"), + sp("\"1969-12-31T23:59:59.999Z\"", "1969-12-31 23:59:59.999"), + Some("23:59:59.999"), + Some("1969-12-31") + ), + TextDoc("t5", Some("e"), sp("5", "5.0"), sp("16777216", "16777216.0"), None, None, None), + TextDoc( + "t6", + Some("f"), + sp("7.168E25", "7.1680000000000004e+25"), + sp("0.5", "0.5"), + sp("\"2024-03-01T06:00:00.5Z\"", "2024-03-01 06:00:00.5"), + Some("06:00:00.5"), + Some("2024-03-01") + ), + TextDoc( + "t7", + Some("g"), + sp("-1.0E-5", "-1e-05"), + None, + sp("\"2024-02-29T00:00:00Z\"", "2024-02-29 00:00:00"), + Some("00:00:00"), + Some("2024-02-29") + ), + TextDoc( + "t8", + Some("h"), + sp("0.30000000000000004", "0.30000000000000004"), + sp("1.25", "1.25"), + sp("\"2024-01-31T12:30:00+02:00\"", "2024-01-31 10:30:00"), + Some("10:30:00"), + Some("2024-01-31") + ) + ) + + /** `bin`, a `binary` field -- the base64 Elasticsearch holds -- by document. */ + private val binaryOf: Map[String, String] = Map( + "t1" -> "YWI=", + "t2" -> "eA==", + "t4" -> "AAEC", + "t5" -> "aGVsbG8=", + "t7" -> "/w==", + "t8" -> "YQ==" + ) + + /** `u`, a keyword with a character outside ASCII, which DuckDB cannot cast to a BLOB. */ + private val nonAsciiOf: Map[String, String] = Map("t1" -> "\u00e9t\u00e9", "t3" -> "a") + + /** `tags`, a keyword holding several values in one document (its JSON, an array). */ + private val tagsOf: Map[String, String] = Map("t1" -> "[\"a\",\"b\"]", "t2" -> "\"c\"") + + /** `geo`, a `geo_point` written as `"lat,lon"` text, and `items`, a list of objects (a `nested` + * field, an `ARRAY`), by document. + */ + private val geoOf: Map[String, String] = Map("t1" -> "48.85,2.35", "t2" -> "40.71,-74.0") + + /** The same points stored as an object (`geoo`) and as an array `[lon, lat]` (`geoa`), and `obj`, + * an object, by document: the JSON each holds. + */ + private val geoObjectOf: Map[String, String] = + Map("t1" -> """{"lat":48.85,"lon":2.35}""", "t2" -> """{"lat":40.71,"lon":-74.0}""") + private val geoArrayOf: Map[String, String] = + Map("t1" -> "[2.35,48.85]", "t2" -> "[-74.0,40.71]") + private val objOf: Map[String, (Int, String)] = Map("t1" -> (1 -> "x"), "t3" -> (3 -> "z")) + + /** `oi`, `od` and `ov`, three objects with one field `a` -- an `INT`, a `DATE` and a `KEYWORD` + * -- by document: the JSON each holds. + */ + private val fieldOf: Map[String, Map[String, String]] = Map( + "oi" -> Map("t1" -> """{"a":1}"""), + "od" -> Map("t1" -> """{"a":"2024-01-31"}"""), + "ov" -> Map("t1" -> """{"a":"x"}""") + ) + + /** The value generator's types on Elasticsearch, each as declared, at four depths: `c_` a + * column, `f_` an object's field `a`, `n_` the field `a` of a list of objects' + * element (a list of values is read without a type today), `s_` the field `c` of an + * object's field `p`. + */ + private val generatedTypes: Seq[(String, String)] = Seq( + "INT" -> "INT", + "BIGINT" -> "BIGINT", + "DOUBLE" -> "DOUBLE", + "BOOLEAN" -> "BOOLEAN", + "DATE" -> "DATE", + "TIMESTAMP" -> "TIMESTAMP", + "VARBINARY" -> "VARBINARY", + "VARCHAR" -> "KEYWORD" + ) + + /** Each type's values, by document, as the JSON written -- in the forms Elasticsearch 8.18.3 + * accepts and core reads differently (MEASURED): a `BIGINT` or a `DOUBLE` whose JSON fits an int + * (an `Integer`); a number written as text, which Elasticsearch indexes as its field's type -- + * truncated toward zero in an integer field (`"5.5"`, `5.5` are 5), an exponent applied (`"5e2"` + * is 500), a space trimmed in an `INT` field; `"true"`, `"false"` and an empty text (`false`) in + * a `BOOLEAN` field; a `TIMESTAMP` written as a date (a `LocalDate`), without a zone (a + * `LocalDateTime`), with one or an offset (a `ZonedDateTime`), or as epoch milliseconds, a + * number (a `Long`) or a text; a `DATE` written as epoch milliseconds or with a time. + */ + private val storedOf: Map[String, Map[String, String]] = Map( + "INT" -> Map("t1" -> "1", "t2" -> "-2", "t3" -> "\"5\"", "t4" -> "\" 5 \"", "t5" -> "\"5.5\""), + "BIGINT" -> Map( + "t1" -> "1", + "t2" -> "3000000000", + "t3" -> "\"5\"", + "t4" -> "\"5.0\"", + "t5" -> "\"5e2\"", + "t6" -> "5.5", + "t7" -> "\"-5.9\"" + ), + "DOUBLE" -> Map( + "t1" -> "2", + "t2" -> "2.5", + "t3" -> "\"2.5\"", + "t4" -> "\"1e3\"", + "t5" -> "\" 2.5 \"", + "t6" -> "\"5\"" + ), + "BOOLEAN" -> Map( + "t1" -> "true", + "t2" -> "false", + "t3" -> "\"true\"", + "t4" -> "\"false\"", + "t5" -> "\"\"" + ), + "DATE" -> Map( + "t1" -> "\"2024-01-31\"", + "t2" -> "\"2023-12-25\"", + "t3" -> "\"1706697000000\"", + "t4" -> "1706697000000", + "t5" -> "\"2024-01-31T10:30:00Z\"" + ), + "TIMESTAMP" -> Map( + "t1" -> "\"2024-01-31\"", + "t2" -> "\"2024-01-31T10:30:00\"", + "t3" -> "\"2024-01-31T10:30:00Z\"", + "t4" -> "1706697000000", + "t5" -> "\"2024-01-31T12:30:00+02:00\"", + "t6" -> "\"1706697000000\"" + ), + "VARBINARY" -> Map("t1" -> "\"YWI=\"", "t2" -> "\"eA==\""), + "VARCHAR" -> Map("t1" -> "\"x\"", "t2" -> "\"y\"") + ) + + private def generatedJson(id: String): Seq[(String, String)] = + generatedTypes.flatMap { case (t, _) => + storedOf(t).get(id).toSeq.flatMap { v => + Seq( + s"c_$t" -> v, + s"f_$t" -> s"""{"a":$v}""", + s"n_$t" -> s"""[{"a":$v}]""", + s"s_$t" -> s"""{"p":{"c":$v}}""" + ) + } + } + + /** Each point, `{lat, lon}`, by document -- every storage holds the same two. */ + private val pointOf: Map[String, (Double, Double)] = + Map("t1" -> (48.85 -> 2.35), "t2" -> (40.71 -> -74.0)) + private val itemsOf: Map[String, String] = + Map("t1" -> """[{"p":"a","q":1}]""", "t2" -> """[{"p":"b","q":2},{"p":"c","q":3}]""") + + /** `kw`, keywords that LOOK like a time and a timestamp, which core re-reads as such (#418). */ + private val temporalKeywordOf: Map[String, String] = + Map("t1" -> "10:30:00", "t2" -> "2024-01-31T10:30:00Z", "t3" -> "plain") + + private def textJson(d: TextDoc): String = + Seq( + Some("id" -> ("\"" + d.id + "\"")), + d.s.map(v => "s" -> ("\"" + v + "\"")), + d.x.map(v => "x" -> v.json), + d.f.map(v => "f" -> v.json), + d.ts.map(v => "ts" -> v.json), + binaryOf.get(d.id).map(v => "bin" -> ("\"" + v + "\"")), + nonAsciiOf.get(d.id).map(v => "u" -> ("\"" + v + "\"")), + tagsOf.get(d.id).map(v => "tags" -> v), + temporalKeywordOf.get(d.id).map(v => "kw" -> ("\"" + v + "\"")), + geoOf.get(d.id).map(v => "geo" -> ("\"" + v + "\"")), + itemsOf.get(d.id).map(v => "items" -> v), + geoObjectOf.get(d.id).map(v => "geoo" -> v), + geoArrayOf.get(d.id).map(v => "geoa" -> v), + objOf.get(d.id).map { case (a, b) => "obj" -> s"""{"a":$a,"b":"$b"}""" }, + fieldOf("oi").get(d.id).map(v => "oi" -> v), + fieldOf("od").get(d.id).map(v => "od" -> v), + fieldOf("ov").get(d.id).map(v => "ov" -> v) + ).flatten + .++(generatedJson(d.id)) + .map { case (k, v) => s""""$k":$v""" } + .mkString("{", ",", "}") + + /** `f_` / `c_`: a date field of a mapping format, as a column and as an object's field + * `a`, and the values written to it by document (JSON), each with the instant Elasticsearch + * indexed -- MEASURED on 6.8.23, 7.17.29 and 8.18.3 (core's date-format corpora); `es6` the + * instant where Elasticsearch 6 indexes another one. `None` is Elasticsearch's default format. + */ + private final case class Formatted( + key: String, + declared: String, + format: Option[String], + values: Map[String, (String, Instant, Map[Int, Instant])] + ) + + private val formatTable = "set_operation_formats" + + private val formatted: Seq[Formatted] = { + val at1030 = Instant.parse("2024-01-31T10:30:00Z") + val midnight = Instant.parse("2024-01-31T00:00:00Z") + val year2024 = Instant.parse("2024-01-01T00:00:00Z") + // the instant Elasticsearch 8 (and 9) indexed, and the majors that indexed another one + def one(json: String, at: Instant): (String, Instant, Map[Int, Instant]) = (json, at, Map.empty) + Seq( + // seconds, not milliseconds; a fraction kept to the millisecond, truncated to the second + // by Elasticsearch 6 + Formatted( + "es", + "TIMESTAMP", + Some("epoch_second"), + Map( + "f1" -> one("1706697000", at1030), + "f2" -> ("\"1706697000.5\"", Instant.parse("2024-01-31T10:30:00.500Z"), Map(6 -> at1030)), + "f3" -> one("-86400", Instant.parse("1969-12-31T00:00:00Z")) + ) + ), + Formatted( + "em", + "TIMESTAMP", + Some("epoch_millis"), + Map( + "f1" -> one("\"1706697000000\"", at1030), + "f2" -> one("1706697000123", Instant.parse("2024-01-31T10:30:00.123Z")) + ) + ), + Formatted( + "c1", + "TIMESTAMP", + Some("yyyy/MM/dd"), + Map("f1" -> one("\"2024/01/31\"", midnight)) + ), + Formatted( + "c3", + "TIMESTAMP", + Some("dd/MM/yyyy HH:mm"), + Map("f1" -> one("\"31/01/2024 10:30\"", at1030)) + ), + Formatted( + "c4", + "TIMESTAMP", + Some("yyyy-MM-dd HH:mm:ss Z"), + Map("f1" -> one("\"2024-01-31 12:30:00 +0200\"", at1030)) + ), + Formatted( + "c7", + "TIMESTAMP", + Some("MMM dd, yyyy"), + Map("f1" -> one("\"Jan 31, 2024\"", midnight)) + ), + Formatted( + "bd", + "TIMESTAMP", + Some("basic_date"), + Map("f1" -> one("\"20240131\"", midnight), "f2" -> one("20240131", midnight)) + ), + Formatted( + "sdt", + "TIMESTAMP", + Some("strict_date_time"), + Map( + "f1" -> one( + "\"2024-01-31T12:30:00.123+02:00\"", + Instant.parse("2024-01-31T10:30:00.123Z") + ) + ) + ), + // a year of one to nine digits; a fraction kept to the millisecond + Formatted( + "dot", + "TIMESTAMP", + Some("date_optional_time"), + Map( + "f1" -> one("\"2024-01-31T10:30:00.123456Z\"", Instant.parse("2024-01-31T10:30:00.123Z")), + "f2" -> one("\"24\"", Instant.parse("0024-01-01T00:00:00Z")) + ) + ), + // the first alternative that parses decides + Formatted( + "l3", + "TIMESTAMP", + Some("yyyy-dd-MM||strict_date_optional_time"), + Map( + "f1" -> one("\"2024-01-02\"", Instant.parse("2024-02-01T00:00:00Z")), + "f2" -> one("\"2024-01-31T10:30:00Z\"", at1030) + ) + ), + Formatted( + "l2", + "TIMESTAMP", + Some("epoch_second||yyyy-MM-dd"), + Map("f1" -> one("\"2024-01-31\"", midnight), "f2" -> one("1706697000", at1030)) + ), + // Elasticsearch's default format reads four digits as a year, before epoch milliseconds + Formatted( + "def", + "TIMESTAMP", + None, + Map( + "f1" -> one("\"2024\"", year2024), + "f2" -> one("2024", year2024), + "f3" -> one("\"2024-01-31T12:30:00+0200\"", at1030), + "f4" -> one("\"2024-01-31T10:30:00.123456Z\"", Instant.parse("2024-01-31T10:30:00.123Z")) + ) + ), + // a week-based year (MEASURED): Elasticsearch 8 reads the first day of its first week, weeks + // starting on Sunday -- the month and the day ignored; Elasticsearch 7 the same with ISO weeks; + // Elasticsearch 6's (Joda) `Y` is the year of era + Formatted( + "wy", + "TIMESTAMP", + Some("YYYY-MM-dd"), + Map( + "f1" -> ( + "\"2024-01-31\"", + Instant.parse("2023-12-31T00:00:00Z"), + Map(7 -> Instant.parse("2024-01-01T00:00:00Z"), 6 -> midnight) + ), + "f2" -> ( + "\"2021-01-01\"", + Instant.parse("2020-12-27T00:00:00Z"), + Map( + 7 -> Instant.parse("2021-01-04T00:00:00Z"), + 6 -> Instant.parse("2021-01-01T00:00:00Z") + ) + ) + ) + ), + // a day's name (MEASURED): Elasticsearch 7 and 8 ignore it beside a year of era, Elasticsearch 6 + // moves the date to that day of its week + Formatted( + "dn", + "TIMESTAMP", + Some("yyyy-MM-dd EEE"), + Map( + "f1" -> one("\"2024-01-31 Wed\"", midnight), + "f2" -> ( + "\"2024-01-31 Mon\"", + midnight, + Map(6 -> Instant.parse("2024-01-29T00:00:00Z")) + ) + ) + ), + // a DATE field is the day of the instant, in UTC + Formatted( + "dfmt", + "DATE", + Some("dd/MM/yyyy HH:mm"), + Map("f1" -> one("\"31/01/2024 23:30\"", Instant.parse("2024-01-31T23:30:00Z"))) + ), + Formatted( + "des", + "DATE", + Some("epoch_second"), + Map("f1" -> one("1706697000", at1030)) + ) + ) + } + + /** The documents of [[formatTable]]: `f1`..`f4` carry the formatted fields, `p1` the partners -- + * `pd` a `DATE`, `pts` a `TIMESTAMP`, `s` a keyword, as columns and as objects' field `a`. + */ + private val formatDocs: Seq[String] = Seq("f1", "f2", "f3", "f4", "p1") + + private def formatJson(id: String): String = + (Seq("id" -> ("\"" + id + "\"")) ++ + formatted.flatMap(f => + f.values.get(id).toSeq.flatMap { case (v, _, _) => + Seq(s"c_${f.key}" -> v, s"f_${f.key}" -> s"""{"a":$v}""") + } + ) ++ (if (id == "p1") + Seq( + "pd" -> "\"2024-02-01\"", + "pts" -> "\"2024-02-01T00:00:00Z\"", + "s" -> "\"x\"", + "fd" -> """{"a":"2024-02-01"}""", + "fts" -> """{"a":"2024-02-01T00:00:00Z"}""", + "fs" -> """{"a":"x"}""" + ) + else Nil)) + .map { case (k, v) => s""""$k":$v""" } + .mkString("{", ",", "}") + + override def beforeAll(): Unit = { + super.beforeAll() + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $table"), 60.seconds)) + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $textTable"), 60.seconds)) + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $formatTable"), 60.seconds)) + run( + s"CREATE TABLE $table (id KEYWORD, s KEYWORD, txt TEXT, n INT, x DOUBLE, f REAL, " + + "bg BIGINT, sm SMALLINT, d DATE, d2 DATE, ts TIMESTAMP, b BOOLEAN, PRIMARY KEY (id))" + ) + run( + s"CREATE TABLE $textTable (id KEYWORD, s KEYWORD, x DOUBLE, f REAL, ts TIMESTAMP, " + + "bin VARBINARY, u KEYWORD, tags KEYWORD, kw KEYWORD, geo GEO_POINT, " + + "items ARRAY FIELDS(p KEYWORD, q INT), geoo GEO_POINT, geoa GEO_POINT, " + + "obj STRUCT FIELDS(a INT, b KEYWORD), oi STRUCT FIELDS(a INT), od STRUCT FIELDS(a DATE), " + + "ov STRUCT FIELDS(a KEYWORD), " + + generatedTypes + .map { case (t, declared) => + s"c_$t $declared, f_$t STRUCT FIELDS(a $declared), " + + s"n_$t ARRAY FIELDS(a $declared), s_$t STRUCT FIELDS(p STRUCT FIELDS(c $declared))" + } + .mkString(", ") + + ", PRIMARY KEY (id))" + ) + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + def load(index: String, rows: List[String]): Unit = { + implicit val bulkOptions: BulkOptions = BulkOptions(defaultIndex = index, logEvery = 100) + client.bulk[String](rows, identity, idKey = Some(Set("id"))) match { + case ElasticSuccess(r) if r.failedCount == 0 => client.refresh(index) + case other => fail(s"bulk failed: $other") + } + } + load(table, docs.map(json).toList) + load(textTable, textDocs.map(textJson).toList) + run( + s"CREATE TABLE $formatTable (id KEYWORD, pd DATE, pts TIMESTAMP, s KEYWORD, " + + "fd STRUCT FIELDS(a DATE), fts STRUCT FIELDS(a TIMESTAMP), fs STRUCT FIELDS(a KEYWORD), " + + formatted + .map { f => + val options = f.format.fold("")(spec => s" OPTIONS (format = '$spec')") + s"c_${f.key} ${f.declared}$options, f_${f.key} STRUCT FIELDS(a ${f.declared}$options)" + } + .mkString(", ") + + ", PRIMARY KEY (id))" + ) + load(formatTable, formatDocs.map(formatJson).toList) + () + } + + override def afterAll(): Unit = { + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $table"), 60.seconds)) + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $textTable"), 60.seconds)) + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $formatTable"), 60.seconds)) + super.afterAll() + } + + private def run(sql: String): Unit = + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(_) => () + case ElasticFailure(error) => fail(s"[$sql] failed: ${error.message}") + } + + // ------------------------------------------------------------------------------------------- + // The oracle + // ------------------------------------------------------------------------------------------- + + /** The class DuckDB's type is read as, and the type JDBC / Arrow report for it. */ + private final case class ColumnType(name: String, cls: Class[_], reported: SQLType) + + private val INTEGER = ColumnType("INTEGER", classOf[java.lang.Integer], SQLTypes.Int) + private val BIGINT = ColumnType("BIGINT", classOf[java.lang.Long], SQLTypes.BigInt) + private val DOUBLE = ColumnType("DOUBLE", classOf[java.lang.Double], SQLTypes.Double) + private val FLOAT = ColumnType("FLOAT", classOf[java.lang.Float], SQLTypes.Real) + private val VARCHAR = ColumnType("VARCHAR", classOf[String], SQLTypes.Varchar) + private val DATE = ColumnType("DATE", classOf[LocalDate], SQLTypes.Date) + private val TIMESTAMP = ColumnType("TIMESTAMP", classOf[ZonedDateTime], SQLTypes.Timestamp) + private val BOOLEAN = ColumnType("BOOLEAN", classOf[java.lang.Boolean], SQLTypes.Boolean) + private val VARBINARY = ColumnType("BLOB", classOf[Array[Byte]], SQLTypes.VarBinary) + + /** A branch: its SELECT item, and its value for each document in the COLUMN's type. */ + private final case class Branch(item: String, value: Doc => Any) + + private def branch(item: String)(value: Doc => Any): Branch = Branch(item, value) + + private def opt(o: Option[Any]): Any = o.orNull + + private def midnight(d: LocalDate): ZonedDateTime = d.atStartOfDay(ZoneOffset.UTC) + + private def utc(i: Instant): ZonedDateTime = i.atZone(ZoneOffset.UTC) + + private def today: LocalDate = LocalDate.now(ZoneOffset.UTC) + + /** One statement: its branches, and the type DuckDB gives its column. */ + private final case class Case(id: String, branches: Seq[Branch], column: ColumnType) { + def sql(limit: Boolean): String = + branches + .map(b => s"SELECT ${b.item} AS k FROM $table" + (if (limit) " LIMIT 100" else "")) + .mkString(" UNION ALL ") + def expected: Seq[Any] = branches.flatMap(b => docs.map(b.value)) + } + + private val cases: Seq[Case] = Seq( + // the brief's pairs DuckDB answers + Case( + "d, CURRENT_DATE", + Seq(branch("d")(r => opt(r.d)), branch("CURRENT_DATE")(_ => today)), + DATE + ), + Case( + "n, '1'", + Seq(branch("n")(r => opt(r.n.map(_.toString))), branch("'1'")(_ => "1")), + VARCHAR + ), + Case( + "d, '2024-01-31'", + Seq(branch("d")(r => opt(r.d.map(_.toString))), branch("'2024-01-31'")(_ => "2024-01-31")), + VARCHAR + ), + Case("id, n", Seq(branch("id")(_.id), branch("n")(r => opt(r.n.map(_.toString)))), VARCHAR), + Case("1, 'a'", Seq(branch("1")(_ => "1"), branch("'a'")(_ => "a")), VARCHAR), + Case( + "d, ts", + Seq(branch("d")(r => opt(r.d.map(midnight))), branch("ts")(r => opt(r.ts.map(utc)))), + TIMESTAMP + ), + Case( + "ts, CAST(d AS DATE)", + Seq( + branch("ts")(r => opt(r.ts.map(utc))), + branch("CAST(d AS DATE)")(r => opt(r.d.map(midnight))) + ), + TIMESTAMP + ), + Case("d, d2", Seq(branch("d")(r => opt(r.d)), branch("d2")(r => opt(r.d2))), DATE), + Case( + "n, x", + Seq(branch("n")(r => opt(r.n.map(_.toDouble))), branch("x")(r => opt(r.x))), + DOUBLE + ), + Case( + "n, CAST(n AS BIGINT)", + Seq( + branch("n")(r => opt(r.n.map(_.toLong))), + branch("CAST(n AS BIGINT)")(r => opt(r.n.map(_.toLong))) + ), + BIGINT + ), + Case("n, 2", Seq(branch("n")(r => opt(r.n)), branch("2")(_ => 2)), INTEGER), + Case("s, id", Seq(branch("s")(r => opt(r.s)), branch("id")(_.id)), VARCHAR), + // the other way round + Case( + "'1', n", + Seq(branch("'1'")(_ => "1"), branch("n")(r => opt(r.n.map(_.toString)))), + VARCHAR + ), + Case( + "ts, d", + Seq(branch("ts")(r => opt(r.ts.map(utc))), branch("d")(r => opt(r.d.map(midnight)))), + TIMESTAMP + ), + Case( + "x, n", + Seq(branch("x")(r => opt(r.x)), branch("n")(r => opt(r.n.map(_.toDouble)))), + DOUBLE + ), + Case( + "CAST(n AS BIGINT), n", + Seq( + branch("CAST(n AS BIGINT)")(r => opt(r.n.map(_.toLong))), + branch("n")(r => opt(r.n.map(_.toLong))) + ), + BIGINT + ), + // a NULL literal takes the other branch's type + Case("n, NULL", Seq(branch("n")(r => opt(r.n)), branch("NULL")(_ => null)), INTEGER), + Case( + "n, NULL, x", + Seq( + branch("n")(r => opt(r.n.map(_.toDouble))), + branch("NULL")(_ => null), + branch("x")(r => opt(r.x)) + ), + DOUBLE + ), + // three branches: the widest type; a text branch makes the column text + Case( + "n, x, CAST(n AS BIGINT)", + Seq( + branch("n")(r => opt(r.n.map(_.toDouble))), + branch("x")(r => opt(r.x)), + branch("CAST(n AS BIGINT)")(r => opt(r.n.map(_.toDouble))) + ), + DOUBLE + ), + Case( + "d, ts, d2", + Seq( + branch("d")(r => opt(r.d.map(midnight))), + branch("ts")(r => opt(r.ts.map(utc))), + branch("d2")(r => opt(r.d2.map(midnight))) + ), + TIMESTAMP + ), + Case( + "d, n, '1'", + Seq( + branch("d")(r => opt(r.d.map(_.toString))), + branch("n")(r => opt(r.n.map(_.toString))), + branch("'1'")(_ => "1") + ), + VARCHAR + ), + Case( + "s, n, d", + Seq( + branch("s")(r => opt(r.s)), + branch("n")(r => opt(r.n.map(_.toString))), + branch("d")(r => opt(r.d.map(_.toString))) + ), + VARCHAR + ), + // a KEYWORD beside a TEXT column, and a TEXT column beside a number + Case("id, txt", Seq(branch("id")(_.id), branch("txt")(r => opt(r.txt))), VARCHAR), + Case( + "txt, n", + Seq(branch("txt")(r => opt(r.txt)), branch("n")(r => opt(r.n.map(_.toString)))), + VARCHAR + ), + // a BOOLEAN reads as 1 / 0 beside a number, and as `true` / `false` beside text + Case( + "b, n", + Seq(branch("b")(r => opt(r.b.map(v => if (v) 1 else 0))), branch("n")(r => opt(r.n))), + INTEGER + ), + Case( + "b, x", + Seq(branch("b")(r => opt(r.b.map(v => if (v) 1.0 else 0.0))), branch("x")(r => opt(r.x))), + DOUBLE + ), + Case( + "b, 'a'", + Seq(branch("b")(r => opt(r.b.map(_.toString))), branch("'a'")(_ => "a")), + VARCHAR + ), + // the numeric lattice + Case( + "n, f", + Seq(branch("n")(r => opt(r.n.map(_.toFloat))), branch("f")(r => opt(r.f))), + FLOAT + ), + Case( + "sm, bg", + Seq(branch("sm")(r => opt(r.sm.map(_.toLong))), branch("bg")(r => opt(r.bg))), + BIGINT + ), + Case( + "bg, x", + Seq(branch("bg")(r => opt(r.bg.map(_.toDouble))), branch("x")(r => opt(r.x))), + DOUBLE + ) + ) + + /** The refusals of rule 1 -- a date beside a number, the pair the relational engine fails on -- + * by name with core's message. + */ + private val refused: Seq[(String, String)] = Seq( + s"SELECT d AS k FROM $table UNION ALL SELECT n AS k FROM $table" -> + "column 1: branch 1 'k' is DATE, branch 2 'k' is INT", + s"SELECT ts AS k FROM $table UNION ALL SELECT n AS k FROM $table" -> + "column 1: branch 1 'k' is TIMESTAMP, branch 2 'k' is INT", + s"SELECT b AS k FROM $table UNION ALL SELECT d AS k FROM $table" -> + "column 1: branch 1 'k' is BOOLEAN, branch 2 'k' is DATE", + s"SELECT id, d FROM $table UNION ALL SELECT s, n FROM $table" -> + "column 2: branch 1 'd' is DATE, branch 2 'n' is INT", + // a TIME does not make the node text: no text, no VARBINARY + s"SELECT CAST(ts AS TIME) AS k FROM $textTable UNION ALL SELECT ts AS k FROM $textTable " + + s"UNION ALL SELECT x AS k FROM $textTable" -> + "column 1: branch 2 'k' is TIMESTAMP, branch 3 'k' is DOUBLE" + ) + + /** Elasticsearch wraps a `script_fields` value in a one-element array. */ + private def scalar(v: Any): Any = v match { + case s: Seq[_] if s.size == 1 => scalar(s.head) + case c: java.util.Collection[_] if c.size == 1 => scalar(c.iterator().next()) + case other => other + } + + /** The same VALUE (the class is asserted on its own): an instant for a timestamp, the number for + * a number. + */ + private def same(expected: Any, actual: Any): Boolean = + (expected, actual) match { + case (null, null) => true + case (null, _) | (_, null) => false + case (e: ZonedDateTime, a: ZonedDateTime) => e.toInstant == a.toInstant + case (e: java.lang.Number, a: java.lang.Number) => e.doubleValue() == a.doubleValue() + case (e: Array[Byte], a: Array[Byte]) => java.util.Arrays.equals(e, a) + case (e, a) => e == a + } + + /** A value as a failure message shows it: bytes in hexadecimal. */ + private def show(v: Any): String = v match { + case b: Array[Byte] => b.map(x => f"${x & 0xff}%02x").mkString("0x", "", "") + case other => String.valueOf(other) + } + + private def rowsOf(sql: String, route: String): Either[String, Seq[ListMap[String, Any]]] = + Try { + route match { + case "search" => + implicit val ctx: ConversionContext = NativeContext + client.search(SelectStatement(sql)) match { + case ElasticSuccess(response) => Right(response.results) + case ElasticFailure(error) => Left(error.message) + } + case _ => + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) + case ElasticSuccess(QueryStream(stream, _)) => + Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) + case ElasticSuccess(other) => Left(s"unexpected result $other") + case ElasticFailure(error) => Left(error.message) + } + } + }.toEither.left.map(t => s"${t.getClass.getSimpleName}: ${t.getMessage}").joinRight + + /** What differs from DuckDB's answer, if anything. */ + private def differences(c: Case, route: String, limit: Boolean): Seq[String] = + differencesOf(c.sql(limit), c.expected, c.column, route) + + private def differencesOf( + sql: String, + want: Seq[Any], + column: ColumnType, + route: String + ): Seq[String] = + rowsOf(sql, route) match { + case Left(error) => Seq(s"[$route] [$sql] failed: $error") + case Right(rows) => + val values = rows.map(r => r.getOrElse("k", null)) + val got = values.map(scalar) + // the values, as a multiset + var unmatched = got + val missing = want.filterNot { w => + unmatched.indexWhere(g => same(w, g)) match { + case -1 => false + case i => unmatched = unmatched.patch(i, Nil, 1); true + } + } + val valueErrors = + if (missing.isEmpty && unmatched.isEmpty) Nil + else + Seq( + s"values: missing ${missing.map(show).mkString(", ")} / unexpected " + + unmatched.map(show).mkString(", ") + ) + // ONE class: the column's + val classes = got.filter(_ != null).map(_.getClass).distinct + val classErrors = + if (classes.forall(_ == column.cls)) Nil + else + Seq( + s"classes ${classes.map(_.getSimpleName).mkString(", ")} instead of " + + column.cls.getSimpleName + ) + // what JDBC and Arrow report: the type of the column's first value + val reported = ValueCoercion.inferType(values.find(_ != null).orNull) + val reportErrors = + if (reported == column.reported) Nil + else Seq(s"reported $reported instead of ${column.reported}") + (valueErrors ++ classErrors ++ reportErrors).map(e => s"[$route] [$sql] $e") + } + + "A UNION ALL" should "answer DuckDB's column type, values and one class per column, on every route" in { + val wrong = for { + c <- cases + (route, limited) <- Seq(("run", false), ("search", false), ("search", true)) + error <- differences(c, route, limited) + } yield error + wrong shouldBe empty + } + + /** A `TIMESTAMP`, a `DOUBLE`, a `REAL` or a `TIME` beside text: a `VARCHAR` column, each value + * spelled as DuckDB 1.5.5 casts its own type to `VARCHAR` (the lead's ruling of 2026-10-06). + */ + private val textCases: Seq[(Seq[(String, TextDoc => Any)], String)] = { + val ts: TextDoc => Any = d => d.ts.map(_.text).orNull + val x: TextDoc => Any = d => d.x.map(_.text).orNull + val f: TextDoc => Any = d => d.f.map(_.text).orNull + val s: TextDoc => Any = d => d.s.orNull + Seq( + Seq("ts" -> ts, "s" -> s) -> "ts, s", + Seq("s" -> s, "ts" -> ts) -> "s, ts", + Seq("ts" -> ts, "'x'" -> ((_: TextDoc) => "x")) -> "ts, 'x'", + Seq("x" -> x, "s" -> s) -> "x, s", + Seq("'a'" -> ((_: TextDoc) => "a"), "x" -> x) -> "'a', x", + Seq("f" -> f, "s" -> s) -> "f, s", + Seq("CAST(ts AS TIME)" -> ((d: TextDoc) => d.time.orNull), "s" -> s) -> "TIME, s", + Seq("x" -> x, "ts" -> ts, "s" -> s) -> "x, ts, s", + Seq( + "CAST(ts AS DATE)" -> ((d: TextDoc) => d.date.orNull), + "ts" -> ts, + "'x'" -> ((_: TextDoc) => "x") + ) -> "DATE, ts, 'x'" + ) + } + + it should "spell a TIMESTAMP, a DOUBLE, a REAL or a TIME beside text as DuckDB does, on every route" in { + val wrong = for { + (branches, _) <- textCases + (route, limited) <- Seq(("run", false), ("search", false), ("search", true)) + sql = branches + .map { case (item, _) => + s"SELECT $item AS k FROM $textTable" + (if (limited) " LIMIT 100" else "") + } + .mkString(" UNION ALL ") + want = branches.flatMap { case (_, value) => textDocs.map(value) } + error <- differencesOf(sql, want, VARCHAR, route) + } yield error + wrong shouldBe empty + } + + private def utf8(s: String): Array[Byte] = s.getBytes(java.nio.charset.StandardCharsets.UTF_8) + + private val routes = Seq(("run", false), ("search", false), ("search", true)) + + private def textUnion(items: Seq[String], limited: Boolean): String = + items + .map(item => s"SELECT $item AS k FROM $textTable" + (if (limited) " LIMIT 100" else "")) + .mkString(" UNION ALL ") + + /** A `VARBINARY` beside text: DuckDB's `BLOB` (the lead's ruling of 2026-10-07). Every row is + * BYTES (`Array[Byte]`, core's `VARBINARY` cell): a `binary` field's base64 read as its UTF-8 + * bytes, as the relational engine and the JDBC driver read it, and a text value cast to bytes as + * DuckDB 1.5.5 casts a `VARCHAR` to a `BLOB` (MEASURED: an ASCII character is its byte). + */ + it should "answer a VARBINARY beside text as DuckDB's BLOB, on every route" in { + val bin: TextDoc => Any = d => binaryOf.get(d.id).map(utf8).orNull + val s: TextDoc => Any = d => d.s.map(utf8).orNull + val x: TextDoc => Any = _ => utf8("x") + val cases: Seq[Seq[(String, TextDoc => Any)]] = Seq( + Seq("bin" -> bin, "s" -> s), + Seq("s" -> s, "bin" -> bin), + Seq("bin" -> bin, "'x'" -> x), + Seq("bin" -> bin, "NULL" -> ((_: TextDoc) => null), "s" -> s) + ) + val wrong = for { + branches <- cases + (route, limited) <- routes + sql = textUnion(branches.map(_._1), limited) + want = branches.flatMap { case (_, value) => textDocs.map(value) } + error <- differencesOf(sql, want, VARBINARY, route) + } yield error + wrong shouldBe empty + } + + it should "fail a VARBINARY beside a text DuckDB cannot cast to bytes, as DuckDB fails it" in { + // `u` holds a character outside ASCII: DuckDB's cast refuses it, at run time + val wrong = routes.flatMap { case (route, limited) => + val sql = textUnion(Seq("bin", "u"), limited) + rowsOf(sql, route) match { + case Left(msg) if msg.contains("Invalid byte encountered in STRING -> BLOB conversion") => + None + case other => Some(s"[$route] [$sql] answered $other instead of failing the cast") + } + } + wrong shouldBe empty + } + + /** A value with SEVERAL values -- a multi-valued keyword -- in a column that converts its values + * fails the statement: the column holds one value of its type per row, and such a value has no + * one value to convert. The message is in DuckDB 1.5.5's words for the same column (MEASURED, + * each shape below). The CONTROL: the same value in a column whose branches have one type, which + * converts nothing, answers. + */ + it should "fail a multi-valued value in a column that converts, on every route" in { + def failure(other: String): String = + s"Conversion Error: Unimplemented type for cast ($other -> VARCHAR[]) when casting from " + + "source column k" + def limitedIf(limited: Boolean, statement: String): String = + if (limited) statement.replace(" UNION ALL ", " LIMIT 100 UNION ALL ") + " LIMIT 100" + else statement + def union(left: String, right: String, rightTable: String = textTable): String = + s"SELECT $left AS k FROM $textTable UNION ALL SELECT $right AS k FROM $rightTable" + val shapes: Seq[(String, String)] = Seq( + union("tags", "n", table) -> failure("INTEGER"), + union("tags", "ts") -> failure("TIMESTAMP"), + union("tags", "bin") -> failure("BLOB"), + union("x", "tags") -> failure("DOUBLE") + ) + val wrong = for { + (statement, message) <- shapes + (route, limited) <- routes + sql = limitedIf(limited, statement) + error <- rowsOf(sql, route) match { + case Left(msg) if msg.contains(message) => None + case other => Some(s"[$route] [$sql] answered $other instead of failing with $message") + } + } yield error + // the control: `tags` beside another KEYWORD, every row answered, the two values of t1 as the + // branch reads them + def bothTags(row: ListMap[String, Any]): Boolean = row.getOrElse("k", null) match { + case s: Seq[_] => s.map(e => String.valueOf(e)) == Seq("a", "b") + case l: java.util.List[_] => l.size == 2 && l.get(0) == "a" && l.get(1) == "b" + case _ => false + } + val unconverted = routes.flatMap { case (route, limited) => + val sql = limitedIf(limited, union("tags", "kw")) + rowsOf(sql, route) match { + case Right(rows) if rows.size == 2 * textDocs.size && rows.exists(bothTags) => None + case other => Some(s"[$route] [$sql] answered $other instead of every row, unconverted") + } + } + (wrong ++ unconverted) shouldBe empty + } + + /** A keyword core re-reads as a temporal when the response is parsed (issue #418) is spelled back + * with its seconds -- `10:30:00`, `2024-01-31T10:30:00Z` -- its own text, which DuckDB answers + * since it reads the keyword as text. + */ + it should "spell a keyword that looks like a time or a timestamp as its text, on every route" in { + val kw: TextDoc => Any = d => temporalKeywordOf.get(d.id).orNull + val textOfN: Doc => Any = r => opt(r.n.map(_.toString)) + val bytesOfKw: TextDoc => Any = d => temporalKeywordOf.get(d.id).map(utf8).orNull + val bin: TextDoc => Any = d => binaryOf.get(d.id).map(utf8).orNull + val wrong = routes.flatMap { case (route, limited) => + val limit = if (limited) " LIMIT 100" else "" + val asText = + s"SELECT kw AS k FROM $textTable$limit UNION ALL SELECT n AS k FROM $table$limit" + val asBytes = + s"SELECT kw AS k FROM $textTable$limit UNION ALL SELECT bin AS k FROM $textTable$limit" + differencesOf(asText, textDocs.map(kw) ++ docs.map(textOfN), VARCHAR, route) ++ + differencesOf(asBytes, textDocs.map(bytesOfKw) ++ textDocs.map(bin), VARBINARY, route) + } + wrong shouldBe empty + } + + /** Where DuckDB gives the column no type and rule 1 does not refuse it (the lead's rule 2 of + * 2026-10-07): a `VARCHAR` column, every value spelled as text -- a `binary` field its base64, + * the text Elasticsearch holds; a `TIME`, a `DOUBLE`, a `TIMESTAMP` and a `DATE` as DuckDB + * spells them in a `VARCHAR` column. + */ + it should "answer VARCHAR where DuckDB gives the column no type, on every route" in { + val bin: TextDoc => Any = d => binaryOf.get(d.id).orNull + val x: TextDoc => Any = d => d.x.map(_.text).orNull + val ts: TextDoc => Any = d => d.ts.map(_.text).orNull + val s: TextDoc => Any = d => d.s.orNull + val time: TextDoc => Any = d => d.time.orNull + val date: TextDoc => Any = d => d.date.orNull + val cases: Seq[Seq[(String, TextDoc => Any)]] = Seq( + // a VARBINARY beside a number, a temporal: DuckDB has no cast to BLOB + Seq("bin" -> bin, "x" -> x), + Seq("bin" -> bin, "s" -> s, "ts" -> ts), + Seq("bin" -> bin, "CAST(ts AS TIME)" -> time), + // ... beside a TIMESTAMP AND a number: text to the relational engine, so never refused + Seq("bin" -> bin, "ts" -> ts, "x" -> x), + Seq("x" -> x, "bin" -> bin, "ts" -> ts), + Seq("ts" -> ts, "x" -> x, "bin" -> bin), + // a TIME beside a TIMESTAMP, a DATE, a number + Seq("CAST(ts AS TIME)" -> time, "ts" -> ts), + Seq("CAST(ts AS TIME)" -> time, "CAST(ts AS DATE)" -> date), + Seq("x" -> x, "CAST(ts AS TIME)" -> time) + ) + val wrong = for { + branches <- cases + (route, limited) <- routes + sql = textUnion(branches.map(_._1), limited) + want = branches.flatMap { case (_, value) => textDocs.map(value) } + error <- differencesOf(sql, want, VARCHAR, route) + } yield error + wrong shouldBe empty + } + + /** A `GEO_POINT`, a `STRUCT` or an `ARRAY` (here a `nested` field) beside a type of another + * family is refused, text included -- the relational engine casts nothing to a list or a struct + * (MEASURED through arrow) -- and a list of objects beside itself answered as each branch reads + * it. + */ + it should "refuse an object or a list beside another type, answer it beside itself" in { + def limitedIf(limited: Boolean, statement: String): String = + if (limited) statement.replace(" UNION ALL ", " LIMIT 100 UNION ALL ") + " LIMIT 100" + else statement + def union(left: String, right: String): String = + s"SELECT $left AS k FROM $textTable UNION ALL SELECT $right AS k FROM $textTable" + val refusedShapes: Seq[(String, String)] = Seq( + union("geo", "s") -> "column 1: branch 1 'k' is GEO_POINT, branch 2 'k' is KEYWORD", + union("x", "geo") -> "column 1: branch 1 'k' is DOUBLE, branch 2 'k' is GEO_POINT", + union("geo", "bin") -> "column 1: branch 1 'k' is GEO_POINT, branch 2 'k' is VARBINARY", + union("items", "s") -> "column 1: branch 1 'k' is ARRAY, branch 2 'k' is KEYWORD", + union("ts", "items") -> "column 1: branch 1 'k' is TIMESTAMP, branch 2 'k' is ARRAY", + // an object beside text, a number, a date or bytes: the relational engine fails on each + union("obj", "s") -> "column 1: branch 1 'k' is STRUCT, branch 2 'k' is KEYWORD", + union("x", "obj") -> "column 1: branch 1 'k' is DOUBLE, branch 2 'k' is STRUCT", + union("obj", "ts") -> "column 1: branch 1 'k' is STRUCT, branch 2 'k' is TIMESTAMP", + union("bin", "obj") -> "column 1: branch 1 'k' is VARBINARY, branch 2 'k' is STRUCT" + ) + val notRefused = for { + (statement, reason) <- refusedShapes + (route, limited) <- routes + sql = limitedIf(limited, statement) + error <- rowsOf(sql, route) match { + case Left(msg) + if msg.contains(s"Set operation branches must project compatible types at $reason") => + None + case other => Some(s"[$route] [$sql] answered $other instead of refusing at $reason") + } + } yield error + // beside its own type: every row -- a list of objects as the branch reads it + val sameType = routes.flatMap { case (route, limited) => + (rowsOf(limitedIf(limited, union("items", "items")), route) match { + case Right(rows) if rows.size == 2 * textDocs.size => Nil + case other => Seq(s"[$route] items beside items answered $other instead of every row") + }) + } + (notRefused ++ sameType) shouldBe empty + } + + /** What a STRUCT column holds, compared as DuckDB's: each map's fields IN ORDER, every value with + * its class. + */ + private def structDifferencesOf(sql: String, want: Seq[Any], route: String): Seq[String] = { + def norm(v: Any): Any = v match { + case m: scala.collection.Map[_, _] => + m.toSeq.map { case (k, x) => String.valueOf(k) -> norm(x) } + case l: Seq[_] => l.map(norm) + case other => shown(other) + } + rowsOf(sql, route) match { + case Left(error) => Seq(s"[$route] [$sql] failed: $error") + case Right(rows) => + val got = rows.map(r => norm(r.getOrElse("k", null))) + var unmatched = got + val missing = want.map(norm).filterNot { w => + unmatched.indexOf(w) match { + case -1 => false + case i => unmatched = unmatched.patch(i, Nil, 1); true + } + } + if (missing.isEmpty && unmatched.isEmpty) Nil + else + Seq( + s"[$route] [$sql] values: missing ${missing.mkString(", ")} / unexpected ${unmatched.mkString(", ")}" + ) + } + } + + /** A `GEO_POINT` beside a `STRUCT`: DuckDB 1.5.5's merged struct (the lead's ruling of + * 2026-10-07), MEASURED -- every field of every branch in the order it first appears, a field a + * branch does not have NULL -- and every point a `{lat, lon}`, whether Elasticsearch stored it + * as an object, a `"lat,lon"` text or an array `[lon, lat]`; two points stored differently, both + * `{lat, lon}`. + */ + it should "answer a GEO_POINT beside a STRUCT as DuckDB's merged STRUCT, on every route" in { + def merged(fields: Seq[String])(values: Map[String, Any]): Any = + ListMap(fields.map(f => f -> values.getOrElse(f, null)): _*) + val pointFirst = merged(Seq("lat", "lon", "a", "b")) _ + val objectFirst = merged(Seq("a", "b", "lat", "lon")) _ + def points(shape: Map[String, Any] => Any): Seq[Any] = + textDocs.map(d => + pointOf.get(d.id).map { case (lat, lon) => shape(Map("lat" -> lat, "lon" -> lon)) }.orNull + ) + def objects(shape: Map[String, Any] => Any): Seq[Any] = + textDocs.map(d => + objOf.get(d.id).map { case (a, b) => shape(Map("a" -> a, "b" -> b)) }.orNull + ) + val point = merged(Seq("lat", "lon")) _ + val cases: Seq[(Seq[String], Seq[Any])] = Seq( + Seq("geoo", "obj") -> (points(pointFirst) ++ objects(pointFirst)), + Seq("geo", "obj") -> (points(pointFirst) ++ objects(pointFirst)), + Seq("geoa", "obj") -> (points(pointFirst) ++ objects(pointFirst)), + Seq("obj", "geoo") -> (objects(objectFirst) ++ points(objectFirst)), + Seq("obj", "geo") -> (objects(objectFirst) ++ points(objectFirst)), + // two points stored differently: one GEO_POINT, every point {lat, lon} + Seq("geo", "geoa") -> (points(point) ++ points(point)), + Seq("geoo", "geo", "geoa") -> (points(point) ++ points(point) ++ points(point)) + ) + val wrong = for { + (branches, want) <- cases + (route, limited) <- routes + error <- structDifferencesOf(textUnion(branches, limited), want, route) + } yield error + wrong shouldBe empty + } + + /** A field several objects have is typed over every branch at once, as DuckDB 1.5.5 types it + * (MEASURED, every order): an `INT`, a `DATE` and a `KEYWORD` field `a` make a `VARCHAR` field, + * `{a: '1'}`, `{a: '2024-01-31'}`, `{a: 'x'}`, whatever the order of the branches -- although an + * `INT` beside a `DATE` alone fails. + */ + it should "type a field the objects share over every branch at once, on every route" in { + val want: Seq[Any] = + Seq("oi" -> "1", "od" -> "2024-01-31", "ov" -> "x").flatMap { case (column, a) => + textDocs.map(d => if (fieldOf(column).contains(d.id)) ListMap("a" -> a) else null) + } + val wrong = for { + order <- Seq("oi", "od", "ov").permutations.toSeq + (route, limited) <- routes + error <- structDifferencesOf(textUnion(order, limited), want, route) + } yield error + wrong shouldBe empty + rowsOf(textUnion(Seq("oi", "od"), limited = false), "run") match { + case Left(msg) => + msg should include( + "Set operation branches must project compatible types at column 1: branch 1 'k' field " + + "'a' is INT, branch 2 'k' field 'a' is DATE" + ) + case other => fail(s"an INT beside a DATE answered $other") + } + } + + /** A value and its class -- a timestamp by its instant, bytes by their content. */ + private def shown(v: Any): (Any, String) = + v match { + case null => (null, "null") + case z: ZonedDateTime => (z.toInstant, "ZonedDateTime") + case b: Array[Byte] => (b.toSeq, "bytes") + case other => (other, other.getClass.getSimpleName) + } + + /** The column rule's type for two of the generated types -- the sql spec's table + * (`SetOperationTypeRuleSpec`), the same at every depth -- or REFUSED. + */ + private val pairType: Map[Set[String], String] = Map( + Set("INT", "BIGINT") -> "BIGINT", + Set("INT", "DOUBLE") -> "DOUBLE", + Set("INT", "BOOLEAN") -> "INT", + Set("INT", "DATE") -> "REFUSED", + Set("INT", "TIMESTAMP") -> "REFUSED", + Set("INT", "VARBINARY") -> "VARCHAR", + Set("INT", "VARCHAR") -> "VARCHAR", + Set("BIGINT", "DOUBLE") -> "DOUBLE", + Set("BIGINT", "BOOLEAN") -> "BIGINT", + Set("BIGINT", "DATE") -> "REFUSED", + Set("BIGINT", "TIMESTAMP") -> "REFUSED", + Set("BIGINT", "VARBINARY") -> "VARCHAR", + Set("BIGINT", "VARCHAR") -> "VARCHAR", + Set("DOUBLE", "BOOLEAN") -> "DOUBLE", + Set("DOUBLE", "DATE") -> "REFUSED", + Set("DOUBLE", "TIMESTAMP") -> "REFUSED", + Set("DOUBLE", "VARBINARY") -> "VARCHAR", + Set("DOUBLE", "VARCHAR") -> "VARCHAR", + Set("BOOLEAN", "DATE") -> "REFUSED", + Set("BOOLEAN", "TIMESTAMP") -> "REFUSED", + Set("BOOLEAN", "VARBINARY") -> "VARCHAR", + Set("BOOLEAN", "VARCHAR") -> "VARCHAR", + Set("DATE", "TIMESTAMP") -> "TIMESTAMP", + Set("DATE", "VARBINARY") -> "VARCHAR", + Set("DATE", "VARCHAR") -> "VARCHAR", + Set("TIMESTAMP", "VARBINARY") -> "VARCHAR", + Set("TIMESTAMP", "VARCHAR") -> "VARCHAR", + Set("VARBINARY", "VARCHAR") -> "VARBINARY" + ) + + /** What each stored value becomes at a converting position of each type: its value AND its class, + * written here -- DuckDB's casts and spellings, MEASURED in the rounds before. + */ + private val convertedOf: Map[(String, String), Map[String, Any]] = { + val at1030 = ZonedDateTime.parse("2024-01-31T10:30:00Z") + def midnight(day: String) = ZonedDateTime.parse(s"${day}T00:00:00Z") + // a whole number at a BIGINT, DOUBLE, VARCHAR (and INT) position + def whole(n: Long, withInt: Boolean = false): Map[String, Any] = + Map[String, Any]( + "BIGINT" -> java.lang.Long.valueOf(n), + "DOUBLE" -> java.lang.Double.valueOf(n.toDouble), + "VARCHAR" -> n.toString + ) ++ (if (withInt) Map("INT" -> Integer.valueOf(n.toInt)) else Map.empty) + def truth(b: Boolean): Map[String, Any] = { + val n = if (b) 1 else 0 + Map( + "INT" -> Integer.valueOf(n), + "BIGINT" -> java.lang.Long.valueOf(n.toLong), + "DOUBLE" -> java.lang.Double.valueOf(n.toDouble), + "VARCHAR" -> b.toString + ) + } + Map( + ("INT", "t1") -> Map( + "BIGINT" -> java.lang.Long.valueOf(1L), + "DOUBLE" -> java.lang.Double.valueOf(1.0), + "INT" -> Integer.valueOf(1), + "VARCHAR" -> "1" + ), + ("INT", "t2") -> Map( + "BIGINT" -> java.lang.Long.valueOf(-2L), + "DOUBLE" -> java.lang.Double.valueOf(-2.0), + "INT" -> Integer.valueOf(-2), + "VARCHAR" -> "-2" + ), + ("BIGINT", "t1") -> Map( + "BIGINT" -> java.lang.Long.valueOf(1L), + "DOUBLE" -> java.lang.Double.valueOf(1.0), + "VARCHAR" -> "1" + ), + ("BIGINT", "t2") -> Map( + "BIGINT" -> java.lang.Long.valueOf(3000000000L), + "DOUBLE" -> java.lang.Double.valueOf(3.0e9), + "VARCHAR" -> "3000000000" + ), + ("INT", "t3") -> whole(5L, withInt = true), + ("INT", "t4") -> whole(5L, withInt = true), + ("INT", "t5") -> whole(5L, withInt = true), + ("BIGINT", "t3") -> whole(5L), + ("BIGINT", "t4") -> whole(5L), + ("BIGINT", "t5") -> whole(500L), + ("BIGINT", "t6") -> whole(5L), + ("BIGINT", "t7") -> whole(-5L), + ("DOUBLE", "t1") -> Map("DOUBLE" -> java.lang.Double.valueOf(2.0), "VARCHAR" -> "2.0"), + ("DOUBLE", "t2") -> Map("DOUBLE" -> java.lang.Double.valueOf(2.5), "VARCHAR" -> "2.5"), + ("DOUBLE", "t3") -> Map("DOUBLE" -> java.lang.Double.valueOf(2.5), "VARCHAR" -> "2.5"), + ("DOUBLE", "t4") -> Map("DOUBLE" -> java.lang.Double.valueOf(1000.0), "VARCHAR" -> "1000.0"), + ("DOUBLE", "t5") -> Map("DOUBLE" -> java.lang.Double.valueOf(2.5), "VARCHAR" -> "2.5"), + ("DOUBLE", "t6") -> Map("DOUBLE" -> java.lang.Double.valueOf(5.0), "VARCHAR" -> "5.0"), + ("BOOLEAN", "t3") -> truth(true), + ("BOOLEAN", "t4") -> truth(false), + ("BOOLEAN", "t5") -> truth(false), + ("DATE", "t3") -> Map("TIMESTAMP" -> midnight("2024-01-31"), "VARCHAR" -> "2024-01-31"), + ("DATE", "t4") -> Map("TIMESTAMP" -> midnight("2024-01-31"), "VARCHAR" -> "2024-01-31"), + ("DATE", "t5") -> Map("TIMESTAMP" -> midnight("2024-01-31"), "VARCHAR" -> "2024-01-31"), + ("TIMESTAMP", "t6") -> Map("TIMESTAMP" -> at1030, "VARCHAR" -> "2024-01-31 10:30:00"), + ("BOOLEAN", "t1") -> Map( + "INT" -> Integer.valueOf(1), + "BIGINT" -> java.lang.Long.valueOf(1L), + "DOUBLE" -> java.lang.Double.valueOf(1.0), + "VARCHAR" -> "true" + ), + ("BOOLEAN", "t2") -> Map( + "INT" -> Integer.valueOf(0), + "BIGINT" -> java.lang.Long.valueOf(0L), + "DOUBLE" -> java.lang.Double.valueOf(0.0), + "VARCHAR" -> "false" + ), + ("DATE", "t1") -> Map("TIMESTAMP" -> midnight("2024-01-31"), "VARCHAR" -> "2024-01-31"), + ("DATE", "t2") -> Map("TIMESTAMP" -> midnight("2023-12-25"), "VARCHAR" -> "2023-12-25"), + ("TIMESTAMP", "t1") -> + Map("TIMESTAMP" -> midnight("2024-01-31"), "VARCHAR" -> "2024-01-31 00:00:00"), + ("TIMESTAMP", "t2") -> Map("TIMESTAMP" -> at1030, "VARCHAR" -> "2024-01-31 10:30:00"), + ("TIMESTAMP", "t3") -> Map("TIMESTAMP" -> at1030, "VARCHAR" -> "2024-01-31 10:30:00"), + ("TIMESTAMP", "t4") -> Map("TIMESTAMP" -> at1030, "VARCHAR" -> "2024-01-31 10:30:00"), + ("TIMESTAMP", "t5") -> Map("TIMESTAMP" -> at1030, "VARCHAR" -> "2024-01-31 10:30:00"), + ("VARBINARY", "t1") -> Map("VARBINARY" -> utf8("YWI="), "VARCHAR" -> "YWI="), + ("VARBINARY", "t2") -> Map("VARBINARY" -> utf8("eA=="), "VARCHAR" -> "eA=="), + ("VARCHAR", "t1") -> Map("VARBINARY" -> utf8("x"), "VARCHAR" -> "x"), + ("VARCHAR", "t2") -> Map("VARBINARY" -> utf8("y"), "VARCHAR" -> "y") + ) + } + + /** ONE class per converting position, at every depth -- a column, an object's field, the field of + * a list of objects' element, the field of an object inside an object -- on every route: every + * pair of two different types, in both orders; every value checked with its class, in every form + * core reads it in. A date beside a number is refused, the field named by its path. + */ + it should "hold one class at every converting position, at every depth, on every route" in { + val depths = Seq("column" -> "c", "field" -> "f", "element" -> "n", "nested" -> "s") + def at(depth: String, k: Any): Seq[Any] = + (depth, k) match { + case (_, null) => Seq(null) + case ("column", v) => Seq(v) + case ("field", m: scala.collection.Map[_, _]) => + Seq(m.asInstanceOf[scala.collection.Map[String, Any]].getOrElse("a", null)) + case ("element", l: Seq[_]) => + l.map { + case m: scala.collection.Map[_, _] => + m.asInstanceOf[scala.collection.Map[String, Any]].getOrElse("a", null) + case other => ("not an object", other) + } + case ("nested", m: scala.collection.Map[_, _]) => + m.asInstanceOf[scala.collection.Map[String, Any]].get("p") match { + case Some(p: scala.collection.Map[_, _]) => + Seq(p.asInstanceOf[scala.collection.Map[String, Any]].getOrElse("c", null)) + case other => Seq(("no p", other)) + } + case (_, other) => Seq(("unexpected", other)) + } + val pairs = generatedTypes.map(_._1).combinations(2).flatMap(_.permutations).toSeq + pairs should have size 56 + val wrong = for { + (depth, prefix) <- depths + pair <- pairs + (route, limited) <- routes + sql = textUnion(pair.map(t => s"${prefix}_$t"), limited) + error <- (pairType(pair.toSet), rowsOf(sql, route)) match { + case ("REFUSED", Left(msg)) => + val named = depth match { + case "column" => s"branch 1 'k' is " + case "nested" => "field 'p.c' is " + case _ => "field 'a' is " + } + if (msg.contains("must project compatible types") && msg.contains(named)) None + else Some(s"[$route] [$sql] refused as $msg") + case ("REFUSED", other) => Some(s"[$route] [$sql] answered $other instead of refusing") + case (_, Left(error)) => Some(s"[$route] [$sql] failed: $error") + case (target, Right(rows)) => + val got = rows.flatMap(r => at(depth, r.getOrElse("k", null))).map(shown) + val want = pair.flatMap(t => + textDocs.map(d => + if (storedOf(t).contains(d.id)) shown(convertedOf((t, d.id))(target)) else shown(null) + ) + ) + var unmatched = got + val missing = want.filterNot { w => + unmatched.indexOf(w) match { + case -1 => false + case i => unmatched = unmatched.patch(i, Nil, 1); true + } + } + if (missing.isEmpty && unmatched.isEmpty) None + else + Some( + s"[$route] [$sql] as $target: missing ${missing.mkString(", ")} / unexpected " + + unmatched.mkString(", ") + ) + } + } yield error + wrong shouldBe empty + } + + private lazy val esMajor: Int = + client.version match { + case ElasticSuccess(v) => v.takeWhile(_ != '.').toInt + case ElasticFailure(_) => 0 + } + + /** DuckDB's spelling of a `TIMESTAMP`, in UTC: `2024-01-31 10:30:00`, `2024-01-31 10:30:00.123`. + */ + private def duckTimestamp(i: Instant): String = { + val l = java.time.LocalDateTime.ofInstant(i, ZoneOffset.UTC) + val base = f"${l.toLocalDate} ${l.getHour}%02d:${l.getMinute}%02d:${l.getSecond}%02d" + if (l.getNano == 0) base + else base + "." + f"${l.getNano}%09d".reverse.dropWhile(_ == '0').reverse + } + + /** A date field's value is read with ITS mapping format, as the Elasticsearch major the statement + * runs on reads it -- seconds, custom patterns, `||` lists, a DATE's day, the default format's + * years -- at every converting position: a column and an object's field, beside a type that + * makes it convert to a `TIMESTAMP` and to a `VARCHAR`, on every route; every value checked with + * its class. + */ + it should "read a date field with its mapping format, as the major reads it, at every depth, on every route" in { + def depthValue(depth: String, k: Any): Any = + (depth, k) match { + case (_, null) => null + case ("column", v) => v + case ("field", m: scala.collection.Map[_, _]) => + m.asInstanceOf[scala.collection.Map[String, Any]].getOrElse("a", null) + case (_, other) => ("unexpected", other) + } + val wrong = for { + f <- formatted + (partner, target) <- (if (f.declared == "DATE") Seq("pts" -> "TIMESTAMP") + else Seq("pd" -> "TIMESTAMP")) :+ ("s" -> "VARCHAR") + (depth, item, other) <- Seq( + ("column", s"c_${f.key}", partner), + ("field", s"f_${f.key}", "f" + partner.stripPrefix("p")) + ) + (route, limited) <- routes + sql = Seq(item, other) + .map(i => s"SELECT $i AS k FROM $formatTable" + (if (limited) " LIMIT 100" else "")) + .mkString(" UNION ALL ") + error <- rowsOf(sql, route) match { + case Left(e) => Some(s"[$route] [$sql] failed: $e") + case Right(rows) => + def converted(at: Instant): Any = { + val read = + if (f.declared == "DATE") + at.atZone(ZoneOffset.UTC).toLocalDate.atStartOfDay.toInstant(ZoneOffset.UTC) + else at + target match { + case "TIMESTAMP" => read.atZone(ZoneOffset.UTC) + case _ => + if (f.declared == "DATE") read.atZone(ZoneOffset.UTC).toLocalDate.toString + else duckTimestamp(read) + } + } + val partnerValue: Any = (partner, target) match { + case (_, "VARCHAR") => "x" + case _ => ZonedDateTime.parse("2024-02-01T00:00:00Z") + } + val want = formatDocs.map(id => + f.values.get(id).fold[Any](null) { case (_, at, byMajor) => + converted(byMajor.getOrElse(esMajor, at)) + } + ) ++ formatDocs.map(id => if (id == "p1") partnerValue else null) + val got = rows.map(r => depthValue(depth, r.getOrElse("k", null))) + var unmatched = got.map(shown) + val missing = want.map(shown).filterNot { w => + unmatched.indexOf(w) match { + case -1 => false + case i => unmatched = unmatched.patch(i, Nil, 1); true + } + } + if (missing.isEmpty && unmatched.isEmpty) None + else + Some( + s"[$route] [$sql] as $target: missing ${missing.mkString(", ")} / unexpected " + + unmatched.mkString(", ") + ) + } + } yield error + wrong shouldBe empty + } + + it should "refuse by name only a date beside a number, before anything runs" in { + val wrong = refused.flatMap { case (sql, reason) => + rowsOf(sql, "run") match { + case Left(msg) + if msg.contains(s"Set operation branches must project compatible types at $reason") => + None + case other => Some(s"[$sql] answered $other instead of refusing at $reason") + } + } + wrong shouldBe empty + } +}