feat: Joom 1.1-1 — Delta scan, correctness fixes, wide-row performance, cost-based engine choice

.github/workflows/pr_build_linux.yml

@@ -515,11 +515,13 @@ jobs:
               org.apache.spark.sql.comet.execution.shuffle.CometDiskBlockWriterSuite
               org.apache.comet.exec.CometShuffleEncryptionSuite
               org.apache.comet.exec.CometShuffleManagerSuite
+              org.apache.comet.exec.CometShuffleReadCoalesceSuite
               org.apache.comet.exec.CometAsyncShuffleSuite
               org.apache.comet.exec.DisableAQECometShuffleSuite
               org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
               org.apache.spark.shuffle.comet.CometUnboundedShuffleMemoryAllocatorSuite
               org.apache.spark.shuffle.sort.SpillSorterSuite
+              org.apache.spark.shuffle.sort.CometShuffleExternalSorterSpillSuite
           - name: "exec"
             value: |
               org.apache.comet.exec.CometAggregateSuite
@@ -528,8 +530,11 @@ jobs:
               org.apache.comet.exec.CometEmptyRelationExecSuite
               org.apache.comet.exec.CometInMemoryCacheSuite
               org.apache.comet.exec.CometInMemoryCacheKryoSuite
+              org.apache.spark.sql.comet.CometBroadcastKryoPayloadSuite
+              org.apache.spark.sql.comet.CometBroadcastDefaultKryoSuite
               org.apache.comet.exec.CometGenerateExecSuite
               org.apache.comet.exec.CometWindowExecSuite
+              org.apache.comet.exec.CometPartitionAggregateWindowSuite
               org.apache.comet.exec.CometJoinSuite
               org.apache.spark.sql.comet.CometMapInBatchSuite
               org.apache.spark.sql.execution.python.CometArrowPythonRunnerSuite
@@ -557,12 +562,17 @@ jobs:
               org.apache.spark.sql.comet.CometPlanEqualitySuite
               org.apache.comet.rules.CometEmptyRelationExecRuleSuite
               org.apache.comet.rules.RevertNativeForTransitionHeavyStagesSuite
+              org.apache.comet.rules.ChooseBoundaryFormatsSuite
+              org.apache.comet.rules.CostBasedEngineChoiceSuite
+              org.apache.comet.rules.WideRowSortFallbackSuite
+              org.apache.comet.rules.WideRowShuffleFallbackSuite
               org.apache.spark.sql.CometTPCDSQuerySuite
               org.apache.spark.sql.CometTPCDSQueryTestSuite
               org.apache.spark.sql.CometTPCHQuerySuite
               org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite
               org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite
               org.apache.spark.sql.comet.CometTaskMetricsSuite
+              org.apache.spark.sql.comet.CometBatchRowProjectionSuite
               org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite
               org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite
               org.apache.spark.sql.comet.PlanDataInjectorSuite

.github/workflows/pr_build_macos.yml

@@ -163,11 +163,13 @@ jobs:
               org.apache.spark.sql.comet.execution.shuffle.CometDiskBlockWriterSuite
               org.apache.comet.exec.CometShuffleEncryptionSuite
               org.apache.comet.exec.CometShuffleManagerSuite
+              org.apache.comet.exec.CometShuffleReadCoalesceSuite
               org.apache.comet.exec.CometAsyncShuffleSuite
               org.apache.comet.exec.DisableAQECometShuffleSuite
               org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
               org.apache.spark.shuffle.comet.CometUnboundedShuffleMemoryAllocatorSuite
               org.apache.spark.shuffle.sort.SpillSorterSuite
+              org.apache.spark.shuffle.sort.CometShuffleExternalSorterSpillSuite
           - name: "exec"
             value: |
               org.apache.comet.exec.CometAggregateSuite
@@ -176,8 +178,11 @@ jobs:
               org.apache.comet.exec.CometEmptyRelationExecSuite
               org.apache.comet.exec.CometInMemoryCacheSuite
               org.apache.comet.exec.CometInMemoryCacheKryoSuite
+              org.apache.spark.sql.comet.CometBroadcastKryoPayloadSuite
+              org.apache.spark.sql.comet.CometBroadcastDefaultKryoSuite
               org.apache.comet.exec.CometGenerateExecSuite
               org.apache.comet.exec.CometWindowExecSuite
+              org.apache.comet.exec.CometPartitionAggregateWindowSuite
               org.apache.comet.exec.CometJoinSuite
               org.apache.spark.sql.comet.CometMapInBatchSuite
               org.apache.spark.sql.execution.python.CometArrowPythonRunnerSuite
@@ -205,12 +210,17 @@ jobs:
               org.apache.spark.sql.comet.CometPlanEqualitySuite
               org.apache.comet.rules.CometEmptyRelationExecRuleSuite
               org.apache.comet.rules.RevertNativeForTransitionHeavyStagesSuite
+              org.apache.comet.rules.ChooseBoundaryFormatsSuite
+              org.apache.comet.rules.CostBasedEngineChoiceSuite
+              org.apache.comet.rules.WideRowSortFallbackSuite
+              org.apache.comet.rules.WideRowShuffleFallbackSuite
               org.apache.spark.sql.CometTPCDSQuerySuite
               org.apache.spark.sql.CometTPCDSQueryTestSuite
               org.apache.spark.sql.CometTPCHQuerySuite
               org.apache.spark.sql.comet.CometTPCDSV1_4_PlanStabilitySuite
               org.apache.spark.sql.comet.CometTPCDSV2_7_PlanStabilitySuite
               org.apache.spark.sql.comet.CometTaskMetricsSuite
+              org.apache.spark.sql.comet.CometBatchRowProjectionSuite
               org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite
               org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite
               org.apache.spark.sql.comet.PlanDataInjectorSuite

contrib/delta-spark/README.md

@@ -0,0 +1,70 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Comet Delta Lake Contrib (experimental)
+
+Native Delta Lake reads for Comet. Delta tables are scanned through Comet's
+existing native Parquet reader, so they get row-group pruning, page-index
+pruning, and filter pushdown, with deletion vectors applied inside the scan.
+
+Support is experimental and explicitly opt-in. Two things are required:
+
+1. This module's jar (`comet-contrib-delta-spark`) on the classpath, alongside
+   `delta-spark`. It is never bundled into `comet-spark`; without it, Comet
+   has no Delta surface at all.
+2. `spark.comet.scan.delta.enabled=true`. The default is `false`, so the jar
+   alone does nothing.
+
+Unsupported tables and features fall back to Spark's reader. See the
+[user guide](https://datafusion.apache.org/comet/user-guide/latest/delta.html)
+for configuration details.
+
+## Supported versions
+
+| Spark | Delta          | Status                                        |
+| ----- | -------------- | --------------------------------------------- |
+| 3.5   | 3.3.x          | supported                                     |
+| 4.0   | 4.0.x          | supported                                     |
+| 4.1   | 4.3.x          | supported                                     |
+| 3.4   | delta-core 2.4 | not supported (older Delta, would need shims) |
+| 4.2   | none released  | inert until Delta ships a Spark 4.2 release   |
+
+## Building and testing
+
+The module builds under the `delta` Maven profile. It resolves `comet-spark`
+from the local Maven repository, so install `common` and `spark` from the same
+checkout immediately before, as CI does, with the `delta` profile active so that
+install produces the spark test-jar the contrib suites depend on; a stale sibling
+install is the trap the contributor guide warns about:
+
+```shell
+./mvnw -Pspark-3.5,delta install -pl common,spark -DskipTests
+./mvnw -Pspark-3.5,delta install -pl contrib/delta-spark
+```
+
+Run the test suites the same way (`test` instead of `install` on the second
+line). CI runs them via `.github/workflows/delta_contrib_test.yml`: on Spark
+3.5 in the merge queue, and on 4.0 and 4.1 nightly or on a pull request
+carrying the `run-delta-tests` label.
+`CometDeltaS3Suite` starts MinIO through Testcontainers and cancels itself when
+no Docker daemon is reachable; setting `COMET_DELTA_S3_REQUIRED=1`, as the
+`contrib-delta-s3` CI job does, turns that cancel into a suite failure.
+
+`dev/` contains a benchmark script (`bench_delta_comet.py`) and a harness for
+running Delta's own test suites against Comet (`run-delta-regression.sh`).