Skip to content

test: accept CometHashAggregateExec in the Spark 4.0 CollationSuite hash agg check #6220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andygrove merged 1 commit into from
Sep 25, 2026
Merged

test: accept CometHashAggregateExec in the Spark 4.0 CollationSuite hash agg check #6220

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions dev/diffs/4.0.4.diff
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1415,18 +1415,19 @@ index a40e34d94d0..abc1f035d15 100644
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala
index 11e9547dfc5..637411056ae 100644
index 11e9547dfc5..ba340c4ebcf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.{AnalysisException, Row}
@@ -24,6 +24,8 @@ import org.apache.spark.sql.{AnalysisException, Row}
import org.apache.spark.sql.catalyst.ExtendedAnalysisException
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.util.CollationFactory
+import org.apache.spark.sql.comet.{CometBroadcastHashJoinExec, CometHashJoinExec, CometSortMergeJoinExec}
+import org.apache.spark.sql.comet.CometHashAggregateExec
import org.apache.spark.sql.connector.{DatasourceV2SQLBase, FakeV2ProviderWithCustomSchema}
import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryTable}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.CatalogHelper
@@ -55,7 +56,9 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
@@ -55,7 +57,9 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
assert(
collectFirst(queryPlan) {
case _: SortMergeJoinExec => assert(isSortMergeForced)
Expand All @@ -1436,7 +1437,15 @@ index 11e9547dfc5..637411056ae 100644
}.nonEmpty
)
}
@@ -1547,10 +1550,14 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
@@ -463,6 +467,7 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
val dfBinary = sql(s"SELECT COUNT(*), c FROM $tableNameBinary GROUP BY c")
assert(collectFirst(dfBinary.queryExecution.executedPlan) {
case _: HashAggregateExec | _: ObjectHashAggregateExec => ()
+ case _: CometHashAggregateExec => ()
}.nonEmpty)
}
}
@@ -1547,10 +1552,14 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
if (!CollationFactory.fetchCollation(t.collation).supportsBinaryEquality) {
assert(collectFirst(queryPlan) {
case b: HashJoin => b.leftKeys.head
Expand All @@ -1451,7 +1460,7 @@ index 11e9547dfc5..637411056ae 100644
}.head.isInstanceOf[CollationKey])
}
}
@@ -1606,11 +1613,13 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
@@ -1606,11 +1615,13 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
if (!CollationFactory.fetchCollation(t.collation).supportsBinaryEquality) {
assert(collectFirst(queryPlan) {
case b: BroadcastHashJoinExec => b.leftKeys.head
Expand All @@ -1465,7 +1474,7 @@ index 11e9547dfc5..637411056ae 100644
}.head.isInstanceOf[ArrayTransform])
}
}
@@ -1676,6 +1685,7 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
@@ -1676,6 +1687,7 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
} else {
assert(!collectFirst(queryPlan) {
case b: BroadcastHashJoinExec => b.leftKeys.head
Expand Down Expand Up @@ -3030,7 +3039,7 @@ index 30503af0fab..1491f4bc2d5 100644

import testImplicits._
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
index 08fd8a9ecb5..06967aec8e1 100644
index 08fd8a9ecb5..e0b8cada307 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
@@ -27,6 +27,7 @@ import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
Expand Down
Loading