Skip to content

Commit f955312

Browse files
andygroveandygrove
committed
rebase
1 parent c6136aa commit f955312

File tree

9 files changed

+41
-49
lines changed

9 files changed

+41
-49
lines changed

‎common/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -459,16 +459,21 @@ object CometConf extends ShimCometConf {
459459
.booleanConf
460460
.createWithDefault(false)
461461

462-
val COMET_EXPLAIN_VERBOSE_ENABLED: ConfigEntry[Boolean] =
463-
conf("spark.comet.explain.verbose.enabled")
462+
val COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE = "verbose"
463+
val COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK = "fallback"
464+
465+
val COMET_EXTENDED_EXPLAIN_FORMAT: ConfigEntry[String] =
466+
conf("spark.comet.explain.format")
464467
.category(CATEGORY_EXEC_EXPLAIN)
465-
.doc(
466-
"When this setting is enabled, Comet's extended explain output will provide the full " +
467-
"query plan annotated with fallback reasons as well as a summary of how much of " +
468-
"the plan was accelerated by Comet. When this setting is disabled, a list of fallback " +
469-
"reasons will be provided instead.")
470-
.booleanConf
471-
.createWithDefault(false)
468+
.doc("Choose extended explain output. The default format of " +
469+
s"'$COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE' will provide the full query plan annotated " +
470+
"with fallback reasons as well as a summary of how much of the plan was accelerated " +
471+
s"by Comet. The format '$COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK' provides a list of " +
472+
"fallback reasons instead.")
473+
.stringConf
474+
.checkValues(
475+
Set(COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE, COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK))
476+
.createWithDefault(COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE)
472477

473478
val COMET_EXPLAIN_NATIVE_ENABLED: ConfigEntry[Boolean] =
474479
conf("spark.comet.explain.native.enabled")

‎docs/source/user-guide/latest/configs.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,9 @@ These settings can be used to determine which parts of the plan are accelerated
8282
<!--BEGIN:CONFIG_TABLE[exec_explain]-->
8383
| Config | Description | Default Value |
8484
|--------|-------------|---------------|
85+
| `spark.comet.explain.format` | Choose extended explain output. The default format of 'verbose' will provide the full query plan annotated with fallback reasons as well as a summary of how much of the plan was accelerated by Comet. The format 'fallback' provides a list of fallback reasons instead. | verbose |
8586
| `spark.comet.explain.native.enabled` | When this setting is enabled, Comet will provide a tree representation of the native query plan before execution and again after execution, with metrics. | false |
8687
| `spark.comet.explain.rules` | When this setting is enabled, Comet will log all plan transformations performed in physical optimizer rules. Default: false | false |
87-
| `spark.comet.explain.verbose.enabled` | When this setting is enabled, Comet's extended explain output will provide the full query plan annotated with fallback reasons as well as a summary of how much of the plan was accelerated by Comet. When this setting is disabled, a list of fallback reasons will be provided instead. | false |
8888
| `spark.comet.explainFallback.enabled` | When this setting is enabled, Comet will provide logging explaining the reason(s) why a query stage cannot be executed natively. Set this to false to reduce the amount of logging. | false |
8989
| `spark.comet.logFallbackReasons.enabled` | When this setting is enabled, Comet will log warnings for all fallback reasons. | false |
9090
<!--END:CONFIG_TABLE-->

‎spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -34,16 +34,26 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator {
3434

3535
override def title: String = "Comet"
3636

37-
override def generateExtendedInfo(plan: SparkPlan): String = {
38-
if (CometConf.COMET_EXPLAIN_VERBOSE_ENABLED.get()) {
39-
generateVerboseExtendedInfo(plan)
40-
} else {
41-
val info = getFallbackReasons(plan)
42-
info.toSeq.sorted.mkString("\n").trim
37+
def generateExtendedInfo(plan: SparkPlan): String = {
38+
CometConf.COMET_EXTENDED_EXPLAIN_FORMAT.get() match {
39+
case CometConf.COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE =>
40+
// Generates the extended info in a verbose manner, printing each node along with the
41+
// extended information in a tree display.
42+
val planStats = new CometCoverageStats()
43+
val outString = new StringBuilder()
44+
generateTreeString(getActualPlan(plan), 0, Seq(), 0, outString, planStats)
45+
s"${outString.toString()}\n$planStats"
46+
case CometConf.COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK =>
47+
// Generates the extended info as a list of fallback reasons
48+
getFallbackReasons(plan).mkString("\n").trim
4349
}
4450
}
4551

46-
def getFallbackReasons(node: TreeNode[_]): Set[String] = {
52+
def getFallbackReasons(plan: SparkPlan): Seq[String] = {
53+
extensionInfo(plan).toSeq.sorted
54+
}
55+
56+
private[comet] def extensionInfo(node: TreeNode[_]): Set[String] = {
4757
var info = mutable.Seq[String]()
4858
val sorted = sortup(node)
4959
sorted.foreach { p =>
@@ -80,23 +90,6 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator {
8090
ordered.reverse
8191
}
8292

83-
// generates the extended info in a verbose manner, printing each node along with the
84-
// extended information in a tree display
85-
def generateVerboseExtendedInfo(plan: SparkPlan): String = {
86-
val planStats = new CometCoverageStats()
87-
val outString = new StringBuilder()
88-
generateTreeString(getActualPlan(plan), 0, Seq(), 0, outString, planStats)
89-
s"${outString.toString()}\n$planStats"
90-
}
91-
92-
/** Get the coverage statistics without the full plan */
93-
def generateCoverageInfo(plan: SparkPlan): String = {
94-
val planStats = new CometCoverageStats()
95-
val outString = new StringBuilder()
96-
generateTreeString(getActualPlan(plan), 0, Seq(), 0, outString, planStats)
97-
planStats.toString()
98-
}
99-
10093
// Simplified generateTreeString from Spark TreeNode. Appends explain info to the node if any
10194
def generateTreeString(
10295
node: TreeNode[_],

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -651,12 +651,12 @@ case class CometExecRule(session: SparkSession) extends Rule[SparkPlan] {
651651
// config is enabled)
652652
if (CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.get()) {
653653
val info = new ExtendedExplainInfo()
654-
if (info.getFallbackReasons(newPlan).nonEmpty) {
654+
if (info.extensionInfo(newPlan).nonEmpty) {
655655
logWarning(
656656
"Comet cannot execute some parts of this plan natively " +
657657
s"(set ${CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key}=false " +
658658
"to disable this logging):\n" +
659-
s"${info.generateVerboseExtendedInfo(newPlan)}")
659+
s"${info.generateExtendedInfo(newPlan)}")
660660
}
661661
}
662662

‎spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919

2020
package org.apache.comet
2121

22-
import scala.collection.immutable.HashSet
2322
import scala.util.Random
2423

2524
import org.apache.hadoop.fs.Path
@@ -126,11 +125,11 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp
126125
spark.read.parquet(path.toString).createOrReplaceTempView("t1")
127126
sql("SELECT array(struct(_1, _2)) as a, struct(_1, _2) as b FROM t1")
128127
.createOrReplaceTempView("t2")
129-
val expectedFallbackReasons = HashSet(
130-
"data type not supported: ArrayType(StructType(StructField(_1,BooleanType,true),StructField(_2,ByteType,true)),false)")
131-
checkSparkAnswerAndFallbackReasons(
128+
val expectedFallbackReason =
129+
"data type not supported: ArrayType(StructType(StructField(_1,BooleanType,true),StructField(_2,ByteType,true)),false)"
130+
checkSparkAnswerAndFallbackReason(
132131
sql("SELECT array_remove(a, b) FROM t2"),
133-
expectedFallbackReasons)
132+
expectedFallbackReason)
134133
}
135134
}
136135
}

‎spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,7 @@ class CometExecSuite extends CometTestBase {
119119
val infos = new ExtendedExplainInfo().generateExtendedInfo(cometPlan)
120120
assert(infos.contains("Dynamic Partition Pruning is not supported"))
121121

122-
withSQLConf(CometConf.COMET_EXPLAIN_VERBOSE_ENABLED.key -> "true") {
123-
val extendedExplain = new ExtendedExplainInfo().generateExtendedInfo(cometPlan)
124-
assert(extendedExplain.contains("Comet accelerated"))
125-
}
122+
assert(infos.contains("Comet accelerated"))
126123
}
127124
}
128125
}

‎spark/src/test/scala/org/apache/spark/sql/CometTPCQueryListBase.scala‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,6 @@ trait CometTPCQueryListBase
8787
CometConf.COMET_ENABLED.key -> "true",
8888
CometConf.COMET_EXEC_ENABLED.key -> "true",
8989
CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
90-
CometConf.COMET_EXPLAIN_VERBOSE_ENABLED.key -> "true",
9190
// Lower bloom filter thresholds to allows us to simulate the plan produced at larger scale.
9291
"spark.sql.optimizer.runtime.bloomFilter.creationSideThreshold" -> "1MB",
9392
"spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold" -> "1MB") {

‎spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ abstract class CometTestBase
283283
if (actualFallbacks.isEmpty) {
284284
fail(
285285
s"Expected fallback reason '$reason' but no fallback reasons were found. Explain: ${explainInfo
286-
.generateVerboseExtendedInfo(cometPlan)}")
286+
.generateExtendedInfo(cometPlan)}")
287287
} else {
288288
fail(
289289
s"Expected fallback reason '$reason' not found in [${actualFallbacks.mkString(", ")}]")
@@ -375,7 +375,7 @@ abstract class CometTestBase
375375
assert(
376376
false,
377377
s"Expected only Comet native operators, but found ${op.nodeName}.\n" +
378-
s"plan: ${new ExtendedExplainInfo().generateVerboseExtendedInfo(plan)}")
378+
s"plan: ${new ExtendedExplainInfo().generateExtendedInfo(plan)}")
379379
case _ =>
380380
}
381381
}

‎spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,6 @@ trait CometPlanStabilitySuite extends DisableAdaptiveExecutionSuite with TPCDSBa
217217

218218
withSQLConf(
219219
CometConf.COMET_EXPLAIN_FALLBACK_ENABLED.key -> "true",
220-
CometConf.COMET_EXPLAIN_VERBOSE_ENABLED.key -> "true",
221220
CometConf.COMET_ENABLED.key -> "true",
222221
CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true",
223222
CometConf.COMET_EXEC_ENABLED.key -> "true",

0 commit comments

Comments
 (0)