Skip to content

Commit 7a7f9ba

Browse files
mihailoale-dbmihailoale-db
committed
initial commit
1 parent aecd932 commit 7a7f9ba

File tree

3 files changed

+51
-33
lines changed

3 files changed

+51
-33
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -319,15 +319,15 @@ class Analyzer(
319319
AnalysisContext.reset()
320320
try {
321321
AnalysisHelper.markInAnalyzer {
322-
HybridAnalyzer.fromLegacyAnalyzer(legacyAnalyzer = this).apply(plan, tracker)
322+
HybridAnalyzer.fromLegacyAnalyzer(legacyAnalyzer = this, tracker = tracker).apply(plan)
323323
}
324324
} finally {
325325
AnalysisContext.reset()
326326
}
327327
} else {
328328
AnalysisContext.withNewAnalysisContext {
329329
AnalysisHelper.markInAnalyzer {
330-
HybridAnalyzer.fromLegacyAnalyzer(legacyAnalyzer = this).apply(plan, tracker)
330+
HybridAnalyzer.fromLegacyAnalyzer(legacyAnalyzer = this, tracker = tracker).apply(plan)
331331
}
332332
}
333333
}

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ class HybridAnalyzer(
6161
legacyAnalyzer: Analyzer,
6262
resolverGuard: ResolverGuard,
6363
resolver: Resolver,
64+
tracker: QueryPlanningTracker,
6465
extendedResolutionChecks: Seq[LogicalPlan => Unit] = Seq.empty,
6566
extendedRewriteRules: Seq[Rule[LogicalPlan]] = Seq.empty,
6667
exposeExplicitlyUnsupportedResolverFeature: Boolean = false)
@@ -74,7 +75,7 @@ class HybridAnalyzer(
7475
)
7576
private val sampleRateGenerator = new Random()
7677

77-
def apply(plan: LogicalPlan, tracker: QueryPlanningTracker): LogicalPlan = {
78+
def apply(plan: LogicalPlan): LogicalPlan = {
7879
val dualRun =
7980
conf.getConf(SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER) &&
8081
checkDualRunSampleRate() &&
@@ -296,7 +297,8 @@ object HybridAnalyzer {
296297
*/
297298
def fromLegacyAnalyzer(
298299
legacyAnalyzer: Analyzer,
299-
exposeExplicitlyUnsupportedResolverFeature: Boolean = false): HybridAnalyzer = {
300+
exposeExplicitlyUnsupportedResolverFeature: Boolean = false,
301+
tracker: QueryPlanningTracker): HybridAnalyzer = {
300302
new HybridAnalyzer(
301303
legacyAnalyzer = legacyAnalyzer,
302304
resolverGuard = new ResolverGuard(legacyAnalyzer.catalogManager),
@@ -307,6 +309,7 @@ object HybridAnalyzer {
307309
metadataResolverExtensions = legacyAnalyzer.singlePassMetadataResolverExtensions,
308310
externalRelationResolution = Some(legacyAnalyzer.getRelationResolution)
309311
),
312+
tracker = tracker,
310313
extendedResolutionChecks = legacyAnalyzer.singlePassExtendedResolutionChecks,
311314
extendedRewriteRules = legacyAnalyzer.singlePassPostHocResolutionRules,
312315
exposeExplicitlyUnsupportedResolverFeature = exposeExplicitlyUnsupportedResolverFeature

‎sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/HybridAnalyzerSuite.scala‎

Lines changed: 44 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,8 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
148148
extends HybridAnalyzer(
149149
legacyAnalyzer = legacyAnalyzer,
150150
resolverGuard = resolverGuard,
151-
resolver = resolver
151+
resolver = resolver,
152+
tracker = new QueryPlanningTracker
152153
) {
153154
override protected[sql] def normalizePlan(plan: LogicalPlan): LogicalPlan = {
154155
throw new Exception("Broken plan normalization")
@@ -177,8 +178,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
177178
new HybridAnalyzer(
178179
new ValidatingAnalyzer(bridgeRelations = true),
179180
new ResolverGuard(spark.sessionState.catalogManager),
180-
new ValidatingResolver(bridgeRelations = true)
181-
).apply(unresolvedPlan, new QueryPlanningTracker),
181+
new ValidatingResolver(bridgeRelations = true),
182+
new QueryPlanningTracker
183+
).apply(unresolvedPlan),
182184
resolvedPlan
183185
)
184186
}
@@ -192,8 +194,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
192194
new BrokenResolver(
193195
QueryCompilationErrors.unsupportedSinglePassAnalyzerFeature("test"),
194196
bridgeRelations = true
195-
)
196-
).apply(unresolvedPlan, new QueryPlanningTracker)
197+
),
198+
new QueryPlanningTracker
199+
).apply(unresolvedPlan)
197200
),
198201
condition = "UNSUPPORTED_SINGLE_PASS_ANALYZER_FEATURE",
199202
parameters = Map("feature" -> "test")
@@ -208,8 +211,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
208211
new BrokenResolver(
209212
new StackOverflowError("Stack Overflow"),
210213
bridgeRelations = true
211-
)
212-
).apply(unresolvedPlan, new QueryPlanningTracker)
214+
),
215+
new QueryPlanningTracker
216+
).apply(unresolvedPlan)
213217
)
214218
}
215219

@@ -219,8 +223,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
219223
new HybridAnalyzer(
220224
new ValidatingAnalyzer(bridgeRelations = true),
221225
new ResolverGuard(spark.sessionState.catalogManager),
222-
new HardCodedResolver(resolvedPlan, bridgeRelations = true)
223-
).apply(malformedUnresolvedPlan, new QueryPlanningTracker)
226+
new HardCodedResolver(resolvedPlan, bridgeRelations = true),
227+
new QueryPlanningTracker
228+
).apply(malformedUnresolvedPlan)
224229
),
225230
condition = "HYBRID_ANALYZER_EXCEPTION.FIXED_POINT_FAILED_SINGLE_PASS_SUCCEEDED",
226231
parameters = Map("singlePassOutput" -> resolvedPlan.toString)
@@ -233,8 +238,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
233238
new HybridAnalyzer(
234239
new ValidatingAnalyzer(bridgeRelations = true),
235240
new ResolverGuard(spark.sessionState.catalogManager),
236-
new ValidatingResolver(bridgeRelations = true)
237-
).apply(malformedUnresolvedPlan, new QueryPlanningTracker)
241+
new ValidatingResolver(bridgeRelations = true),
242+
new QueryPlanningTracker
243+
).apply(malformedUnresolvedPlan)
238244
),
239245
condition = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
240246
parameters = Map(
@@ -250,8 +256,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
250256
new HybridAnalyzer(
251257
new ValidatingAnalyzer(bridgeRelations = true),
252258
new ResolverGuard(spark.sessionState.catalogManager),
253-
new HardCodedResolver(malformedResolvedPlan, bridgeRelations = true)
254-
).apply(unresolvedPlan, new QueryPlanningTracker)
259+
new HardCodedResolver(malformedResolvedPlan, bridgeRelations = true),
260+
new QueryPlanningTracker
261+
).apply(unresolvedPlan)
255262
),
256263
condition = "HYBRID_ANALYZER_EXCEPTION.LOGICAL_PLAN_COMPARISON_MISMATCH",
257264
parameters = Map(
@@ -277,8 +284,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
277284
new HybridAnalyzer(
278285
new ValidatingAnalyzer(bridgeRelations = true),
279286
new ResolverGuard(spark.sessionState.catalogManager),
280-
new HardCodedResolver(resolvedPlan, bridgeRelations = true)
281-
).apply(plan, new QueryPlanningTracker)
287+
new HardCodedResolver(resolvedPlan, bridgeRelations = true),
288+
new QueryPlanningTracker
289+
).apply(plan)
282290
),
283291
condition = "HYBRID_ANALYZER_EXCEPTION.OUTPUT_SCHEMA_COMPARISON_MISMATCH",
284292
parameters = Map(
@@ -294,7 +302,7 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
294302
new ValidatingAnalyzer(bridgeRelations = true),
295303
new ResolverGuard(spark.sessionState.catalogManager),
296304
new HardCodedResolver(resolvedPlan, bridgeRelations = true)
297-
).apply(unresolvedPlan, new QueryPlanningTracker)
305+
).apply(unresolvedPlan)
298306
}
299307
}
300308

@@ -306,8 +314,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
306314
new BrokenResolver(
307315
new ExplicitlyUnsupportedResolverFeature("FAILURE"),
308316
bridgeRelations = true
309-
)
310-
).apply(unresolvedPlan, new QueryPlanningTracker),
317+
),
318+
new QueryPlanningTracker
319+
).apply(unresolvedPlan),
311320
resolvedPlan
312321
)
313322
}
@@ -331,8 +340,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
331340
new BrokenResolver(
332341
new Exception("Single-pass resolver should not be invoked"),
333342
bridgeRelations = false
334-
)
335-
).apply(plan, new QueryPlanningTracker)
343+
),
344+
new QueryPlanningTracker
345+
).apply(plan)
336346
},
337347
resolvedPlan
338348
)
@@ -358,8 +368,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
358368
bridgeRelations = false
359369
),
360370
new ResolverGuard(spark.sessionState.catalogManager),
361-
new ValidatingResolver(bridgeRelations = false)
362-
).apply(plan, new QueryPlanningTracker)
371+
new ValidatingResolver(bridgeRelations = false),
372+
new QueryPlanningTracker
373+
).apply(plan)
363374
},
364375
resolvedPlan
365376
)
@@ -387,8 +398,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
387398
bridgeRelations = false
388399
),
389400
new ResolverGuard(spark.sessionState.catalogManager),
390-
new ValidatingResolver(bridgeRelations = false)
391-
).apply(plan, new QueryPlanningTracker)
401+
new ValidatingResolver(bridgeRelations = false),
402+
new QueryPlanningTracker
403+
).apply(plan)
392404
},
393405
resolvedPlan
394406
)
@@ -401,8 +413,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
401413
bridgeRelations = true
402414
),
403415
new ResolverGuard(spark.sessionState.catalogManager),
404-
new ValidatingResolver(bridgeRelations = true)
405-
).apply(plan, new QueryPlanningTracker),
416+
new ValidatingResolver(bridgeRelations = true),
417+
new QueryPlanningTracker
418+
).apply(plan),
406419
resolvedPlan
407420
)
408421
}
@@ -425,8 +438,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
425438
legacyAnalyzer = new ValidatingAnalyzer(bridgeRelations = true),
426439
resolverGuard = new ResolverGuard(spark.sessionState.catalogManager),
427440
resolver = new ValidatingResolver(bridgeRelations = true),
428-
extendedResolutionChecks = Seq(new BrokenCheckRule)
429-
).apply(plan, new QueryPlanningTracker)
441+
tracker = new QueryPlanningTracker,
442+
extendedResolutionChecks = Seq(new BrokenCheckRule),
443+
).apply(plan)
430444
}
431445

432446
withSQLConf(
@@ -437,8 +451,9 @@ class HybridAnalyzerSuite extends QueryTest with SharedSparkSession {
437451
legacyAnalyzer = new ValidatingAnalyzer(bridgeRelations = true),
438452
resolverGuard = new ResolverGuard(spark.sessionState.catalogManager),
439453
resolver = new ValidatingResolver(bridgeRelations = true),
454+
tracker = new QueryPlanningTracker,
440455
extendedResolutionChecks = Seq(new BrokenCheckRule)
441-
).apply(plan, new QueryPlanningTracker),
456+
).apply(plan),
442457
resolvedPlan
443458
)
444459
}

0 commit comments

Comments
 (0)