Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit 53699b3

Browse files
juanjuxjuanjux
authored
Merge pull request #48 from juanjux/fix/expressions_statements
Added expression and statement roles
2 parents 250f13a + 71826f9 commit 53699b3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+588
-591
lines changed

‎ANNOTATION.md‎

Lines changed: 42 additions & 43 deletions
Large diffs are not rendered by default.

‎driver/normalizer/annotation.go‎

Lines changed: 42 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ var AnnotationRules = On(Any).Self(
9595
On(Not(HasInternalType(pyast.Module))).Error(errors.New("root must be Module")),
9696
On(HasInternalType(pyast.Module)).Roles(File).Descendants(
9797
// Binary Expressions
98-
On(HasInternalType(pyast.BinOp)).Roles(BinaryExpression).Children(
98+
On(HasInternalType(pyast.BinOp)).Roles(BinaryExpression, Expression).Children(
9999
On(HasInternalRole("op")).Roles(BinaryExpressionOp),
100100
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
101101
On(HasInternalRole("right")).Roles(BinaryExpressionRight),
@@ -143,23 +143,23 @@ var AnnotationRules = On(Any).Self(
143143
On(HasInternalType(pyast.UAdd)).Roles(OpPositive),
144144
On(HasInternalType(pyast.USub)).Roles(OpNegative),
145145

146-
On(HasInternalType(pyast.StringLiteral)).Roles(StringLiteral),
147-
On(HasInternalType(pyast.ByteLiteral)).Roles(ByteStringLiteral),
148-
On(HasInternalType(pyast.NumLiteral)).Roles(NumberLiteral),
149-
On(HasInternalType(pyast.Str)).Roles(StringLiteral),
150-
On(HasInternalType(pyast.BoolLiteral)).Roles(BooleanLiteral),
151-
On(HasInternalType(pyast.JoinedStr)).Roles(StringLiteral).Children(
146+
On(HasInternalType(pyast.StringLiteral)).Roles(StringLiteral, Expression),
147+
On(HasInternalType(pyast.ByteLiteral)).Roles(ByteStringLiteral, Expression),
148+
On(HasInternalType(pyast.NumLiteral)).Roles(NumberLiteral, Expression),
149+
On(HasInternalType(pyast.Str)).Roles(StringLiteral, Expression),
150+
On(HasInternalType(pyast.BoolLiteral)).Roles(BooleanLiteral, Expression),
151+
On(HasInternalType(pyast.JoinedStr)).Roles(StringLiteral, Expression).Children(
152152
// FIXME: should be StringInterpolatedExpression or something like that
153153
On(HasInternalType(pyast.FormattedValue)).Roles(Expression),
154154
),
155-
On(HasInternalType(pyast.NoneLiteral)).Roles(NullLiteral),
156-
On(HasInternalType(pyast.Set)).Roles(SetLiteral),
157-
On(HasInternalType(pyast.List)).Roles(ListLiteral),
158-
On(HasInternalType(pyast.Dict)).Roles(MapLiteral).Children(
155+
On(HasInternalType(pyast.NoneLiteral)).Roles(NullLiteral, Expression),
156+
On(HasInternalType(pyast.Set)).Roles(SetLiteral, Expression),
157+
On(HasInternalType(pyast.List)).Roles(ListLiteral, Expression),
158+
On(HasInternalType(pyast.Dict)).Roles(MapLiteral, Expression).Children(
159159
On(HasInternalRole("keys")).Roles(MapKey),
160160
On(HasInternalRole("values")).Roles(MapValue),
161161
),
162-
On(HasInternalType(pyast.Tuple)).Roles(TupleLiteral),
162+
On(HasInternalType(pyast.Tuple)).Roles(TupleLiteral, Expression),
163163

164164
// FIXME: decorators
165165
// FIXME: the FunctionDeclarationReceiver is not set for methods; it should be taken from the parent
@@ -186,7 +186,7 @@ var AnnotationRules = On(Any).Self(
186186
),
187187
),
188188

189-
On(HasInternalType(pyast.Call)).Roles(Call).Children(
189+
On(HasInternalType(pyast.Call)).Roles(Call, Expression).Children(
190190
On(HasInternalRole("args")).Roles(CallPositionalArgument),
191191
On(HasInternalRole("keywords")).Roles(CallNamedArgument).Children(
192192
On(HasInternalRole("value")).Roles(CallNamedArgumentValue),
@@ -203,21 +203,21 @@ var AnnotationRules = On(Any).Self(
203203
// targets[] => AssignmentVariable
204204
// value => AssignmentValue
205205
//
206-
On(HasInternalType(pyast.Assign)).Roles(Assignment).Children(
206+
On(HasInternalType(pyast.Assign)).Roles(Assignment, Statement).Children(
207207
On(HasInternalRole("targets")).Roles(AssignmentVariable),
208208
On(HasInternalRole("value")).Roles(AssignmentValue),
209209
),
210210

211-
On(HasInternalType(pyast.AugAssign)).Roles(AugmentedAssignment).Children(
211+
On(HasInternalType(pyast.AugAssign)).Roles(AugmentedAssignment, Statement).Children(
212212
On(HasInternalRole("op")).Roles(AugmentedAssignmentOperator),
213213
On(HasInternalRole("target")).Roles(AugmentedAssignmentVariable),
214214
On(HasInternalRole("value")).Roles(AugmentedAssignmentValue),
215215
),
216216

217217
On(HasInternalType(pyast.Expression)).Roles(Expression),
218218
On(HasInternalType(pyast.Expr)).Roles(Expression),
219-
On(HasInternalType(pyast.Name)).Roles(SimpleIdentifier),
220-
On(HasInternalType(pyast.Attribute)).Roles(QualifiedIdentifier),
219+
On(HasInternalType(pyast.Name)).Roles(SimpleIdentifier, Expression),
220+
On(HasInternalType(pyast.Attribute)).Roles(QualifiedIdentifier, Expression),
221221

222222
// Comments and non significative whitespace
223223
On(HasInternalType(pyast.SameLineNoops)).Roles(Comment),
@@ -229,24 +229,24 @@ var AnnotationRules = On(Any).Self(
229229
),
230230

231231
// TODO: check what Constant nodes are generated in the python AST and improve this
232-
On(HasInternalType(pyast.Constant)).Roles(SimpleIdentifier),
233-
On(HasInternalType(pyast.Try)).Roles(Try).Children(
232+
On(HasInternalType(pyast.Constant)).Roles(SimpleIdentifier, Expression),
233+
On(HasInternalType(pyast.Try)).Roles(Try, Statement).Children(
234234
On(HasInternalRole("body")).Roles(TryBody),
235235
On(HasInternalRole("finalbody")).Roles(TryFinally),
236236
On(HasInternalRole("handlers")).Roles(TryCatch),
237237
On(HasInternalRole("orelse")).Roles(IfElse),
238238
),
239-
On(HasInternalType(pyast.TryExcept)).Roles(TryCatch), // py2
240-
On(HasInternalType(pyast.ExceptHandler)).Roles(TryCatch), // py3
241-
On(HasInternalType(pyast.TryFinally)).Roles(TryFinally),
242-
On(HasInternalType(pyast.Raise)).Roles(Throw),
239+
On(HasInternalType(pyast.TryExcept)).Roles(TryCatch, Statement), // py2
240+
On(HasInternalType(pyast.ExceptHandler)).Roles(TryCatch, Statement), // py3
241+
On(HasInternalType(pyast.TryFinally)).Roles(TryFinally, Statement),
242+
On(HasInternalType(pyast.Raise)).Roles(Throw, Statement),
243243
// FIXME: review, add path for the body and items childs
244244
// FIXME: withitem on Python to RAII on a resource and can aditionally create and alias on it,
245245
// both of which currently doesn't have representation in the UAST
246-
On(HasInternalType(pyast.With)).Roles(BlockScope),
247-
On(HasInternalType(pyast.Return)).Roles(Return),
248-
On(HasInternalType(pyast.Break)).Roles(Break),
249-
On(HasInternalType(pyast.Continue)).Roles(Continue),
246+
On(HasInternalType(pyast.With)).Roles(BlockScope, Statement),
247+
On(HasInternalType(pyast.Return)).Roles(Return, Statement),
248+
On(HasInternalType(pyast.Break)).Roles(Break, Statement),
249+
On(HasInternalType(pyast.Continue)).Roles(Continue, Statement),
250250
// FIXME: IfCondition bodies in Python take the form:
251251
// 1 < a < 10
252252
// - left (internalRole): 1 (first element)
@@ -260,11 +260,11 @@ var AnnotationRules = On(Any).Self(
260260
// and SDK feature to mix lists (also needed for default and keyword arguments and
261261
// boolean operators).
262262
// "If that sounds awkward is because it is" (their words)
263-
On(HasInternalType(pyast.If)).Roles(If).Children(
263+
On(HasInternalType(pyast.If)).Roles(If, Statement).Children(
264264
On(HasInternalType("If.body")).Roles(IfBody),
265265
On(HasInternalRole("test")).Roles(IfCondition),
266266
On(HasInternalType("If.orelse")).Roles(IfElse),
267-
On(HasInternalType(pyast.Compare)).Roles(BinaryExpression).Children(
267+
On(HasInternalType(pyast.Compare)).Roles(BinaryExpression, Expression).Children(
268268
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
269269
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
270270
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
@@ -276,44 +276,42 @@ var AnnotationRules = On(Any).Self(
276276
On(HasInternalRole("test")).Roles(IfCondition),
277277
On(HasInternalRole("orelse")).Roles(IfElse),
278278
),
279-
// One liner if, like a normal If but it will be inside an Assign (like the ternary if in C)
280-
On(HasInternalType(pyast.IfExp)).Roles(If),
281-
On(HasInternalType(pyast.Import)).Roles(ImportDeclaration),
282-
On(HasInternalType(pyast.ImportFrom)).Roles(ImportDeclaration),
279+
On(HasInternalType(pyast.Import)).Roles(ImportDeclaration, Statement),
280+
On(HasInternalType(pyast.ImportFrom)).Roles(ImportDeclaration, Statement),
283281
On(HasInternalType(pyast.Alias)).Roles(ImportAlias, SimpleIdentifier),
284-
On(HasInternalType(pyast.ClassDef)).Roles(TypeDeclaration, SimpleIdentifier).Children(
282+
On(HasInternalType(pyast.ClassDef)).Roles(TypeDeclaration, SimpleIdentifier, Statement).Children(
285283
On(HasInternalType("ClassDef.body")).Roles(TypeDeclarationBody),
286284
On(HasInternalType("ClassDef.bases")).Roles(TypeDeclarationBases),
287285
),
288286

289-
On(HasInternalType(pyast.For)).Roles(ForEach).Children(
287+
On(HasInternalType(pyast.For)).Roles(ForEach, Statement).Children(
290288
On(HasInternalType("For.body")).Roles(ForBody),
291289
On(HasInternalRole("iter")).Roles(ForExpression),
292290
On(HasInternalRole("target")).Roles(ForUpdate),
293291
On(HasInternalType("For.orelse")).Roles(IfElse),
294292
),
295-
On(HasInternalType(pyast.While)).Roles(While).Children(
293+
On(HasInternalType(pyast.While)).Roles(While, Statement).Children(
296294
On(HasInternalType("While.body")).Roles(WhileBody),
297295
On(HasInternalRole("test")).Roles(WhileCondition),
298296
On(HasInternalType("While.orelse")).Roles(IfElse),
299297
),
300-
On(HasInternalType(pyast.Pass)).Roles(Noop),
301-
On(HasInternalType(pyast.Num)).Roles(NumberLiteral),
298+
On(HasInternalType(pyast.Pass)).Roles(Noop, Statement),
299+
On(HasInternalType(pyast.Num)).Roles(NumberLiteral, Expression),
302300
// FIXME: this is the annotated assignment (a: annotation = 3) not exactly Assignment
303301
// it also lacks AssignmentValue and AssignmentVariable (see how to add them)
304-
On(HasInternalType(pyast.AnnAssign)).Roles(Assignment),
305-
On(HasInternalType(pyast.Assert)).Roles(Assert),
302+
On(HasInternalType(pyast.AnnAssign)).Roles(Assignment, Statement),
303+
On(HasInternalType(pyast.Assert)).Roles(Assert, Statement),
306304

307305
// These are AST nodes in Python2 but we convert them to functions in the UAST like
308306
// they are in Python3
309-
On(HasInternalType(pyast.Exec)).Roles(Call).Children(
307+
On(HasInternalType(pyast.Exec)).Roles(Call, Expression).Children(
310308
On(HasInternalRole("body")).Roles(CallPositionalArgument),
311309
On(HasInternalRole("globals")).Roles(CallPositionalArgument),
312310
On(HasInternalRole("locals")).Roles(CallPositionalArgument),
313311
),
314312
// Repr already comes as a Call \o/
315313
// Print as a function too.
316-
On(HasInternalType(pyast.Print)).Roles(Call, CallCallee, SimpleIdentifier).Children(
314+
On(HasInternalType(pyast.Print)).Roles(Call, CallCallee, SimpleIdentifier, Expression).Children(
317315
On(HasInternalRole("dest")).Roles(CallPositionalArgument),
318316
On(HasInternalRole("nl")).Roles(CallPositionalArgument),
319317
On(HasInternalRole("values")).Roles(CallPositionalArgument).Children(
@@ -336,8 +334,8 @@ var AnnotationRules = On(Any).Self(
336334
// List/Map/Set comprehensions. We map the "for x in y" to ForEach roles and the
337335
// "if something" to If* roles. FIXME: missing the top comprehension roles in the UAST, change
338336
// once they've been merged
339-
On(HasInternalType(pyast.Comprehension)).Roles(ForEach).Children(
340-
On(HasInternalRole("iter")).Roles(ForUpdate),
337+
On(HasInternalType(pyast.Comprehension)).Roles(ForEach, Expression).Children(
338+
On(HasInternalRole("iter")).Roles(ForUpdate, Statement),
341339
On(HasInternalRole("target")).Roles(ForExpression),
342340
// FIXME: see the comment on IfCondition above
343341
On(HasInternalType(pyast.Compare)).Roles(IfCondition, BinaryExpression).Children(

‎tests/aritmeticops.py.uast‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ Module {
3131
. . . }
3232
. . . Children: {
3333
. . . . 0: BinOp {
34-
. . . . . Roles: BinaryExpression
34+
. . . . . Roles: BinaryExpression,Expression
3535
. . . . . StartPosition: {
3636
. . . . . . Offset: 0
3737
. . . . . . Line: 1
@@ -47,7 +47,7 @@ Module {
4747
. . . . . }
4848
. . . . . Children: {
4949
. . . . . . 0: NumLiteral {
50-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
50+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
5151
. . . . . . . TOKEN "1"
5252
. . . . . . . StartPosition: {
5353
. . . . . . . . Offset: 0
@@ -82,7 +82,7 @@ Module {
8282
. . . . . . . }
8383
. . . . . . }
8484
. . . . . . 2: NumLiteral {
85-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
85+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
8686
. . . . . . . TOKEN "2"
8787
. . . . . . . StartPosition: {
8888
. . . . . . . . Offset: 2
@@ -120,7 +120,7 @@ Module {
120120
. . . }
121121
. . . Children: {
122122
. . . . 0: BinOp {
123-
. . . . . Roles: BinaryExpression
123+
. . . . . Roles: BinaryExpression,Expression
124124
. . . . . StartPosition: {
125125
. . . . . . Offset: 4
126126
. . . . . . Line: 2
@@ -136,7 +136,7 @@ Module {
136136
. . . . . }
137137
. . . . . Children: {
138138
. . . . . . 0: NumLiteral {
139-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
139+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
140140
. . . . . . . TOKEN "1"
141141
. . . . . . . StartPosition: {
142142
. . . . . . . . Offset: 4
@@ -171,7 +171,7 @@ Module {
171171
. . . . . . . }
172172
. . . . . . }
173173
. . . . . . 2: NumLiteral {
174-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
174+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
175175
. . . . . . . TOKEN "2"
176176
. . . . . . . StartPosition: {
177177
. . . . . . . . Offset: 6
@@ -209,7 +209,7 @@ Module {
209209
. . . }
210210
. . . Children: {
211211
. . . . 0: BinOp {
212-
. . . . . Roles: BinaryExpression
212+
. . . . . Roles: BinaryExpression,Expression
213213
. . . . . StartPosition: {
214214
. . . . . . Offset: 8
215215
. . . . . . Line: 3
@@ -225,7 +225,7 @@ Module {
225225
. . . . . }
226226
. . . . . Children: {
227227
. . . . . . 0: NumLiteral {
228-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
228+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
229229
. . . . . . . TOKEN "1"
230230
. . . . . . . StartPosition: {
231231
. . . . . . . . Offset: 8
@@ -260,7 +260,7 @@ Module {
260260
. . . . . . . }
261261
. . . . . . }
262262
. . . . . . 2: NumLiteral {
263-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
263+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
264264
. . . . . . . TOKEN "2"
265265
. . . . . . . StartPosition: {
266266
. . . . . . . . Offset: 10
@@ -298,7 +298,7 @@ Module {
298298
. . . }
299299
. . . Children: {
300300
. . . . 0: BinOp {
301-
. . . . . Roles: BinaryExpression
301+
. . . . . Roles: BinaryExpression,Expression
302302
. . . . . StartPosition: {
303303
. . . . . . Offset: 12
304304
. . . . . . Line: 4
@@ -314,7 +314,7 @@ Module {
314314
. . . . . }
315315
. . . . . Children: {
316316
. . . . . . 0: NumLiteral {
317-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
317+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
318318
. . . . . . . TOKEN "1"
319319
. . . . . . . StartPosition: {
320320
. . . . . . . . Offset: 12
@@ -349,7 +349,7 @@ Module {
349349
. . . . . . . }
350350
. . . . . . }
351351
. . . . . . 2: NumLiteral {
352-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
352+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
353353
. . . . . . . TOKEN "2"
354354
. . . . . . . StartPosition: {
355355
. . . . . . . . Offset: 14
@@ -387,7 +387,7 @@ Module {
387387
. . . }
388388
. . . Children: {
389389
. . . . 0: BinOp {
390-
. . . . . Roles: BinaryExpression
390+
. . . . . Roles: BinaryExpression,Expression
391391
. . . . . StartPosition: {
392392
. . . . . . Offset: 16
393393
. . . . . . Line: 5
@@ -403,7 +403,7 @@ Module {
403403
. . . . . }
404404
. . . . . Children: {
405405
. . . . . . 0: NumLiteral {
406-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
406+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
407407
. . . . . . . TOKEN "1"
408408
. . . . . . . StartPosition: {
409409
. . . . . . . . Offset: 16
@@ -438,7 +438,7 @@ Module {
438438
. . . . . . . }
439439
. . . . . . }
440440
. . . . . . 2: NumLiteral {
441-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
441+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
442442
. . . . . . . TOKEN "2"
443443
. . . . . . . StartPosition: {
444444
. . . . . . . . Offset: 19
@@ -476,7 +476,7 @@ Module {
476476
. . . }
477477
. . . Children: {
478478
. . . . 0: BinOp {
479-
. . . . . Roles: BinaryExpression
479+
. . . . . Roles: BinaryExpression,Expression
480480
. . . . . StartPosition: {
481481
. . . . . . Offset: 21
482482
. . . . . . Line: 6
@@ -492,7 +492,7 @@ Module {
492492
. . . . . }
493493
. . . . . Children: {
494494
. . . . . . 0: NumLiteral {
495-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
495+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
496496
. . . . . . . TOKEN "1"
497497
. . . . . . . StartPosition: {
498498
. . . . . . . . Offset: 21
@@ -522,7 +522,7 @@ Module {
522522
. . . . . . . }
523523
. . . . . . }
524524
. . . . . . 2: NumLiteral {
525-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
525+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
526526
. . . . . . . TOKEN "2"
527527
. . . . . . . StartPosition: {
528528
. . . . . . . . Offset: 23
@@ -560,7 +560,7 @@ Module {
560560
. . . }
561561
. . . Children: {
562562
. . . . 0: BinOp {
563-
. . . . . Roles: BinaryExpression
563+
. . . . . Roles: BinaryExpression,Expression
564564
. . . . . StartPosition: {
565565
. . . . . . Offset: 25
566566
. . . . . . Line: 7
@@ -576,7 +576,7 @@ Module {
576576
. . . . . }
577577
. . . . . Children: {
578578
. . . . . . 0: NumLiteral {
579-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
579+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
580580
. . . . . . . TOKEN "1"
581581
. . . . . . . StartPosition: {
582582
. . . . . . . . Offset: 25
@@ -611,7 +611,7 @@ Module {
611611
. . . . . . . }
612612
. . . . . . }
613613
. . . . . . 2: NumLiteral {
614-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
614+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
615615
. . . . . . . TOKEN "2"
616616
. . . . . . . StartPosition: {
617617
. . . . . . . . Offset: 28

0 commit comments

Comments
 (0)