Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit bd9e6c4

Browse files
author
Juanjo Alvarez
committed
Added expression and statement roles
Regenerated integration-tests
1 parent 9d1871a commit bd9e6c4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+590
-593
lines changed

‎ANNOTATION.md‎

Lines changed: 43 additions & 44 deletions
Large diffs are not rendered by default.

‎driver/normalizer/annotation.go‎

Lines changed: 43 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ var AnnotationRules = On(Any).Self(
9595
On(Not(HasInternalType(pyast.Module))).Error(errors.New("root must be Module")),
9696
On(HasInternalType(pyast.Module)).Roles(File).Descendants(
9797
// Binary Expressions
98-
On(HasInternalType(pyast.BinOp)).Roles(BinaryExpression).Children(
98+
On(HasInternalType(pyast.BinOp)).Roles(BinaryExpression, Expression).Children(
9999
On(HasInternalRole("op")).Roles(BinaryExpressionOp),
100100
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
101101
On(HasInternalRole("right")).Roles(BinaryExpressionRight),
@@ -143,23 +143,23 @@ var AnnotationRules = On(Any).Self(
143143
On(HasInternalType(pyast.UAdd)).Roles(OpPositive),
144144
On(HasInternalType(pyast.USub)).Roles(OpNegative),
145145

146-
On(HasInternalType(pyast.StringLiteral)).Roles(StringLiteral),
147-
On(HasInternalType(pyast.ByteLiteral)).Roles(ByteStringLiteral),
148-
On(HasInternalType(pyast.NumLiteral)).Roles(NumberLiteral),
149-
On(HasInternalType(pyast.Str)).Roles(StringLiteral),
150-
On(HasInternalType(pyast.BoolLiteral)).Roles(BooleanLiteral),
151-
On(HasInternalType(pyast.JoinedStr)).Roles(StringLiteral).Children(
146+
On(HasInternalType(pyast.StringLiteral)).Roles(StringLiteral, Expression),
147+
On(HasInternalType(pyast.ByteLiteral)).Roles(ByteStringLiteral, Expression),
148+
On(HasInternalType(pyast.NumLiteral)).Roles(NumberLiteral, Expression),
149+
On(HasInternalType(pyast.Str)).Roles(StringLiteral, Expression),
150+
On(HasInternalType(pyast.BoolLiteral)).Roles(BooleanLiteral, Expression),
151+
On(HasInternalType(pyast.JoinedStr)).Roles(StringLiteral, Expression).Children(
152152
// FIXME: should be StringInterpolatedExpression or something like that
153153
On(HasInternalType(pyast.FormattedValue)).Roles(Expression),
154154
),
155-
On(HasInternalType(pyast.NoneLiteral)).Roles(NullLiteral),
156-
On(HasInternalType(pyast.Set)).Roles(SetLiteral),
157-
On(HasInternalType(pyast.List)).Roles(ListLiteral),
158-
On(HasInternalType(pyast.Dict)).Roles(MapLiteral).Children(
155+
On(HasInternalType(pyast.NoneLiteral)).Roles(NullLiteral, Expression),
156+
On(HasInternalType(pyast.Set)).Roles(SetLiteral, Expression),
157+
On(HasInternalType(pyast.List)).Roles(ListLiteral, Expression),
158+
On(HasInternalType(pyast.Dict)).Roles(MapLiteral, Expression).Children(
159159
On(HasInternalRole("keys")).Roles(MapKey),
160160
On(HasInternalRole("values")).Roles(MapValue),
161161
),
162-
On(HasInternalType(pyast.Tuple)).Roles(TupleLiteral),
162+
On(HasInternalType(pyast.Tuple)).Roles(TupleLiteral, Expression),
163163

164164
// FIXME: decorators
165165
// FIXME: the FunctionDeclarationReceiver is not set for methods; it should be taken from the parent
@@ -186,7 +186,7 @@ var AnnotationRules = On(Any).Self(
186186
),
187187
),
188188

189-
On(HasInternalType(pyast.Call)).Roles(Call).Children(
189+
On(HasInternalType(pyast.Call)).Roles(Call, Expression).Children(
190190
On(HasInternalRole("args")).Roles(CallPositionalArgument),
191191
On(HasInternalRole("keywords")).Roles(CallNamedArgument).Children(
192192
On(HasInternalRole("value")).Roles(CallNamedArgumentValue),
@@ -203,21 +203,21 @@ var AnnotationRules = On(Any).Self(
203203
// targets[] => AssignmentVariable
204204
// value => AssignmentValue
205205
//
206-
On(HasInternalType(pyast.Assign)).Roles(Assignment).Children(
206+
On(HasInternalType(pyast.Assign)).Roles(Assignment, Statement).Children(
207207
On(HasInternalRole("targets")).Roles(AssignmentVariable),
208208
On(HasInternalRole("value")).Roles(AssignmentValue),
209209
),
210210

211-
On(HasInternalType(pyast.AugAssign)).Roles(AugmentedAssignment).Children(
211+
On(HasInternalType(pyast.AugAssign)).Roles(AugmentedAssignment, Statement).Children(
212212
On(HasInternalRole("op")).Roles(AugmentedAssignmentOperator),
213213
On(HasInternalRole("target")).Roles(AugmentedAssignmentVariable),
214214
On(HasInternalRole("value")).Roles(AugmentedAssignmentValue),
215215
),
216216

217217
On(HasInternalType(pyast.Expression)).Roles(Expression),
218218
On(HasInternalType(pyast.Expr)).Roles(Expression),
219-
On(HasInternalType(pyast.Name)).Roles(SimpleIdentifier),
220-
On(HasInternalType(pyast.Attribute)).Roles(QualifiedIdentifier),
219+
On(HasInternalType(pyast.Name)).Roles(SimpleIdentifier, Expression),
220+
On(HasInternalType(pyast.Attribute)).Roles(QualifiedIdentifier, Expression),
221221

222222
// Comments and non significative whitespace
223223
On(HasInternalType(pyast.SameLineNoops)).Roles(Comment),
@@ -229,24 +229,24 @@ var AnnotationRules = On(Any).Self(
229229
),
230230

231231
// TODO: check what Constant nodes are generated in the python AST and improve this
232-
On(HasInternalType(pyast.Constant)).Roles(SimpleIdentifier),
233-
On(HasInternalType(pyast.Try)).Roles(Try).Children(
232+
On(HasInternalType(pyast.Constant)).Roles(SimpleIdentifier, Expression),
233+
On(HasInternalType(pyast.Try)).Roles(Try, Statement).Children(
234234
On(HasInternalRole("body")).Roles(TryBody),
235235
On(HasInternalRole("finalbody")).Roles(TryFinally),
236236
On(HasInternalRole("handlers")).Roles(TryCatch),
237237
On(HasInternalRole("orelse")).Roles(IfElse),
238238
),
239-
On(HasInternalType(pyast.TryExcept)).Roles(TryCatch), // py2
240-
On(HasInternalType(pyast.ExceptHandler)).Roles(TryCatch), // py3
241-
On(HasInternalType(pyast.TryFinally)).Roles(TryFinally),
242-
On(HasInternalType(pyast.Raise)).Roles(Throw),
239+
On(HasInternalType(pyast.TryExcept)).Roles(TryCatch, Statement), // py2
240+
On(HasInternalType(pyast.ExceptHandler)).Roles(TryCatch, Statement), // py3
241+
On(HasInternalType(pyast.TryFinally)).Roles(TryFinally, Statement),
242+
On(HasInternalType(pyast.Raise)).Roles(Throw, Statement),
243243
// FIXME: review, add path for the body and items childs
244244
// FIXME: withitem on Python to RAII on a resource and can aditionally create and alias on it,
245245
// both of which currently doesn't have representation in the UAST
246-
On(HasInternalType(pyast.With)).Roles(BlockScope),
247-
On(HasInternalType(pyast.Return)).Roles(Return),
248-
On(HasInternalType(pyast.Break)).Roles(Break),
249-
On(HasInternalType(pyast.Continue)).Roles(Continue),
246+
On(HasInternalType(pyast.With)).Roles(BlockScope, Statement),
247+
On(HasInternalType(pyast.Return)).Roles(Return, Statement),
248+
On(HasInternalType(pyast.Break)).Roles(Break, Statement),
249+
On(HasInternalType(pyast.Continue)).Roles(Continue, Statement),
250250
// FIXME: IfCondition bodies in Python take the form:
251251
// 1 < a < 10
252252
// - left (internalRole): 1 (first element)
@@ -260,60 +260,58 @@ var AnnotationRules = On(Any).Self(
260260
// and SDK feature to mix lists (also needed for default and keyword arguments and
261261
// boolean operators).
262262
// "If that sounds awkward is because it is" (their words)
263-
On(HasInternalType(pyast.If)).Roles(If).Children(
263+
On(HasInternalType(pyast.If)).Roles(If, Statement).Children(
264264
On(HasInternalType("If.body")).Roles(IfBody),
265265
On(HasInternalRole("test")).Roles(IfCondition),
266266
On(HasInternalType("If.orelse")).Roles(IfElse),
267-
On(HasInternalType(pyast.Compare)).Roles(BinaryExpression).Children(
267+
On(HasInternalType(pyast.Compare)).Roles(BinaryExpression, Expression).Children(
268268
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
269269
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
270270
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
271271
),
272272
),
273-
On(HasInternalType(pyast.IfExp)).Roles(If, Expression).Children(
273+
On(HasInternalType(pyast.IfExp)).Roles(If, Expression, Statement).Children(
274274
// These are used on ifexpressions (a = 1 if x else 2)
275275
On(HasInternalRole("body")).Roles(IfBody),
276276
On(HasInternalRole("test")).Roles(IfCondition),
277277
On(HasInternalRole("orelse")).Roles(IfElse),
278278
),
279-
// One liner if, like a normal If but it will be inside an Assign (like the ternary if in C)
280-
On(HasInternalType(pyast.IfExp)).Roles(If),
281-
On(HasInternalType(pyast.Import)).Roles(ImportDeclaration),
282-
On(HasInternalType(pyast.ImportFrom)).Roles(ImportDeclaration),
279+
On(HasInternalType(pyast.Import)).Roles(ImportDeclaration, Statement),
280+
On(HasInternalType(pyast.ImportFrom)).Roles(ImportDeclaration, Statement),
283281
On(HasInternalType(pyast.Alias)).Roles(ImportAlias, SimpleIdentifier),
284-
On(HasInternalType(pyast.ClassDef)).Roles(TypeDeclaration, SimpleIdentifier).Children(
282+
On(HasInternalType(pyast.ClassDef)).Roles(TypeDeclaration, SimpleIdentifier, Statement).Children(
285283
On(HasInternalType("ClassDef.body")).Roles(TypeDeclarationBody),
286284
On(HasInternalType("ClassDef.bases")).Roles(TypeDeclarationBases),
287285
),
288286

289-
On(HasInternalType(pyast.For)).Roles(ForEach).Children(
287+
On(HasInternalType(pyast.For)).Roles(ForEach, Statement).Children(
290288
On(HasInternalType("For.body")).Roles(ForBody),
291289
On(HasInternalRole("iter")).Roles(ForExpression),
292290
On(HasInternalRole("target")).Roles(ForUpdate),
293291
On(HasInternalType("For.orelse")).Roles(IfElse),
294292
),
295-
On(HasInternalType(pyast.While)).Roles(While).Children(
293+
On(HasInternalType(pyast.While)).Roles(While, Statement).Children(
296294
On(HasInternalType("While.body")).Roles(WhileBody),
297295
On(HasInternalRole("test")).Roles(WhileCondition),
298296
On(HasInternalType("While.orelse")).Roles(IfElse),
299297
),
300-
On(HasInternalType(pyast.Pass)).Roles(Noop),
301-
On(HasInternalType(pyast.Num)).Roles(NumberLiteral),
298+
On(HasInternalType(pyast.Pass)).Roles(Noop, Statement),
299+
On(HasInternalType(pyast.Num)).Roles(NumberLiteral, Expression),
302300
// FIXME: this is the annotated assignment (a: annotation = 3) not exactly Assignment
303301
// it also lacks AssignmentValue and AssignmentVariable (see how to add them)
304-
On(HasInternalType(pyast.AnnAssign)).Roles(Assignment),
305-
On(HasInternalType(pyast.Assert)).Roles(Assert),
302+
On(HasInternalType(pyast.AnnAssign)).Roles(Assignment, Statement),
303+
On(HasInternalType(pyast.Assert)).Roles(Assert, Statement),
306304

307305
// These are AST nodes in Python2 but we convert them to functions in the UAST like
308306
// they are in Python3
309-
On(HasInternalType(pyast.Exec)).Roles(Call).Children(
307+
On(HasInternalType(pyast.Exec)).Roles(Call, Expression).Children(
310308
On(HasInternalRole("body")).Roles(CallPositionalArgument),
311309
On(HasInternalRole("globals")).Roles(CallPositionalArgument),
312310
On(HasInternalRole("locals")).Roles(CallPositionalArgument),
313311
),
314312
// Repr already comes as a Call \o/
315313
// Print as a function too.
316-
On(HasInternalType(pyast.Print)).Roles(Call, CallCallee, SimpleIdentifier).Children(
314+
On(HasInternalType(pyast.Print)).Roles(Call, CallCallee, SimpleIdentifier, Expression).Children(
317315
On(HasInternalRole("dest")).Roles(CallPositionalArgument),
318316
On(HasInternalRole("nl")).Roles(CallPositionalArgument),
319317
On(HasInternalRole("values")).Roles(CallPositionalArgument).Children(
@@ -336,8 +334,8 @@ var AnnotationRules = On(Any).Self(
336334
// List/Map/Set comprehensions. We map the "for x in y" to ForEach roles and the
337335
// "if something" to If* roles. FIXME: missing the top comprehension roles in the UAST, change
338336
// once they've been merged
339-
On(HasInternalType(pyast.Comprehension)).Roles(ForEach).Children(
340-
On(HasInternalRole("iter")).Roles(ForUpdate),
337+
On(HasInternalType(pyast.Comprehension)).Roles(ForEach, Expression).Children(
338+
On(HasInternalRole("iter")).Roles(ForUpdate, Statement),
341339
On(HasInternalRole("target")).Roles(ForExpression),
342340
// FIXME: see the comment on IfCondition above
343341
On(HasInternalType(pyast.Compare)).Roles(IfCondition, BinaryExpression).Children(

‎tests/aritmeticops.py.uast‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ Module {
2727
. . . }
2828
. . . Children: {
2929
. . . . 0: BinOp {
30-
. . . . . Roles: BinaryExpression
30+
. . . . . Roles: BinaryExpression,Expression
3131
. . . . . StartPosition: {
3232
. . . . . . Offset: 0
3333
. . . . . . Line: 1
@@ -40,7 +40,7 @@ Module {
4040
. . . . . }
4141
. . . . . Children: {
4242
. . . . . . 0: NumLiteral {
43-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
43+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
4444
. . . . . . . TOKEN "1"
4545
. . . . . . . StartPosition: {
4646
. . . . . . . . Offset: 0
@@ -69,7 +69,7 @@ Module {
6969
. . . . . . . }
7070
. . . . . . }
7171
. . . . . . 2: NumLiteral {
72-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
72+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
7373
. . . . . . . TOKEN "2"
7474
. . . . . . . StartPosition: {
7575
. . . . . . . . Offset: 2
@@ -101,7 +101,7 @@ Module {
101101
. . . }
102102
. . . Children: {
103103
. . . . 0: BinOp {
104-
. . . . . Roles: BinaryExpression
104+
. . . . . Roles: BinaryExpression,Expression
105105
. . . . . StartPosition: {
106106
. . . . . . Offset: 4
107107
. . . . . . Line: 2
@@ -114,7 +114,7 @@ Module {
114114
. . . . . }
115115
. . . . . Children: {
116116
. . . . . . 0: NumLiteral {
117-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
117+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
118118
. . . . . . . TOKEN "1"
119119
. . . . . . . StartPosition: {
120120
. . . . . . . . Offset: 4
@@ -143,7 +143,7 @@ Module {
143143
. . . . . . . }
144144
. . . . . . }
145145
. . . . . . 2: NumLiteral {
146-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
146+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
147147
. . . . . . . TOKEN "2"
148148
. . . . . . . StartPosition: {
149149
. . . . . . . . Offset: 6
@@ -175,7 +175,7 @@ Module {
175175
. . . }
176176
. . . Children: {
177177
. . . . 0: BinOp {
178-
. . . . . Roles: BinaryExpression
178+
. . . . . Roles: BinaryExpression,Expression
179179
. . . . . StartPosition: {
180180
. . . . . . Offset: 8
181181
. . . . . . Line: 3
@@ -188,7 +188,7 @@ Module {
188188
. . . . . }
189189
. . . . . Children: {
190190
. . . . . . 0: NumLiteral {
191-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
191+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
192192
. . . . . . . TOKEN "1"
193193
. . . . . . . StartPosition: {
194194
. . . . . . . . Offset: 8
@@ -217,7 +217,7 @@ Module {
217217
. . . . . . . }
218218
. . . . . . }
219219
. . . . . . 2: NumLiteral {
220-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
220+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
221221
. . . . . . . TOKEN "2"
222222
. . . . . . . StartPosition: {
223223
. . . . . . . . Offset: 10
@@ -249,7 +249,7 @@ Module {
249249
. . . }
250250
. . . Children: {
251251
. . . . 0: BinOp {
252-
. . . . . Roles: BinaryExpression
252+
. . . . . Roles: BinaryExpression,Expression
253253
. . . . . StartPosition: {
254254
. . . . . . Offset: 12
255255
. . . . . . Line: 4
@@ -262,7 +262,7 @@ Module {
262262
. . . . . }
263263
. . . . . Children: {
264264
. . . . . . 0: NumLiteral {
265-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
265+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
266266
. . . . . . . TOKEN "1"
267267
. . . . . . . StartPosition: {
268268
. . . . . . . . Offset: 12
@@ -291,7 +291,7 @@ Module {
291291
. . . . . . . }
292292
. . . . . . }
293293
. . . . . . 2: NumLiteral {
294-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
294+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
295295
. . . . . . . TOKEN "2"
296296
. . . . . . . StartPosition: {
297297
. . . . . . . . Offset: 14
@@ -323,7 +323,7 @@ Module {
323323
. . . }
324324
. . . Children: {
325325
. . . . 0: BinOp {
326-
. . . . . Roles: BinaryExpression
326+
. . . . . Roles: BinaryExpression,Expression
327327
. . . . . StartPosition: {
328328
. . . . . . Offset: 16
329329
. . . . . . Line: 5
@@ -336,7 +336,7 @@ Module {
336336
. . . . . }
337337
. . . . . Children: {
338338
. . . . . . 0: NumLiteral {
339-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
339+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
340340
. . . . . . . TOKEN "1"
341341
. . . . . . . StartPosition: {
342342
. . . . . . . . Offset: 16
@@ -365,7 +365,7 @@ Module {
365365
. . . . . . . }
366366
. . . . . . }
367367
. . . . . . 2: NumLiteral {
368-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
368+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
369369
. . . . . . . TOKEN "2"
370370
. . . . . . . StartPosition: {
371371
. . . . . . . . Offset: 19
@@ -397,7 +397,7 @@ Module {
397397
. . . }
398398
. . . Children: {
399399
. . . . 0: BinOp {
400-
. . . . . Roles: BinaryExpression
400+
. . . . . Roles: BinaryExpression,Expression
401401
. . . . . StartPosition: {
402402
. . . . . . Offset: 21
403403
. . . . . . Line: 6
@@ -410,7 +410,7 @@ Module {
410410
. . . . . }
411411
. . . . . Children: {
412412
. . . . . . 0: NumLiteral {
413-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
413+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
414414
. . . . . . . TOKEN "1"
415415
. . . . . . . StartPosition: {
416416
. . . . . . . . Offset: 21
@@ -439,7 +439,7 @@ Module {
439439
. . . . . . . }
440440
. . . . . . }
441441
. . . . . . 2: NumLiteral {
442-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
442+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
443443
. . . . . . . TOKEN "2"
444444
. . . . . . . StartPosition: {
445445
. . . . . . . . Offset: 23
@@ -471,7 +471,7 @@ Module {
471471
. . . }
472472
. . . Children: {
473473
. . . . 0: BinOp {
474-
. . . . . Roles: BinaryExpression
474+
. . . . . Roles: BinaryExpression,Expression
475475
. . . . . StartPosition: {
476476
. . . . . . Offset: 25
477477
. . . . . . Line: 7
@@ -484,7 +484,7 @@ Module {
484484
. . . . . }
485485
. . . . . Children: {
486486
. . . . . . 0: NumLiteral {
487-
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral
487+
. . . . . . . Roles: BinaryExpressionLeft,NumberLiteral,Expression
488488
. . . . . . . TOKEN "1"
489489
. . . . . . . StartPosition: {
490490
. . . . . . . . Offset: 25
@@ -513,7 +513,7 @@ Module {
513513
. . . . . . . }
514514
. . . . . . }
515515
. . . . . . 2: NumLiteral {
516-
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral
516+
. . . . . . . Roles: BinaryExpressionRight,NumberLiteral,Expression
517517
. . . . . . . TOKEN "2"
518518
. . . . . . . StartPosition: {
519519
. . . . . . . . Offset: 28

0 commit comments

Comments
 (0)