Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit 0a9f716

Browse files
juanjuxjuanjux
authored
Merge pull request #69 from juanjux/fix/issue65
Many annotation fixes, pydetector dependency upgraded
2 parents 807768e + e600d0f commit 0a9f716

File tree

116 files changed

+4202
-802
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

116 files changed

+4202
-802
lines changed

‎ANNOTATION.md‎

Lines changed: 186 additions & 0 deletions
Large diffs are not rendered by default.

‎Dockerfile.tpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ MAINTAINER source{d}
33

44
ARG DEVDEPS=native/dev_deps
55
ARG CONTAINER_DEVDEPS=/tmp/dev_deps
6-
ARG PYDETECTOR_VER=0.11.3
6+
ARG PYDETECTOR_VER=0.12.0
77

88
RUN apk add --no-cache --update python python3 py-pip py2-pip git
99

‎driver/normalizer/annotation.go‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -207,17 +207,19 @@ var AnnotationRules = On(Any).Self(
207207
// TODO: check what Constant nodes are generated in the python AST and improve this
208208
On(HasInternalType(pyast.Constant)).Roles(SimpleIdentifier, Expression),
209209
On(HasInternalType(pyast.Try)).Roles(Try, Statement).Children(
210-
On(HasInternalRole("body")).Roles(TryBody),
211-
On(HasInternalRole("finalbody")).Roles(TryFinally),
212-
On(HasInternalRole("handlers")).Roles(TryCatch),
213-
On(HasInternalRole("orelse")).Roles(IfElse),
210+
On(HasInternalType("Try.body")).Roles(TryBody),
211+
On(HasInternalType("Try.finalbody")).Roles(TryFinally),
212+
On(HasInternalType("Try.handlers")).Roles(TryCatch),
213+
On(HasInternalType("Try.orelse")).Roles(IfElse),
214214
),
215215
On(HasInternalType(pyast.TryExcept)).Roles(TryCatch, Statement), // py2
216216
On(HasInternalType(pyast.ExceptHandler)).Roles(TryCatch, Statement), // py3
217217
On(HasInternalType(pyast.TryFinally)).Roles(TryFinally, Statement),
218218
On(HasInternalType(pyast.Raise)).Roles(Throw, Statement),
219219
// FIXME: review, add path for the body and items childs
220220
On(HasInternalType(pyast.With)).Roles(BlockScope, Statement),
221+
On(HasInternalType("With.body")).Roles(BlockScope, Expression, Incomplete),
222+
On(HasInternalType("With.items")).Roles(SimpleIdentifier, Expression, Incomplete),
221223
On(HasInternalType(pyast.AsyncWith)).Roles(BlockScope, Statement, Incomplete),
222224
On(HasInternalType(pyast.Withitem)).Roles(SimpleIdentifier, Incomplete),
223225
On(HasInternalType(pyast.Return)).Roles(Return, Statement),
@@ -236,15 +238,15 @@ var AnnotationRules = On(Any).Self(
236238
// and SDK feature to mix lists (also needed for default and keyword arguments and
237239
// boolean operators).
238240
// "If that sounds awkward is because it is" (their words)
241+
On(HasInternalType(pyast.Compare)).Roles(BinaryExpression, Expression).Children(
242+
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
243+
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
244+
),
245+
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
239246
On(HasInternalType(pyast.If)).Roles(If, Statement).Children(
240247
On(HasInternalType("If.body")).Roles(IfBody),
241248
On(HasInternalRole("test")).Roles(IfCondition),
242249
On(HasInternalType("If.orelse")).Roles(IfElse),
243-
On(HasInternalType(pyast.Compare)).Roles(BinaryExpression, Expression).Children(
244-
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
245-
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
246-
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
247-
),
248250
),
249251
On(HasInternalType(pyast.IfExp)).Roles(If, Expression).Children(
250252
// These are used on ifexpressions (a = 1 if x else 2)
@@ -263,7 +265,9 @@ var AnnotationRules = On(Any).Self(
263265
On(HasInternalType(pyast.ClassDef)).Roles(TypeDeclaration, SimpleIdentifier, Statement).Children(
264266
On(HasInternalType("ClassDef.body")).Roles(TypeDeclarationBody),
265267
On(HasInternalType("ClassDef.bases")).Roles(TypeDeclarationBases),
266-
),
268+
On(HasInternalType("ClassDef.keywords")).Roles(Incomplete).Children(
269+
On(HasInternalType(pyast.Keyword)).Roles(SimpleIdentifier, Incomplete),
270+
)),
267271

268272
On(HasInternalType(pyast.For)).Roles(ForEach, Statement).Children(
269273
On(HasInternalType("For.body")).Roles(ForBody),
@@ -327,12 +331,11 @@ var AnnotationRules = On(Any).Self(
327331
// FIXME: see the comment on IfCondition above
328332
On(HasInternalType(pyast.Compare)).Roles(IfCondition, BinaryExpression).Children(
329333
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
330-
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
331334
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
332335
),
333336
),
334337
On(HasInternalType(pyast.ListComp)).Roles(ListLiteral, Expression, Incomplete),
335-
On(HasInternalType(pyast.SetComp)).Roles(MapLiteral, Expression, Incomplete),
338+
On(HasInternalType(pyast.DictComp)).Roles(MapLiteral, Expression, Incomplete),
336339
On(HasInternalType(pyast.SetComp)).Roles(SetLiteral, Expression, Incomplete),
337340

338341
On(HasInternalType(pyast.Delete)).Roles(Statement, Incomplete),

‎driver/normalizer/parser.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,7 @@ var ToNoder = &native.ObjectToNoder{
6060
"For": {"body": true, "orelse": true},
6161
"AsyncFor": {"body": true, "orelse": true},
6262
"While": {"body": true, "orelse": true},
63-
// FIXME: check if promotion is needed in this case
64-
"Compare": {"comparators": true, "ops": true},
65-
"Import": {"names": true},
66-
"ImportFrom": {"names": true},
63+
"Compare": {"comparators": true, "ops": true},
6764
// FIXME: check call.keywords
6865
//"Call" : { "args": true, "keywords": true},
6966
"With": {"body": true, "items": true},
@@ -73,9 +70,6 @@ var ToNoder = &native.ObjectToNoder{
7370
"Try": {"body": true, "orelse": true, "finalbody": true, "handlers": true},
7471
"Raise": {"args": true},
7572
"ClassDef": {"body": true, "bases": true, "decorator_list": true, "keywords": true},
76-
"ListComp": {"generators": true},
77-
"ListComp.generators": {"ifs": true},
78-
"ListComp.generators.ifs": {"comparators": true, "ops": true},
7973
},
8074
PromotedPropertyStrings: map[string]map[string]bool{
8175
"alias": {"asname": true},

‎native/python_package/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
pydetector-bblfsh==0.11.3
1+
pydetector-bblfsh==0.12.0
22
-e git+git://github.com/python/mypy.git@0bb2d1680e8b9522108b38d203cb73021a617e64#egg=mypy-lang
33
typed-ast==1.0.1

‎native/python_package/setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
]
3232
},
3333
install_requires=[
34-
"pydetector-bblfsh==0.11.3"
34+
"pydetector-bblfsh==0.12.0"
3535
],
3636
classifiers=[
3737
"Development Status :: 4 - Beta",

‎tests/booleanop.py.uast‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ Module {
3131
. . . }
3232
. . . Children: {
3333
. . . . 0: Compare {
34-
. . . . . Roles: Unannotated
34+
. . . . . Roles: BinaryExpression,Expression
3535
. . . . . StartPosition: {
3636
. . . . . . Offset: 0
3737
. . . . . . Line: 1
@@ -47,7 +47,7 @@ Module {
4747
. . . . . }
4848
. . . . . Children: {
4949
. . . . . . 0: Compare.comparators {
50-
. . . . . . . Roles: Unannotated
50+
. . . . . . . Roles: BinaryExpressionRight
5151
. . . . . . . Properties: {
5252
. . . . . . . . promotedPropertyList: true
5353
. . . . . . . }
@@ -72,7 +72,7 @@ Module {
7272
. . . . . . . }
7373
. . . . . . }
7474
. . . . . . 1: NumLiteral {
75-
. . . . . . . Roles: NumberLiteral,Expression
75+
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
7676
. . . . . . . TOKEN "1"
7777
. . . . . . . StartPosition: {
7878
. . . . . . . . Offset: 0
@@ -90,7 +90,7 @@ Module {
9090
. . . . . . . }
9191
. . . . . . }
9292
. . . . . . 2: Compare.ops {
93-
. . . . . . . Roles: Unannotated
93+
. . . . . . . Roles: BinaryExpressionOp
9494
. . . . . . . Properties: {
9595
. . . . . . . . promotedPropertyList: true
9696
. . . . . . . }
@@ -132,7 +132,7 @@ Module {
132132
. . . }
133133
. . . Children: {
134134
. . . . 0: Compare {
135-
. . . . . Roles: Unannotated
135+
. . . . . Roles: BinaryExpression,Expression
136136
. . . . . StartPosition: {
137137
. . . . . . Offset: 7
138138
. . . . . . Line: 2
@@ -148,7 +148,7 @@ Module {
148148
. . . . . }
149149
. . . . . Children: {
150150
. . . . . . 0: Compare.comparators {
151-
. . . . . . . Roles: Unannotated
151+
. . . . . . . Roles: BinaryExpressionRight
152152
. . . . . . . Properties: {
153153
. . . . . . . . promotedPropertyList: true
154154
. . . . . . . }
@@ -173,7 +173,7 @@ Module {
173173
. . . . . . . }
174174
. . . . . . }
175175
. . . . . . 1: NumLiteral {
176-
. . . . . . . Roles: NumberLiteral,Expression
176+
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
177177
. . . . . . . TOKEN "1"
178178
. . . . . . . StartPosition: {
179179
. . . . . . . . Offset: 7
@@ -191,7 +191,7 @@ Module {
191191
. . . . . . . }
192192
. . . . . . }
193193
. . . . . . 2: Compare.ops {
194-
. . . . . . . Roles: Unannotated
194+
. . . . . . . Roles: BinaryExpressionOp
195195
. . . . . . . Properties: {
196196
. . . . . . . . promotedPropertyList: true
197197
. . . . . . . }
@@ -304,7 +304,7 @@ Module {
304304
. . . }
305305
. . . Children: {
306306
. . . . 0: Compare {
307-
. . . . . Roles: Unannotated
307+
. . . . . Roles: BinaryExpression,Expression
308308
. . . . . StartPosition: {
309309
. . . . . . Offset: 20
310310
. . . . . . Line: 4
@@ -320,7 +320,7 @@ Module {
320320
. . . . . }
321321
. . . . . Children: {
322322
. . . . . . 0: Compare.comparators {
323-
. . . . . . . Roles: Unannotated
323+
. . . . . . . Roles: BinaryExpressionRight
324324
. . . . . . . Properties: {
325325
. . . . . . . . promotedPropertyList: true
326326
. . . . . . . }
@@ -362,7 +362,7 @@ Module {
362362
. . . . . . . }
363363
. . . . . . }
364364
. . . . . . 1: NumLiteral {
365-
. . . . . . . Roles: NumberLiteral,Expression
365+
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
366366
. . . . . . . TOKEN "1"
367367
. . . . . . . StartPosition: {
368368
. . . . . . . . Offset: 20
@@ -380,7 +380,7 @@ Module {
380380
. . . . . . . }
381381
. . . . . . }
382382
. . . . . . 2: Compare.ops {
383-
. . . . . . . Roles: Unannotated
383+
. . . . . . . Roles: BinaryExpressionOp
384384
. . . . . . . Properties: {
385385
. . . . . . . . promotedPropertyList: true
386386
. . . . . . . }
@@ -436,7 +436,7 @@ Module {
436436
. . . }
437437
. . . Children: {
438438
. . . . 0: Compare {
439-
. . . . . Roles: Unannotated
439+
. . . . . Roles: BinaryExpression,Expression
440440
. . . . . StartPosition: {
441441
. . . . . . Offset: 32
442442
. . . . . . Line: 5
@@ -452,7 +452,7 @@ Module {
452452
. . . . . }
453453
. . . . . Children: {
454454
. . . . . . 0: Compare.comparators {
455-
. . . . . . . Roles: Unannotated
455+
. . . . . . . Roles: BinaryExpressionRight
456456
. . . . . . . Properties: {
457457
. . . . . . . . promotedPropertyList: true
458458
. . . . . . . }
@@ -494,7 +494,7 @@ Module {
494494
. . . . . . . }
495495
. . . . . . }
496496
. . . . . . 1: NumLiteral {
497-
. . . . . . . Roles: NumberLiteral,Expression
497+
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
498498
. . . . . . . TOKEN "1"
499499
. . . . . . . StartPosition: {
500500
. . . . . . . . Offset: 32
@@ -512,7 +512,7 @@ Module {
512512
. . . . . . . }
513513
. . . . . . }
514514
. . . . . . 2: Compare.ops {
515-
. . . . . . . Roles: Unannotated
515+
. . . . . . . Roles: BinaryExpressionOp
516516
. . . . . . . Properties: {
517517
. . . . . . . . promotedPropertyList: true
518518
. . . . . . . }
@@ -568,7 +568,7 @@ Module {
568568
. . . }
569569
. . . Children: {
570570
. . . . 0: Compare {
571-
. . . . . Roles: Unannotated
571+
. . . . . Roles: BinaryExpression,Expression
572572
. . . . . StartPosition: {
573573
. . . . . . Offset: 42
574574
. . . . . . Line: 6
@@ -584,7 +584,7 @@ Module {
584584
. . . . . }
585585
. . . . . Children: {
586586
. . . . . . 0: Compare.comparators {
587-
. . . . . . . Roles: Unannotated
587+
. . . . . . . Roles: BinaryExpressionRight
588588
. . . . . . . Properties: {
589589
. . . . . . . . promotedPropertyList: true
590590
. . . . . . . }
@@ -626,7 +626,7 @@ Module {
626626
. . . . . . . }
627627
. . . . . . }
628628
. . . . . . 1: NumLiteral {
629-
. . . . . . . Roles: NumberLiteral,Expression
629+
. . . . . . . Roles: NumberLiteral,Expression,BinaryExpressionLeft
630630
. . . . . . . TOKEN "1"
631631
. . . . . . . StartPosition: {
632632
. . . . . . . . Offset: 42
@@ -644,7 +644,7 @@ Module {
644644
. . . . . . . }
645645
. . . . . . }
646646
. . . . . . 2: Compare.ops {
647-
. . . . . . . Roles: Unannotated
647+
. . . . . . . Roles: BinaryExpressionOp
648648
. . . . . . . Properties: {
649649
. . . . . . . . promotedPropertyList: true
650650
. . . . . . . }

‎tests/classdef.py.native‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -160,13 +160,15 @@
160160
{
161161
"ast_type": "Pass",
162162
"col_offset": 9,
163+
"end_col_offset": 12,
164+
"end_lineno": 7,
163165
"lineno": 7
164166
}
165167
],
166168
"col_offset": 9,
167169
"decorator_list": [],
168-
"end_col_offset": 25,
169-
"end_lineno": 6,
170+
"end_col_offset": 12,
171+
"end_lineno": 7,
170172
"lineno": 6,
171173
"name": "method",
172174
"returns": null
@@ -280,7 +282,7 @@
280282
"kw_defaults": [],
281283
"kwarg": null,
282284
"kwonlyargs": [],
283-
"lineno": 13,
285+
"lineno": 14,
284286
"vararg": null
285287
},
286288
"ast_type": "FunctionDef",
@@ -322,7 +324,7 @@
322324
}
323325
}
324326
],
325-
"col_offset": 5,
327+
"col_offset": 9,
326328
"decorator_list": [
327329
{
328330
"ast_type": "Attribute",
@@ -345,7 +347,7 @@
345347
],
346348
"end_col_offset": 22,
347349
"end_lineno": 15,
348-
"lineno": 13,
350+
"lineno": 14,
349351
"name": "a",
350352
"returns": null
351353
}

‎tests/classdef.py.uast‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -233,9 +233,9 @@ Module {
233233
. . . . . . . . Col: 9
234234
. . . . . . . }
235235
. . . . . . . EndPosition: {
236-
. . . . . . . . Offset: 97
237-
. . . . . . . . Line: 6
238-
. . . . . . . . Col: 25
236+
. . . . . . . . Offset: 112
237+
. . . . . . . . Line: 7
238+
. . . . . . . . Col: 12
239239
. . . . . . . }
240240
. . . . . . . Properties: {
241241
. . . . . . . . returns: <nil>
@@ -344,6 +344,11 @@ Module {
344344
. . . . . . . . . . . . Line: 7
345345
. . . . . . . . . . . . Col: 9
346346
. . . . . . . . . . . }
347+
. . . . . . . . . . . EndPosition: {
348+
. . . . . . . . . . . . Offset: 112
349+
. . . . . . . . . . . . Line: 7
350+
. . . . . . . . . . . . Col: 12
351+
. . . . . . . . . . . }
347352
. . . . . . . . . . }
348353
. . . . . . . . . }
349354
. . . . . . . . }
@@ -504,9 +509,9 @@ Module {
504509
. . . . . . . Roles: FunctionDeclaration,FunctionDeclarationName,SimpleIdentifier
505510
. . . . . . . TOKEN "a"
506511
. . . . . . . StartPosition: {
507-
. . . . . . . . Offset: 170
508-
. . . . . . . . Line: 13
509-
. . . . . . . . Col: 5
512+
. . . . . . . . Offset: 188
513+
. . . . . . . . Line: 14
514+
. . . . . . . . Col: 9
510515
. . . . . . . }
511516
. . . . . . . EndPosition: {
512517
. . . . . . . . Offset: 218
@@ -520,8 +525,8 @@ Module {
520525
. . . . . . . . 0: arguments {
521526
. . . . . . . . . Roles: FunctionDeclarationArgument,Incomplete
522527
. . . . . . . . . StartPosition: {
523-
. . . . . . . . . . Offset: 177
524-
. . . . . . . . . . Line: 13
528+
. . . . . . . . . . Offset: 191
529+
. . . . . . . . . . Line: 14
525530
. . . . . . . . . . Col: 12
526531
. . . . . . . . . }
527532
. . . . . . . . . EndPosition: {

0 commit comments

Comments
 (0)