Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit 136e550

Browse files
juanjuxjuanjux
authored
Merge pull request #56 from juanjux/fix/complete_annotations
Completing annotations
2 parents 6b6294f + f79a023 commit 136e550

19 files changed

+556
-67
lines changed

‎ANNOTATION.md‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -53,26 +53,29 @@
5353
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\] | FunctionDeclaration, FunctionDeclarationName, SimpleIdentifier |
5454
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='FunctionDef\.decorator\_list'\] | Call, SimpleIdentifier, Incomplete |
5555
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='FunctionDef\.body'\] | FunctionDeclarationBody |
56-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='arguments'\] | FunctionDeclarationArgument |
56+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='arguments'\] | FunctionDeclarationArgument, Incomplete |
5757
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='args'\] | FunctionDeclarationArgument, FunctionDeclarationArgumentName |
5858
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='vararg'\] | FunctionDeclarationArgument, FunctionDeclarationVarArgsList, FunctionDeclarationArgumentName |
5959
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='kwarg'\] | FunctionDeclarationArgument, FunctionDeclarationVarArgsList, FunctionDeclarationArgumentName, Incomplete |
60-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.defaults'\] | FunctionDeclarationArgumentDefaultValue |
60+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.defaults'\] | FunctionDeclarationArgumentDefaultValue, Incomplete |
61+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='FunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.keywords'\] | FunctionDeclarationArgumentDefaultValue, Incomplete |
6162
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\] | FunctionDeclaration, FunctionDeclarationName, SimpleIdentifier, Incomplete |
6263
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='AsyncFunctionDef\.decorator\_list'\] | Call, SimpleIdentifier, Incomplete |
6364
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='AsyncFunctionDef\.body'\] | FunctionDeclarationBody |
64-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='arguments'\] | FunctionDeclarationArgument |
65+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='arguments'\] | FunctionDeclarationArgument, Incomplete |
6566
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='args'\] | FunctionDeclarationArgument, FunctionDeclarationArgumentName |
6667
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='vararg'\] | FunctionDeclarationArgument, FunctionDeclarationVarArgsList, FunctionDeclarationArgumentName |
6768
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='kwarg'\] | FunctionDeclarationArgument, FunctionDeclarationVarArgsList, FunctionDeclarationArgumentName, Incomplete |
68-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.defaults'\] | FunctionDeclarationArgumentDefaultValue |
69+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.defaults'\] | FunctionDeclarationArgumentDefaultValue, Incomplete |
70+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AsyncFunctionDef'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.keywords'\] | FunctionDeclarationArgumentDefaultValue, Incomplete |
6971
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\] | FunctionDeclaration, SimpleIdentifier, Expression, Incomplete |
7072
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='Lambda\.body'\] | FunctionDeclarationBody |
71-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='arguments'\] | FunctionDeclarationArgument |
73+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='arguments'\] | FunctionDeclarationArgument, Incomplete |
7274
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='args'\] | FunctionDeclarationArgument, FunctionDeclarationArgumentName |
7375
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='vararg'\] | FunctionDeclarationArgument, FunctionDeclarationVarArgsList, FunctionDeclarationArgumentName |
7476
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='arguments'\]/\*\[@internalRole\]\[@internalRole='kwarg'\] | FunctionDeclarationArgument, FunctionDeclarationVarArgsList, FunctionDeclarationArgumentName, Incomplete |
75-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.defaults'\] | FunctionDeclarationArgumentDefaultValue |
77+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.defaults'\] | FunctionDeclarationArgumentDefaultValue, Incomplete |
78+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Lambda'\]/\*\[@InternalType='arguments'\]/\*\[@InternalType='arguments\.keywords'\] | FunctionDeclarationArgumentDefaultValue, Incomplete |
7679
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Attribute'\] | SimpleIdentifier, Expression |
7780
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Attribute'\]/\*\[@InternalType='Name'\] | QualifiedIdentifier |
7881
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Call'\] | Call, Expression |
@@ -147,7 +150,6 @@
147150
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='While'\]/\*\[@InternalType='While\.orelse'\] | IfElse |
148151
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Pass'\] | Noop, Statement |
149152
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Num'\] | NumberLiteral, Expression |
150-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AnnAssign'\] | Assignment, Statement |
151153
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Assert'\] | Assert, Statement |
152154
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Exec'\] | Call, Expression |
153155
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Exec'\]/\*\[@internalRole\]\[@internalRole='body'\] | CallPositionalArgument |
@@ -158,7 +160,8 @@
158160
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Print'\]/\*\[@internalRole\]\[@internalRole='nl'\] | CallPositionalArgument |
159161
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Print'\]/\*\[@internalRole\]\[@internalRole='values'\] | CallPositionalArgument |
160162
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Print'\]/\*\[@internalRole\]\[@internalRole='values'\]/\*\[\*\] | CallPositionalArgument |
161-
| /self::\*\[@InternalType='Module'\]//\*\[@internalRole\]\[@internalRole='annotation'\] | Comment, Incomplete |
163+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='AnnAssign'\] | Assignment, Comment, Incomplete |
164+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Annotation'\] | Comment, Incomplete |
162165
| /self::\*\[@InternalType='Module'\]//\*\[@internalRole\]\[@internalRole='returns'\] | Comment, Incomplete |
163166
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Ellipsis'\] | SimpleIdentifier |
164167
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='comprehension'\] | ForEach, Expression |
@@ -178,3 +181,7 @@
178181
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Yield'\] | Return, Statement, Incomplete |
179182
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='YieldFrom'\] | Return, Statement, Incomplete |
180183
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Yield'\] | ListLiteral, Expression, Incomplete |
184+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Subscript'\] | Expression, Incomplete |
185+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Index'\] | Expression, Incomplete |
186+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Slice'\] | Expression, Incomplete |
187+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='ExtSlice'\] | Expression, Incomplete |

‎driver/normalizer/annotation.go‎

Lines changed: 26 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -26,31 +26,16 @@ https://greentreesnakes.readthedocs.io/en/latest/nodes.html
2626
/*
2727
Unmarked nodes or nodes needing new features from the SDK:
2828
29-
= PR 59:
30-
Subscript
31-
Index
32-
Slice
33-
ExtSlice
29+
These nodes would need a list-mix feature to convert parallel lists
30+
into list of parents and children:
3431
35-
= PR 79:
36-
arguments
37-
38-
= PR 112:
39-
AnnAssign
40-
annotation
41-
42-
=== No PR:
43-
44-
BoolOp collapsing: needs SDK features
45-
arguments.defaults: needs SDK features
32+
BoolOp
33+
arguments.defaults
4634
arguments.keywords: same
47-
48-
These also need SDK list-mix features:
49-
Compare.comparators
50-
Compare.ops
51-
IfCondition.left
35+
Compare.comparators
36+
Compare.ops
37+
IfCondition.left
5238
(see: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#Compare)
53-
5439
*/
5540

5641
var AnnotationRules = On(Any).Self(
@@ -128,7 +113,7 @@ var AnnotationRules = On(Any).Self(
128113
On(HasInternalType("FunctionDef.decorator_list")).Roles(Call, SimpleIdentifier, Incomplete),
129114
On(HasInternalType("FunctionDef.body")).Roles(FunctionDeclarationBody),
130115
// FIXME: change to FunctionDeclarationArgumentS once the PR has been merged
131-
On(HasInternalType("arguments")).Roles(FunctionDeclarationArgument).Children(
116+
On(HasInternalType(pyast.Arguments)).Roles(FunctionDeclarationArgument, Incomplete).Children(
132117
On(HasInternalRole("args")).Roles(FunctionDeclarationArgument, FunctionDeclarationArgumentName),
133118
On(HasInternalRole("vararg")).Roles(FunctionDeclarationArgument, FunctionDeclarationVarArgsList,
134119
FunctionDeclarationArgumentName),
@@ -142,15 +127,16 @@ var AnnotationRules = On(Any).Self(
142127
// args [a,b,c],
143128
// defaults [2,3]
144129
// TODO: create an issue for the SDK
145-
On(HasInternalType("arguments.defaults")).Roles(FunctionDeclarationArgumentDefaultValue),
130+
On(HasInternalType("arguments.defaults")).Roles(FunctionDeclarationArgumentDefaultValue, Incomplete),
131+
On(HasInternalType("arguments.keywords")).Roles(FunctionDeclarationArgumentDefaultValue, Incomplete),
146132
),
147133
),
148134
On(HasInternalType(pyast.AsyncFunctionDef)).Roles(FunctionDeclaration, FunctionDeclarationName, SimpleIdentifier,
149135
Incomplete).Children(
150136
On(HasInternalType("AsyncFunctionDef.decorator_list")).Roles(Call, SimpleIdentifier, Incomplete),
151137
On(HasInternalType("AsyncFunctionDef.body")).Roles(FunctionDeclarationBody),
152138
// FIXME: change to FunctionDeclarationArgumentS once the PR has been merged
153-
On(HasInternalType("arguments")).Roles(FunctionDeclarationArgument).Children(
139+
On(HasInternalType(pyast.Arguments)).Roles(FunctionDeclarationArgument, Incomplete).Children(
154140
On(HasInternalRole("args")).Roles(FunctionDeclarationArgument, FunctionDeclarationArgumentName),
155141
On(HasInternalRole("vararg")).Roles(FunctionDeclarationArgument, FunctionDeclarationVarArgsList,
156142
FunctionDeclarationArgumentName),
@@ -164,14 +150,15 @@ var AnnotationRules = On(Any).Self(
164150
// args [a,b,c],
165151
// defaults [2,3]
166152
// TODO: create an issue for the SDK
167-
On(HasInternalType("arguments.defaults")).Roles(FunctionDeclarationArgumentDefaultValue),
153+
On(HasInternalType("arguments.defaults")).Roles(FunctionDeclarationArgumentDefaultValue, Incomplete),
154+
On(HasInternalType("arguments.keywords")).Roles(FunctionDeclarationArgumentDefaultValue, Incomplete),
168155
),
169156
),
170157
On(HasInternalType(pyast.Lambda)).Roles(FunctionDeclaration, SimpleIdentifier, Expression,
171158
Incomplete).Children(
172159
On(HasInternalType("Lambda.body")).Roles(FunctionDeclarationBody),
173160
// FIXME: change to FunctionDeclarationArgumentS once the PR has been merged
174-
On(HasInternalType("arguments")).Roles(FunctionDeclarationArgument).Children(
161+
On(HasInternalType(pyast.Arguments)).Roles(FunctionDeclarationArgument, Incomplete).Children(
175162
On(HasInternalRole("args")).Roles(FunctionDeclarationArgument, FunctionDeclarationArgumentName),
176163
On(HasInternalRole("vararg")).Roles(FunctionDeclarationArgument, FunctionDeclarationVarArgsList,
177164
FunctionDeclarationArgumentName),
@@ -185,7 +172,8 @@ var AnnotationRules = On(Any).Self(
185172
// args [a,b,c],
186173
// defaults [2,3]
187174
// TODO: create an issue for the SDK
188-
On(HasInternalType("arguments.defaults")).Roles(FunctionDeclarationArgumentDefaultValue),
175+
On(HasInternalType("arguments.defaults")).Roles(FunctionDeclarationArgumentDefaultValue, Incomplete),
176+
On(HasInternalType("arguments.keywords")).Roles(FunctionDeclarationArgumentDefaultValue, Incomplete),
189177
),
190178
),
191179

@@ -220,6 +208,7 @@ var AnnotationRules = On(Any).Self(
220208
On(HasInternalRole("value")).Roles(AugmentedAssignmentValue),
221209
),
222210

211+
223212
On(HasInternalType(pyast.Expression)).Roles(Expression),
224213
On(HasInternalType(pyast.Expr)).Roles(Expression),
225214
On(HasInternalType(pyast.Name)).Roles(SimpleIdentifier, Expression),
@@ -309,7 +298,6 @@ var AnnotationRules = On(Any).Self(
309298
On(HasInternalType(pyast.Num)).Roles(NumberLiteral, Expression),
310299
// FIXME: this is the annotated assignment (a: annotation = 3) not exactly Assignment
311300
// it also lacks AssignmentValue and AssignmentVariable (see how to add them)
312-
On(HasInternalType(pyast.AnnAssign)).Roles(Assignment, Statement),
313301
On(HasInternalType(pyast.Assert)).Roles(Assert, Statement),
314302

315303
// These are AST nodes in Python2 but we convert them to functions in the UAST like
@@ -334,8 +322,9 @@ var AnnotationRules = On(Any).Self(
334322
// them, the runtimes ignore them). The TOKEN will take the annotation in the UAST node so
335323
// the information is keept in any case.
336324
// FIXME: need annotation or type UAST roles
337-
On(HasInternalRole("annotation")).Roles(Comment, Incomplete),
338-
On(HasInternalRole("returns")).Roles(Comment, Incomplete),
325+
On(HasInternalType(pyast.AnnAssign)).Roles(Assignment, Comment, Incomplete),
326+
On(HasInternalType(pyast.Annotation)).Roles(Comment, Incomplete),
327+
On(HasInternalRole(pyast.Returns)).Roles(Comment, Incomplete),
339328

340329
// Python very odd ellipsis operator. Has a special rule in tonoder synthetic tokens
341330
// map to load it with the token "PythonEllipsisOperator" and gets the role SimpleIdentifier
@@ -366,5 +355,11 @@ var AnnotationRules = On(Any).Self(
366355
On(HasInternalType(pyast.Yield)).Roles(Return, Statement, Incomplete),
367356
On(HasInternalType(pyast.YieldFrom)).Roles(Return, Statement, Incomplete),
368357
On(HasInternalType(pyast.Yield)).Roles(ListLiteral, Expression, Incomplete),
358+
359+
On(HasInternalType(pyast.Subscript)).Roles(Expression, Incomplete),
360+
On(HasInternalType(pyast.Index)).Roles(Expression, Incomplete),
361+
On(HasInternalType(pyast.Slice)).Roles(Expression, Incomplete),
362+
On(HasInternalType(pyast.ExtSlice)).Roles(Expression, Incomplete),
363+
369364
),
370365
)

‎driver/normalizer/pyast/pyast.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ const (
1616
Alias = "alias"
1717
And = "And"
1818
AnnAssign = "AnnAssign"
19+
Annotation = "Annotation"
1920
Arg = "arg"
2021
Arguments = "arguments"
2122
Assert = "Assert"
@@ -115,6 +116,7 @@ const (
115116
RemainderNoops = "RemainderNoops"
116117
Repr = "Repr"
117118
Return = "Return"
119+
Returns = "returns"
118120
SameLineNoops = "SameLineNoops"
119121
Set = "Set"
120122
SetComp = "SetComp"

0 commit comments

Comments
 (0)