fix parsing of objects elements; (#197)

this fixes:
* #71
* #77
* #140
* #147
* #149
* #163

Author: kkozik-amplify

hcl2/hcl2.lark

@@ -2,7 +2,6 @@ start : body
 body : (new_line_or_comment? (attribute | block))* new_line_or_comment?
 attribute : identifier EQ expression
 block : identifier (identifier | STRING_LIT | string_with_interpolation)* new_line_or_comment? "{" body "}"
-new_line_and_or_comma: new_line_or_comment | "," | "," new_line_or_comment
 new_line_or_comment: ( NL_OR_COMMENT )+
 NL_OR_COMMENT: /\n[ \t]*/ | /#.*\n/ | /\/\/.*\n/ | /\/\*(.|\n)*?(\*\/)/
 
@@ -70,8 +69,9 @@ EXP_MARK : ("e" | "E") ("+" | "-")? DECIMAL+
 EQ : /[ \t]*=(?!=|>)/
 
 tuple : "[" (new_line_or_comment* expression new_line_or_comment* ",")* (new_line_or_comment* expression)? new_line_or_comment* "]"
-object : "{" new_line_or_comment? (object_elem (new_line_and_or_comma object_elem )* new_line_and_or_comma?)? "}"
-object_elem : (identifier | expression) ( EQ | ":") expression
+object : "{" new_line_or_comment? (new_line_or_comment* (object_elem | (object_elem ",")) new_line_or_comment*)* "}"
+object_elem : object_elem_key ( EQ | ":") expression
+object_elem_key : float_lit | int_lit | identifier | STRING_LIT
 
 
 heredoc_template : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\n?(?:.|\n)*?\n\s*(?P=heredoc)\n/

hcl2/reconstructor.py

@@ -396,15 +396,7 @@ def _is_string_wrapped_tf(interp_s: str) -> bool:
 
         return True
 
-    def _newline(self, level: int, comma: bool = False, count: int = 1) -> Tree:
-        # some rules expect the `new_line_and_or_comma` token
-        if comma:
-            return Tree(
-                Token("RULE", "new_line_and_or_comma"),
-                [self._newline(level=level, comma=False, count=count)],
-            )
-
-        # otherwise, return the `new_line_or_comment` token
+    def _newline(self, level: int, count: int = 1) -> Tree:
         return Tree(
             Token("RULE", "new_line_or_comment"),
             [Token("NL_OR_COMMENT", f"\n{'  ' * level}") for _ in range(count)],
@@ -561,20 +553,27 @@ def _transform_value_to_expr_term(self, value, level) -> Union[Token, Tree]:
             for i, (k, dict_v) in enumerate(value.items()):
                 if k in ["__start_line__", "__end_line__"]:
                     continue
-                identifier = self._name_to_identifier(k)
+
                 value_expr_term = self._transform_value_to_expr_term(dict_v, level + 1)
                 elems.append(
                     Tree(
                         Token("RULE", "object_elem"),
-                        [identifier, Token("EQ", " ="), value_expr_term],
+                        [
+                            Tree(
+                                Token("RULE", "object_elem_key"),
+                                [Tree(Token("RULE", "identifier"), [Token("NAME", k)])],
+                            ),
+                            Token("EQ", " ="),
+                            value_expr_term,
+                        ],
                     )
                 )
 
                 # add indentation appropriately
                 if i < len(value) - 1:
-                    elems.append(self._newline(level + 1, comma=True))
+                    elems.append(self._newline(level + 1))
                 else:
-                    elems.append(self._newline(level, comma=True))
+                    elems.append(self._newline(level))
             return Tree(
                 Token("RULE", "expr_term"), [Tree(Token("RULE", "object"), elems)]
             )
@@ -630,7 +629,7 @@ def _transform_value_to_expr_term(self, value, level) -> Union[Token, Tree]:
                 if parsed_value.data == Token("RULE", "expr_term"):
                     return parsed_value
 
-                # wrap other types of syntax as an expression (in parenthesis)
+                # wrap other types of syntax as an expression (in parentheses)
                 return Tree(Token("RULE", "expr_term"), [parsed_value])
 
             # otherwise it's just a string.

hcl2/transformer.py

@@ -99,12 +99,13 @@ def tuple(self, args: List) -> List:
     def object_elem(self, args: List) -> Dict:
         # This returns a dict with a single key/value pair to make it easier to merge these
         # into a bigger dict that is returned by the "object" function
-        key = self.strip_quotes(args[0])
+        key = self.strip_quotes(str(args[0].children[0]))
         if len(args) == 3:
-            value = self.to_string_dollar(args[2])
+            value = args[2]
         else:
-            value = self.to_string_dollar(args[1])
+            value = args[1]
 
+        value = self.to_string_dollar(value)
         return {key: value}
 
     def object(self, args: List) -> Dict:
@@ -136,9 +137,6 @@ def provider_function_call(self, args: List) -> str:
     def arguments(self, args: List) -> List:
         return args
 
-    def new_line_and_or_comma(self, args: List) -> _DiscardType:
-        return Discard
-
     @v_args(meta=True)
     def block(self, meta: Meta, args: List) -> Dict:
         *block_labels, block_body = args

test/helpers/terraform-config/backend.tf

@@ -19,13 +19,13 @@ terraform {
   backend "gcs" {}
   required_providers {
     aws = {
-      source = "hashicorp/aws"
+      source = "hashicorp/aws",
     }
     null = {
-      source = "hashicorp/null"
+      source = "hashicorp/null",
     }
     template = {
-      source = "hashicorp/template"
+      source = "hashicorp/template",
     }
   }
 }

test/helpers/terraform-config/locals_embedded_condition.tf

@@ -1,6 +1,6 @@
 locals {
   terraform = {
-    channels = (local.running_in_ci ? local.ci_channels : local.local_channels)
-    authentication = []
+    channels = (local.running_in_ci ? local.ci_channels : local.local_channels),
+    authentication = [],
   }
 }

test/helpers/terraform-config/s3.tf

@@ -12,8 +12,8 @@ resource "aws_s3_bucket" "name" {
     }
 
     transition = {
-      days          = 30
-      storage_class = "GLACIER"
+      days          = 30,
+      storage_class = "GLACIER",
     }
   }
 

test/unit/test_hcl2_syntax.py

@@ -104,8 +104,9 @@ def test_tuple(self):
     def test_object(self):
         object_ = """object = {
         key1: identifier, key2: "string", key3: 100,
-        key4: true == false,
+        key4: true == false // comment
         key5: 5 + 5, key6: function(),
+        key7: value == null ? 1 : 0
         }"""
         result = self.load_to_dict(object_)
         self.assertDictEqual(
@@ -118,6 +119,7 @@ def test_object(self):
                     "key4": "${true == false}",
                     "key5": "${5 + 5}",
                     "key6": "${function()}",
+                    "key7": "${value == None ? 1 : 0}",
                 }
             },
         )