add heredoc rules and deserialization;

require heredoc openers to be on their on separate line in lark grammar;
whitespace trimming based on current implementation in dict_transformer.py;

Author: kkozik-amplify

hcl2/rule_transformer/deserializer.py

@@ -1,6 +1,6 @@
 import json
 from functools import lru_cache
-from typing import Any, TextIO, List
+from typing import Any, TextIO, List, Union
 
 from regex import regex
 
@@ -31,6 +31,8 @@
     StringRule,
     InterpolationRule,
     StringPartRule,
+    HeredocTemplateRule, 
+    HeredocTrimTemplateRule,
 )
 from hcl2.rule_transformer.rules.tokens import (
     NAME,
@@ -47,9 +49,11 @@
     COMMA,
     DOT,
     LBRACE,
+    HEREDOC_TRIM_TEMPLATE,
+    HEREDOC_TEMPLATE,
 )
 from hcl2.rule_transformer.transformer import RuleTransformer
-from hcl2.rule_transformer.utils import DeserializationOptions
+from hcl2.rule_transformer.utils import DeserializationOptions, HEREDOC_TRIM_PATTERN, HEREDOC_PATTERN
 
 
 class Deserializer:
@@ -99,7 +103,7 @@ def _deserialize_block_elements(self, value: dict) -> List[LarkRule]:
 
         return children
 
-    def _deserialize_text(self, value) -> LarkRule:
+    def _deserialize_text(self, value: Any) -> LarkRule:
         try:
             int_val = int(value)
             return IntLitRule([IntLiteral(int_val)])
@@ -114,6 +118,16 @@ def _deserialize_text(self, value) -> LarkRule:
 
         if isinstance(value, str):
             if value.startswith('"') and value.endswith('"'):
+                if not self.options.heredocs_to_strings and value.startswith('"<<-'):
+                    match = HEREDOC_TRIM_PATTERN.match(value[1:-1])
+                    if match:
+                        return self._deserialize_heredoc(value[1:-1], True)
+                    
+                if not self.options.heredocs_to_strings and value.startswith('"<<'):
+                    match = HEREDOC_PATTERN.match(value[1:-1])
+                    if match:
+                        return self._deserialize_heredoc(value[1:-1], False)
+                    
                 return self._deserialize_string(value)
 
             if self._is_expression(value):
@@ -131,11 +145,12 @@ def _deserialize_identifier(self, value: str) -> IdentifierRule:
 
     def _deserialize_string(self, value: str) -> StringRule:
         result = []
-
-        pattern = regex.compile(r"(\${1,2}\{(?:[^{}]|(?R))*\})")
-        parts = [part for part in pattern.split(value) if part != ""]
+        # split string into individual parts based on lark grammar
         # e.g. 'aaa$${bbb}ccc${"ddd-${eee}"}' -> ['aaa', '$${bbb}', 'ccc', '${"ddd-${eee}"}']
         # 'aa-${"bb-${"cc-${"dd-${5 + 5}"}"}"}' -> ['aa-', '${"bb-${"cc-${"dd-${5 + 5}"}"}"}']
+        pattern = regex.compile(r"(\${1,2}\{(?:[^{}]|(?R))*\})")
+        parts = [part for part in pattern.split(value) if part != ""]
+
 
         for part in parts:
             if part == '"':
@@ -166,6 +181,11 @@ def _deserialize_string_part(self, value: str) -> StringPartRule:
 
         return StringPartRule([STRING_CHARS(value)])
 
+    def _deserialize_heredoc(self, value: str, trim: bool) -> Union[HeredocTemplateRule, HeredocTrimTemplateRule]:
+        if trim:
+            return HeredocTrimTemplateRule([HEREDOC_TRIM_TEMPLATE(value)])
+        return HeredocTemplateRule([HEREDOC_TEMPLATE(value)]) 
+
     def _deserialize_expression(self, value: str) -> ExprTermRule:
         """Deserialize an expression string into an ExprTermRule."""
         # instead of processing expression manually and trying to recognize what kind of expression it is,

hcl2/rule_transformer/hcl2.lark

@@ -67,8 +67,8 @@ ELLIPSIS : "..."
 COLONS: "::"
 
 // Heredocs
-HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\n?(?:.|\n)*?\n\s*(?P=heredoc)\n/
-HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\n?(?:.|\n)*?\n\s*(?P=heredoc_trim)\n/
+HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:.|\n)*?\n\s*(?P=heredoc)\n/
+HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:.|\n)*?\n\s*(?P=heredoc_trim)\n/
 
 // Ignore whitespace (but not newlines, as they're significant in HCL)
 %ignore /[ \t]+/

hcl2/rule_transformer/rules/strings.py

@@ -1,3 +1,4 @@
+import sys
 from typing import Tuple, List, Any, Union
 
 from hcl2.rule_transformer.rules.abstract import LarkRule
@@ -8,11 +9,15 @@
     DBLQUOTE,
     STRING_CHARS,
     ESCAPED_INTERPOLATION,
+    HEREDOC_TEMPLATE, 
+    HEREDOC_TRIM_TEMPLATE,
 )
 from hcl2.rule_transformer.utils import (
     SerializationOptions,
     SerializationContext,
     to_dollar_string,
+    HEREDOC_TRIM_PATTERN, 
+    HEREDOC_PATTERN,
 )
 
 
@@ -71,3 +76,71 @@ def serialize(
         self, options=SerializationOptions(), context=SerializationContext()
     ) -> Any:
         return '"' + "".join(part.serialize() for part in self.string_parts) + '"'
+
+
+class HeredocTemplateRule(LarkRule):
+    
+    _children: Tuple[HEREDOC_TEMPLATE]
+    _trim_chars = "\n\t "
+    
+    
+    @staticmethod
+    def lark_name() -> str:
+        return "heredoc_template"
+    
+    @property
+    def heredoc(self):
+        return self.children[0]
+    
+    def serialize(
+        self, options=SerializationOptions(), context=SerializationContext()
+    ) -> Any:
+        heredoc = self.heredoc.serialize(options, context)
+        
+        if not options.preserve_heredocs:
+            match = HEREDOC_PATTERN.match(heredoc)
+            if not match:
+                raise RuntimeError(f"Invalid Heredoc token: {heredoc}")
+            heredoc = match.group(2)
+        
+        result = heredoc.rstrip(self._trim_chars)
+        return f'"{result}"'
+
+
+class HeredocTrimTemplateRule(HeredocTemplateRule):
+
+    _children: Tuple[HEREDOC_TRIM_TEMPLATE]
+    
+    @staticmethod
+    def lark_name() -> str:
+        return "heredoc_trim_template"
+    
+    def serialize(
+        self, options=SerializationOptions(), context=SerializationContext()
+    ) -> Any:
+        # See https://github.com/hashicorp/hcl2/blob/master/hcl/hclsyntax/spec.md#template-expressions
+        # This is a special version of heredocs that are declared with "<<-"
+        # This will calculate the minimum number of leading spaces in each line of a heredoc
+        # and then remove that number of spaces from each line
+
+        heredoc = self.heredoc.serialize(options, context)
+
+        if not options.preserve_heredocs:
+            match = HEREDOC_TRIM_PATTERN.match(heredoc)
+            if not match:
+                raise RuntimeError(f"Invalid Heredoc token: {heredoc}")
+            heredoc = match.group(2)
+
+        heredoc = heredoc.rstrip(self._trim_chars)  
+        lines = heredoc.split("\n")
+    
+        # calculate the min number of leading spaces in each line
+        min_spaces = sys.maxsize
+        for line in lines:
+            leading_spaces = len(line) - len(line.lstrip(" "))
+            min_spaces = min(min_spaces, leading_spaces)
+    
+        # trim off that number of leading spaces from each line
+        lines = [line[min_spaces:] for line in lines]
+        return '"' + "\n".join(lines) + '"'
+    

hcl2/rule_transformer/rules/tokens.py

@@ -67,12 +67,14 @@ def serialize_conversion(self) -> Callable[[Any], str]:
 
 
 # explicitly define various kinds of string-based tokens for type hinting
-# variable value
+# variable values
 NAME = StringToken["NAME"]
 STRING_CHARS = StringToken["STRING_CHARS"]
 ESCAPED_INTERPOLATION = StringToken["ESCAPED_INTERPOLATION"]
 BINARY_OP = StringToken["BINARY_OP"]
-# static value
+HEREDOC_TEMPLATE = STRING_CHARS["HEREDOC_TEMPLATE"]
+HEREDOC_TRIM_TEMPLATE = STRING_CHARS["HEREDOC_TRIM_TEMPLATE"]
+# static values
 EQ = StaticStringToken[("EQ", "=")]
 COLON = StaticStringToken[("COLON", ":")]
 LPAR = StaticStringToken[("LPAR", "(")]

hcl2/rule_transformer/transformer.py

@@ -44,7 +44,9 @@
 from hcl2.rule_transformer.rules.strings import (
     InterpolationRule,
     StringRule,
-    StringPartRule,
+    StringPartRule, 
+    HeredocTemplateRule, 
+    HeredocTrimTemplateRule,
 )
 from hcl2.rule_transformer.rules.tokens import (
     NAME,
@@ -127,6 +129,14 @@ def string_part(self, meta: Meta, args) -> StringPartRule:
     def interpolation(self, meta: Meta, args) -> InterpolationRule:
         return InterpolationRule(args, meta)
 
+    @v_args(meta=True)
+    def heredoc_template(self, meta: Meta, args) -> HeredocTemplateRule:
+        return HeredocTemplateRule(args, meta)
+        
+    @v_args(meta=True)
+    def heredoc_template_trim(self, meta: Meta, args) -> HeredocTrimTemplateRule:
+        return HeredocTrimTemplateRule(args, meta)
+    
     @v_args(meta=True)
     def expr_term(self, meta: Meta, args) -> ExprTermRule:
         return ExprTermRule(args, meta)

hcl2/rule_transformer/utils.py

@@ -1,7 +1,12 @@
+import re
 from contextlib import contextmanager
 from dataclasses import dataclass, replace
 from typing import Generator
 
+HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S)
+HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S)
+
+
 
 @dataclass
 class SerializationOptions:
@@ -10,11 +15,12 @@ class SerializationOptions:
     wrap_objects: bool = False
     wrap_tuples: bool = False
     explicit_blocks: bool = True
+    preserve_heredocs: bool = True
 
 
 @dataclass
 class DeserializationOptions:
-    pass
+    heredocs_to_strings: bool = False
 
 
 @dataclass