Merge pull request #366 from dflook/fallback-parse

Improve parsing of tf files

Author: dflook

.github/workflows/test-version.yaml

@@ -888,7 +888,7 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Test terraform-version
+      - name: Test tofu-version
         uses: ./tofu-version
         id: tofu-version
         env:
@@ -907,3 +907,29 @@ jobs:
             echo "::error:: Terraform version not selected"
             exit 1
           fi
+
+  hard_parse:
+    runs-on: ubuntu-24.04
+    name: Get version constraint from hard to parse file
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Test terraform-version
+        uses: ./terraform-version
+        id: terraform-version
+        with:
+          path: tests/workflows/test-version/hard-parse
+
+      - name: Check the version
+        env:
+          DETECTED_TERRAFORM_VERSION: ${{ steps.terraform-version.outputs.terraform }}
+        run: |
+          echo "The terraform version was $DETECTED_TERRAFORM_VERSION"
+
+          if [[ "$DETECTED_TERRAFORM_VERSION" != "1.10.4" ]]; then
+            echo "::error:: Terraform constraint not parsed correctly"
+            exit 1
+          fi

image/src/terraform/fallback_parser.py

@@ -0,0 +1,98 @@
+"""
+Fallback parsing for hcl files
+
+We only need limited information from Terraform Modules:
+- The required_version constraint
+- The backend type
+- A list of sensitive variable names
+- The backend configuration for remote backends and cloud blocks
+
+The easiest way to get this information is to parse the HCL files directly.
+This doesn't always work if our parser fails, or the files are malformed.
+
+This fallback 'parser' does the stupidest thing that might work to get the information we need.
+
+TODO: The backend configuration is not yet implemented.
+"""
+
+import re
+from pathlib import Path
+from typing import Optional
+
+from github_actions.debug import debug
+
+
+def get_required_version(body: str) -> Optional[str]:
+    """Get the required_version constraint string from a tf file"""
+
+    if version := re.search(r'required_version\s*=\s*"(.+)"', body):
+        return version.group(1)
+
+def get_backend_type(body: str) -> Optional[str]:
+    """Get the backend type from a tf file"""
+
+    if backend := re.search(r'backend\s*"(.+)"', body):
+        return backend.group(1)
+
+    if backend := re.search(r'backend\s+(.*)\s*{', body):
+        return backend.group(1).strip()
+
+    if re.search(r'cloud\s+\{', body):
+        return 'cloud'
+
+def get_sensitive_variables(body: str) -> list[str]:
+    """Get the sensitive variable names from a tf file"""
+
+    variables = []
+
+    found = False
+
+    for line in reversed(body.splitlines()):
+        if re.search(r'sensitive\s*=\s*true', line, re.IGNORECASE) or re.search(r'sensitive\s*=\s*"true"', line, re.IGNORECASE):
+            found = True
+            continue
+
+        if found and (variable := re.search(r'variable\s*"(.+)"', line)):
+            variables.append(variable.group(1))
+            found = False
+
+        if found and (variable := re.search(r'variable\s+(.+)\{', line)):
+            variables.append(variable.group(1))
+            found = False
+
+    return variables
+
+def parse(path: Path) -> dict:
+    debug(f'Attempting to parse {path} with fallback parser')
+    body = path.read_text()
+
+    module = {}
+
+    if constraint := get_required_version(body):
+        module['terraform'] = [{
+            'required_version': constraint
+        }]
+
+    if backend_type := get_backend_type(body):
+        if 'terraform' not in module:
+            module['terraform'] = []
+
+        if backend_type == 'cloud':
+            module['terraform'].append({'cloud': [{}]})
+        else:
+            module['terraform'].append({'backend': [{backend_type:{}}]})
+
+    if sensitive_variables := get_sensitive_variables(body):
+        module['variable'] = []
+        for variable in sensitive_variables:
+            module['variable'].append({
+                variable: {
+                    'sensitive': True
+                }
+            })
+
+    return module
+
+if __name__ == '__main__':
+    from pprint import pprint
+    pprint(parse(Path('tests/workflows/test-validate/hard-parse/main.tf')))

image/src/terraform/hcl.py

@@ -8,15 +8,17 @@
 from pathlib import Path
 
 from github_actions.debug import debug
+import terraform.fallback_parser
 
 
 def try_load(path: Path) -> dict:
     try:
         with open(path) as f:
             return hcl2.load(f)
     except Exception as e:
+        debug(f'Failed to load {path}')
         debug(str(e))
-        return {}
+        return terraform.fallback_parser.parse(Path(path))
 
 
 def is_loadable(path: Path) -> bool:

image/tools/convert_validate_report.py

@@ -25,6 +25,13 @@ def convert_to_github(report: Dict, base_path: str) -> Iterable[str]:
                 params['endLine'] = diag['range']['end']['line']
                 params['endColumn'] = diag['range']['end']['column']
 
+            if params.get('line') != params.get('endLine'):
+                # GitHub can't cope with 'col' and 'endColumn' if 'line' and 'endLine' are different values.
+                if 'col' in params:
+                    del params['col']
+                if 'endColumn' in params:
+                    del params['endColumn']
+
         summary = diag['summary'].split('\n')[0]
         params = ','.join(f'{k}={v}' for k, v in params.items())
 

tests/test_validate.py

@@ -122,7 +122,7 @@ def test_invalid_paths():
 
     expected_output = [
         '::error file=tests/validate/invalid/main.tf,line=2,col=1,endLine=2,endColumn=33::Duplicate resource "null_resource" configuration',
-        '::error file=tests/validate/module/invalid.tf,line=2,col=1,endLine=5,endColumn=66::Duplicate resource "null_resource" configuration'
+        '::error file=tests/validate/module/invalid.tf,line=2,endLine=5::Duplicate resource "null_resource" configuration'
     ]
 
     output = list(convert_to_github(input, 'tests/validate/invalid'))

tests/workflows/test-version/hard-parse/main.tf

@@ -0,0 +1,54 @@
+terraform {
+
+}
+
+terraform {
+  required_version  =  "1.10.4"
+}
+
+locals {
+  cloud_run_services = [
+    {
+      service_name = "service-1",
+      output_topics = [
+        {
+          name    = "topic-1",
+          version = "v1"
+        }
+      ]
+    }
+  ]
+}
+
+
+module "pubsub" {
+  for_each = {
+    for service in local.cloud_run_services : service.service_name => service
+  }
+  source = "./module"
+  topics = [
+    for entity in each.value.output_topics : {
+      topic_name        = entity.version != "" ? format("Topic-%s-%s", entity.name, entity.version) : format("Topic-%s", entity.name)
+      subscription_name = entity.version != "" ? format("Sub-%s-%s", entity.name, entity.version) : format("Sub-%s", entity.name)
+    }
+  ]
+}
+
+
+variable "not" {}
+
+variable "should-be-sensitive" {
+  sensitive=true
+}
+
+variable "not-again" {
+  sensitive =          false
+}
+
+variable also_sensitive {
+  sensitive =  "true"
+}
+
+terraform {
+  backend "s3" {}
+}

tests/workflows/test-version/hard-parse/module/main.tf

@@ -0,0 +1 @@
+variable "topics" {}