Add unknown validation state for intrinsic functions in composite validators (#4384)

- Add unknown field to ValidationError and unresolvable_function_mode to Context
- Update composite validators (if, oneOf, anyOf, allOf, not, contains) to
  propagate unknown errors when functions cannot be resolved
- Add default function validators in _keywords_cfn.py that return unknown
  in unresolvable mode, overridden by full function rules at runtime
- Filter unknown errors in Base.py and CfnLintJsonSchema
- Return unknown from BaseFn.validate() in unresolvable mode
- Fix oneOf to collect valid schemas in single pass

Author: kddejong

src/cfnlint/context/context.py

@@ -164,6 +164,11 @@ class Context:
 
     strict_types: bool = field(init=True, default=True)
 
+    # When True, function validators should return unknown errors
+    # instead of validating. Used in composite validators
+    # (if/then/else, oneOf, anyOf) to handle unresolvable functions
+    unresolvable_function_mode: bool = field(init=True, default=False)
+
     pseudo_parameters: Set[str] = field(
         init=True, default_factory=lambda: set(PSEUDOPARAMS)
     )

src/cfnlint/jsonschema/_keywords.py

@@ -85,10 +85,33 @@ def allOf(
     validator = validator.evolve(
         function_filter=validator.function_filter.evolve(
             add_cfn_lint_keyword=False,
-        )
+        ),
+        context=validator.context.evolve(
+            unresolvable_function_mode=True,
+        ),
     )
+    has_unknown = False
+    known_errors = []
+
     for index, subschema in enumerate(allOf):
-        yield from validator.descend(instance, subschema, schema_path=index)
+        errs = list(validator.descend(instance, subschema, schema_path=index))
+
+        if any(getattr(err, "unknown", False) for err in errs):
+            has_unknown = True
+        else:
+            known_errors.extend(errs)
+
+    # If we have unknown branches, we can't determine if allOf is satisfied
+    if has_unknown:
+        yield ValidationError(
+            f"Cannot determine allOf for {instance!r}",
+            unknown=True,
+        )
+        return
+
+    # Yield all known errors
+    for err in known_errors:
+        yield err
 
 
 def anyOf(
@@ -97,10 +120,16 @@ def anyOf(
     validator = validator.evolve(
         function_filter=validator.function_filter.evolve(
             add_cfn_lint_keyword=False,
-        )
+        ),
+        context=validator.context.evolve(
+            unresolvable_function_mode=True,
+        ),
     )
     all_errors = []
     other_errors = []
+    has_valid = False
+    has_unknown = False
+
     for index, subschema in enumerate(anyOf):
         errs = []
         # warning and informational shouldn't count towards if anyOf is
@@ -110,14 +139,32 @@ def anyOf(
                 other_errors.append(err)
                 continue
             errs.append(err)
-        if not errs:
+
+        if any(getattr(err, "unknown", False) for err in errs):
+            has_unknown = True
+        elif not errs:
+            has_valid = True
             break
-        all_errors.extend(errs)
-    else:
+        else:
+            all_errors.extend(errs)
+
+    # If we found a valid branch, we're done
+    if has_valid:
+        return
+
+    # If we have unknown branches, we can't determine
+    if has_unknown:
         yield ValidationError(
-            f"{instance!r} is not valid under any of the given schemas",
-            context=all_errors + other_errors,
+            f"Cannot determine anyOf for {instance!r}",
+            unknown=True,
         )
+        return
+
+    # All known branches failed
+    yield ValidationError(
+        f"{instance!r} is not valid under any of the given schemas",
+        context=all_errors + other_errors,
+    )
 
 
 def const(
@@ -134,22 +181,39 @@ def contains(
         return
 
     matches = 0
+    unknown_count = 0
     min_contains = schema.get("minContains", 1)
     max_contains = schema.get("maxContains", len(instance))
 
     contains_validator = validator.evolve(schema=contains)
 
     for each in instance:
-        if contains_validator.is_valid(each):
-            matches += 1
-            if matches > max_contains:
-                yield ValidationError(
-                    "Too many items match the given schema "
-                    f"(expected at most {max_contains})",
-                    validator="maxContains",
-                    validator_value=max_contains,
-                )
-                return
+        errs = list(contains_validator.iter_errors(each))
+        # Filter unknown errors
+        non_unknown_errs = [err for err in errs if not err.unknown]
+
+        if not non_unknown_errs:
+            # If no non-unknown errors, it's either valid or unknown
+            if any(err.unknown for err in errs):
+                unknown_count += 1
+            else:
+                matches += 1
+                if matches > max_contains:
+                    yield ValidationError(
+                        "Too many items match the given schema "
+                        f"(expected at most {max_contains})",
+                        validator="maxContains",
+                        validator_value=max_contains,
+                    )
+                    return
+
+    # If we have unknown items, we can't determine if contains is satisfied
+    if unknown_count > 0 and matches < min_contains:
+        yield ValidationError(
+            "Cannot determine if contains constraint is satisfied",
+            unknown=True,
+        )
+        return
 
     if matches < min_contains:
         if not matches:
@@ -314,10 +378,22 @@ def if_(
     if_validator = validator.evolve(
         context=validator.context.evolve(
             allow_exceptions=False,
+            unresolvable_function_mode=True,
         )
     )
 
-    if if_validator.evolve(schema=if_schema).is_valid(instance):
+    if_errors = list(if_validator.evolve(schema=if_schema).iter_errors(instance))
+
+    # If any error is unknown, we can't determine the condition
+    if any(getattr(err, "unknown", False) for err in if_errors):
+        yield ValidationError(
+            f"Cannot determine if condition for {instance!r}",
+            unknown=True,
+        )
+        return
+
+    # Original logic
+    if not if_errors:
         if "then" in schema:
             then = schema["then"]
             yield from validator.descend(instance, then, schema_path="then")
@@ -477,9 +553,24 @@ def not_(
     validator = validator.evolve(
         function_filter=validator.function_filter.evolve(
             add_cfn_lint_keyword=False,
-        )
+        ),
+        context=validator.context.evolve(
+            unresolvable_function_mode=True,
+        ),
     )
-    if validator.evolve(schema=not_schema).is_valid(instance):
+
+    errs = list(validator.evolve(schema=not_schema).iter_errors(instance))
+
+    # If there are unknown errors, we can't determine if not is satisfied
+    if any(getattr(err, "unknown", False) for err in errs):
+        yield ValidationError(
+            f"Cannot determine not for {instance!r}",
+            unknown=True,
+        )
+        return
+
+    # If no errors, the schema is valid, so 'not' fails
+    if not errs:
         message = f"{instance!r} should not be valid under {not_schema!r}"
         yield ValidationError(message)
 
@@ -490,30 +581,48 @@ def oneOf(
     validator = validator.evolve(
         function_filter=validator.function_filter.evolve(
             add_cfn_lint_keyword=False,
-        )
+        ),
+        context=validator.context.evolve(
+            unresolvable_function_mode=True,
+        ),
     )
     subschemas = enumerate(oneOf)
     all_errors = []
+    valid_count = 0
+    has_unknown = False
+    valid_schemas = []
+
     for index, subschema in subschemas:
         errs = list(validator.descend(instance, subschema, schema_path=index))
-        if not errs:
-            first_valid = subschema
-            break
-        all_errors.extend(errs)
-    else:
+
+        if any(getattr(err, "unknown", False) for err in errs):
+            has_unknown = True
+        elif not errs:
+            valid_count += 1
+            valid_schemas.append(subschema)
+        else:
+            all_errors.extend(errs)
+
+    # If we can't determine, yield unknown error
+    if has_unknown and valid_count < 2:
+        yield ValidationError(
+            f"Cannot determine oneOf for {instance!r}",
+            unknown=True,
+        )
+        return
+
+    # Definitive results
+    if valid_count == 1:
+        return
+
+    if valid_count == 0:
         yield ValidationError(
             f"{instance!r} is not valid under any of the given schemas",
             context=all_errors,
         )
-
-    more_valid = [
-        each
-        for _, each in subschemas
-        if validator.evolve(schema=each).is_valid(instance)
-    ]
-    if more_valid:
-        more_valid.append(first_valid)
-        reprs = ", ".join(repr(schema) for schema in more_valid)
+    else:
+        # Multiple valid schemas
+        reprs = ", ".join(repr(schema) for schema in valid_schemas)
         yield ValidationError(f"{instance!r} is valid under each of {reprs}")
 
 

src/cfnlint/jsonschema/_keywords_cfn.py

@@ -223,9 +223,40 @@ def cfn_type(validator: Validator, tS: Any, instance: Any, schema: Any):
         yield ValidationError(f"{instance!r} is not of type {reprs}")
 
 
+def _function_unknown(
+    validator: Validator, s: Any, instance: Any, schema: dict[str, Any]
+) -> ValidationResult:
+    """Default validator for CloudFormation intrinsic functions.
+    Returns unknown error in unresolvable mode to prevent incorrect validation.
+    Overridden by function rules in full validation environments."""
+    if validator.context.unresolvable_function_mode:
+        yield ValidationError(
+            "Cannot resolve function in composite validation",
+            unknown=True,
+        )
+    # In normal mode, do nothing - let the function pass through
+    return
+
+
 cfn_validators: dict[str, V] = {
     "additionalProperties": additionalProperties,
     "cfnContext": cfnContext,
     "dynamicValidation": dynamicValidation,
     "type": cfn_type,
+    # CloudFormation intrinsic functions - default to unknown
+    "ref": _function_unknown,
+    "fn_base64": _function_unknown,
+    "fn_cidr": _function_unknown,
+    "fn_findinmap": _function_unknown,
+    "fn_getatt": _function_unknown,
+    "fn_getazs": _function_unknown,
+    "fn_importvalue": _function_unknown,
+    "fn_join": _function_unknown,
+    "fn_select": _function_unknown,
+    "fn_split": _function_unknown,
+    "fn_sub": _function_unknown,
+    "fn_transform": _function_unknown,
+    "fn_tojsonstring": _function_unknown,
+    "fn_length": _function_unknown,
+    "fn_if": _function_unknown,
 }

src/cfnlint/jsonschema/exceptions.py

@@ -56,6 +56,7 @@ def __init__(
         extra_args=None,
         rule: CloudFormationLintRule | None = None,
         path_override: Deque | Tuple = (),
+        unknown: bool = False,
     ):
         super().__init__(
             message,
@@ -83,6 +84,7 @@ def __init__(
         self.extra_args = extra_args or {}
         self.rule = rule
         self.path_override = deque(path_override)
+        self.unknown = unknown
 
         for error in context:
             error.parent = self

src/cfnlint/rules/functions/_BaseFn.py

@@ -174,6 +174,14 @@ def validate(
         instance: Any,
         schema: Any,
     ) -> ValidationResult:
+        # If in unresolvable mode, return unknown error immediately
+        if validator.context.unresolvable_function_mode:
+            yield ValidationError(
+                f"Cannot resolve {self.fn.name} in composite validation",
+                unknown=True,
+            )
+            return
+
         # validate this function will return the correct type
         errs = list(
             self.fix_errors(self.validate_fn_output_types(validator, s, instance))

src/cfnlint/rules/jsonschema/Base.py

@@ -32,6 +32,10 @@ def _convert_validation_errors_to_matches(
         path: Sequence[str],
         e: ValidationError,
     ):
+        # Don't convert unknown errors to matches
+        if getattr(e, "unknown", False):
+            return []
+
         matches = []
         kwargs: dict[Any, Any] = {}
 

src/cfnlint/rules/jsonschema/CfnLintJsonSchema.py

@@ -45,7 +45,7 @@ def message(self, instance: Any, err: ValidationError) -> str:
         return self.shortdesc
 
     def _iter_errors(self, validator, instance):
-        errs = list(validator.iter_errors(instance))
+        errs = [err for err in validator.iter_errors(instance) if not err.unknown]
         if not self.all_matches:
             err = best_match(errs)
             if err is not None:
@@ -75,9 +75,10 @@ def validate(
             ),
             schema=schema,
             context=validator.context.evolve(
-                functions=[],
-                strict_types=True,
+                unresolvable_function_mode=True,
             ),
         )
 
-        yield from self._iter_errors(cfn_validator, instance)
+        for err in self._iter_errors(cfn_validator, instance):
+            if not getattr(err, "unknown", False):
+                yield err