Merge pull request #394 from microsoft/iac-p0/yaml-document-foundation

Add reusable YAML document classification foundation

Author: chanel-y

iac/ql/lib/codeql/iac/YamlDocumentClassification.qll

@@ -0,0 +1,86 @@
+/**
+ * Provides YAML document classification shared across IaC YAML query families.
+ *
+ * This library identifies the broad document kind of a YAML document (Azure
+ * DevOps Pipelines, Kubernetes/Helm, Compose, OpenAPI, or CloudFormation) and
+ * excludes content that is out of scope for YAML-focused IaC queries even
+ * though it may be reachable through the same YAML abstract syntax tree:
+ *
+ * - JSON files. The extractor represents JSON using the same node types as
+ *   YAML, since JSON is a syntactic subset of YAML. Callers that only intend
+ *   to analyze literal YAML syntax must not rely on node type alone.
+ * - Azure Resource Manager (ARM) templates. ARM templates are JSON by
+ *   default (`azuredeploy.json`), but can also carry a `.yaml` extension
+ *   while keeping the ARM `$schema` marker, so the exclusion is schema-based
+ *   rather than purely extension-based.
+ *
+ * Terraform, HCL, and Bicep are not represented as `YamlNode`s at all in this
+ * extractor, so no additional exclusion is required for them here.
+ */
+
+import iac
+private import codeql.iac.YAML
+private import codeql.iac.azure.Pipelines
+private import codeql.iac.helmcharts.HelmChart
+private import codeql.iac.compose.Compose
+private import codeql.iac.openapi.OpenApi
+private import codeql.iac.aws.CloudFormation
+
+module YamlDocumentClassification {
+  /**
+   * A YAML document whose source file uses a YAML extension (`.yml` or
+   * `.yaml`), as opposed to a JSON file that happens to be representable by
+   * the same node types.
+   */
+  class YamlSyntaxDocument extends YamlNode, YamlDocument, YamlMapping {
+    YamlSyntaxDocument() { this.getFile().getExtension() = ["yml", "yaml"] }
+  }
+
+  /**
+   * Holds if `doc` carries the Azure Resource Manager (ARM) template schema
+   * marker. ARM templates are out of scope for YAML-focused IaC queries even
+   * when authored with a `.yaml` extension.
+   */
+  private predicate hasArmSchemaMarker(YamlSyntaxDocument doc) {
+    yamlToString(doc.lookup("$schema")).matches("%schema.management.azure.com%")
+  }
+
+  /**
+   * The kind of a supported in-scope YAML document.
+   */
+  class DocumentKind extends string {
+    DocumentKind() {
+      this = ["ado-pipeline", "kubernetes-helm", "compose", "openapi", "cloudformation"]
+    }
+  }
+
+  /**
+   * Holds if `doc` is a supported, in-scope YAML document of the given
+   * `kind`.
+   *
+   * A document is only classified once its file extension is `.yml`/`.yaml`
+   * and it does not carry the ARM template schema marker. Callers writing
+   * YAML-only IaC queries should use this predicate, rather than the
+   * underlying per-schema `Document` classes directly, so that ARM/JSON
+   * content is consistently excluded.
+   */
+  predicate isSupportedYamlDocument(YamlSyntaxDocument doc, DocumentKind kind) {
+    not hasArmSchemaMarker(doc) and
+    (
+      doc instanceof AzurePipelines::Document and kind = "ado-pipeline"
+      or
+      doc instanceof HelmChart::Document and kind = "kubernetes-helm"
+      or
+      doc instanceof Compose::Document and kind = "compose"
+      or
+      doc instanceof OpenApi::Document and kind = "openapi"
+      or
+      doc instanceof CloudFormation::Document and kind = "cloudformation"
+    )
+  }
+
+  /**
+   * Gets a supported, in-scope YAML document of any kind.
+   */
+  YamlSyntaxDocument getASupportedYamlDocument() { isSupportedYamlDocument(result, _) }
+}

iac/ql/test/library-tests/yaml-classification/AST.expected

@@ -0,0 +1,15 @@
+supportedYamlDocument
+| azure-pipelines.yaml:1:1:8:25 | Azure DevOps Pipeline | ado-pipeline |
+| cloudformation.yaml:1:1:6:29 | CloudFormation Document | cloudformation |
+| compose.yaml:1:1:4:24 | version: "3.9" | compose |
+| deployment.yaml:1:1:8:29 | HelmChart Document | kubernetes-helm |
+| openapi.yaml:1:1:12:26 | OpenApi Document | openapi |
+allYamlDocuments
+| arm-template.yaml:1:1:5:24 | $schema ... .json#" |
+| azure-pipelines.yaml:1:1:8:25 | Azure DevOps Pipeline |
+| cloudformation.yaml:1:1:6:29 | CloudFormation Document |
+| compose.yaml:1:1:4:24 | version: "3.9" |
+| deployment.yaml:1:1:8:29 | HelmChart Document |
+| openapi.json:1:1:8:1 | OpenApi Document |
+| openapi.yaml:1:1:12:26 | OpenApi Document |
+| unrelated.yaml:1:1:4:8 | descrip ... ocument |

iac/ql/test/library-tests/yaml-classification/AST.ql

@@ -0,0 +1,10 @@
+private import iac
+private import codeql.iac.YamlDocumentClassification
+
+query predicate supportedYamlDocument(
+  YamlDocumentClassification::YamlSyntaxDocument doc, YamlDocumentClassification::DocumentKind kind
+) {
+  YamlDocumentClassification::isSupportedYamlDocument(doc, kind)
+}
+
+query predicate allYamlDocuments(YamlDocument doc) { any() }

iac/ql/test/library-tests/yaml-classification/arm-template.yaml

@@ -0,0 +1,5 @@
+$schema: "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#"
+contentVersion: "1.0.0.0"
+resources:
+  - type: Microsoft.Storage/storageAccounts
+    name: samplestorage

iac/ql/test/library-tests/yaml-classification/azure-pipelines.yaml

@@ -0,0 +1,8 @@
+trigger:
+  - main
+
+pool:
+  vmImage: ubuntu-latest
+
+steps:
+  - script: echo "hello"

iac/ql/test/library-tests/yaml-classification/cloudformation.yaml

@@ -0,0 +1,6 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Resources:
+  Bucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      AccessControl: Private

iac/ql/test/library-tests/yaml-classification/compose.yaml

@@ -0,0 +1,4 @@
+version: "3.9"
+services:
+  web:
+    image: nginx:latest

iac/ql/test/library-tests/yaml-classification/deployment.yaml

@@ -0,0 +1,8 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sample
+spec:
+  containers:
+    - name: app
+      image: example/app:1.0

iac/ql/test/library-tests/yaml-classification/openapi.json

@@ -0,0 +1,8 @@
+{
+  "openapi": "3.0.0",
+  "info": {
+    "title": "Sample API",
+    "version": "1.0"
+  },
+  "paths": {}
+}

iac/ql/test/library-tests/yaml-classification/openapi.yaml

@@ -0,0 +1,12 @@
+openapi: "3.0.0"
+info:
+  title: Sample API
+  version: "1.0"
+servers:
+  - url: https://example.com
+paths:
+  /items:
+    get:
+      responses:
+        "200":
+          description: OK