From 8e6b18dacc933359b26c4892d63b4e9e687b0467 Mon Sep 17 00:00:00 2001 From: traut Date: Fri, 1 Aug 2025 23:04:43 +0200 Subject: [PATCH 1/2] Initial commit --- detection_rules/rule.py | 14 ++++++++--- tests/test_schemas.py | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/detection_rules/rule.py b/detection_rules/rule.py index f251a7d5d63..1d1e45c3ffc 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -934,9 +934,15 @@ def validates_esql_data(self, data: dict[str, Any], **_: Any) -> None: # Convert the query string to lowercase to handle case insensitivity query_lower = data["query"].lower() - # Combine both patterns using an OR operator and compile the regex + # Combine both patterns using an OR operator and compile the regex. + # The first part matches the metadata fields in the from clause by allowing one or + # multiple indices and any order of the metadata fields + # The second part matches the stats command with the by clause combined_pattern = re.compile( - r"(from\s+\S+\s+metadata\s+_id,\s*_version,\s*_index)|(\bstats\b.*?\bby\b)", re.DOTALL + r"(from\s+(?:\S+\s*,\s*)*\S+\s+metadata\s+" + r"(?:_id|_version|_index)(?:,\s*(?:_id|_version|_index)){2})" + r"|(\bstats\b.*?\bby\b)", + re.DOTALL, ) # Ensure that non-aggregate queries have metadata @@ -948,7 +954,9 @@ def validates_esql_data(self, data: dict[str, Any], **_: Any) -> None: ) # Enforce KEEP command for ESQL rules - if "| keep" not in query_lower: + # Match | followed by optional whitespace/newlines and then 'keep' + keep_pattern = re.compile(r"\|\s*keep\b", re.IGNORECASE | re.DOTALL) + if not keep_pattern.search(query_lower): raise ValidationError( f"Rule: {data['name']} does not contain a 'keep' command -> Add a 'keep' command to the query." ) diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 0d8a8d3ddd9..5148c3d2b39 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -8,14 +8,18 @@ import copy import unittest import uuid +from pathlib import Path import eql +import pytest +import pytoml from marshmallow import ValidationError from semver import Version from detection_rules import utils from detection_rules.config import load_current_package_version from detection_rules.rule import TOMLRuleContents +from detection_rules.rule_loader import RuleCollection from detection_rules.schemas import RULES_CONFIG, downgrade from detection_rules.version_lock import VersionLockFile @@ -302,3 +306,53 @@ def test_stack_schema_map(self): stack_map = utils.load_etc_dump(["stack-schema-map.yaml"]) err_msg = f"There is no entry defined for the current package ({package_version}) in the stack-schema-map" self.assertIn(package_version, [Version.parse(v) for v in stack_map], err_msg) + + +class TestESQLValidation(unittest.TestCase): + """Test ESQL rule validation""" + + def test_esql_data_validation(self): + """Test ESQL rule data validation""" + + # A random ESQL rule to deliver a test query + rule_path = Path("rules/windows/defense_evasion_posh_obfuscation_index_reversal.toml") + rule_body = rule_path.read_text() + rule_dict = pytoml.loads(rule_body) + + # Most used order of the metadata fields + query = """ + FROM logs-windows.powershell_operational* METADATA _id, _version, _index + | WHERE event.code == "4104" + | KEEP event.count + """ + rule_dict["rule"]["query"] = query + _ = RuleCollection().load_dict(rule_dict, path=rule_path) + + # The order of the metadata fields from the example in the docs - + # https://www.elastic.co/guide/en/security/8.17/rules-ui-create.html#esql-non-agg-query + query = """ + FROM logs-windows.powershell_operational* METADATA _id, _index, _version + | WHERE event.code == "4104" + | KEEP event.count + """ + rule_dict["rule"]["query"] = query + _ = RuleCollection().load_dict(rule_dict, path=rule_path) + + # Different metadata fields + with pytest.raises(ValidationError): + query = """ + FROM logs-windows.powershell_operational* METADATA _foo, _index + | WHERE event.code == "4104" + | KEEP event.count + """ + rule_dict["rule"]["query"] = query + _ = RuleCollection().load_dict(rule_dict, path=rule_path) + + # Missing `keep` + with pytest.raises(ValidationError): + query = """ + FROM logs-windows.powershell_operational* METADATA _id, _index, _version + | WHERE event.code == "4104" + """ + rule_dict["rule"]["query"] = query + _ = RuleCollection().load_dict(rule_dict, path=rule_path) From 895fe929cc9a16e18f7a5873da1f62b77c8fc1ab Mon Sep 17 00:00:00 2001 From: traut Date: Fri, 1 Aug 2025 23:15:31 +0200 Subject: [PATCH 2/2] Python project version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 48f36fe14ac..9b20fd20d66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.3.18" +version = "1.3.19" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12"