From fb8afb7d06674d83df4f81cf65747daf68b447b8 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 11 Apr 2025 23:58:44 +0100 Subject: [PATCH 01/65] add swarm and nomad parse exceptions --- glitch/exceptions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/glitch/exceptions.py b/glitch/exceptions.py index f7da3b3c..6cde3483 100644 --- a/glitch/exceptions.py +++ b/glitch/exceptions.py @@ -16,6 +16,8 @@ "DOCKER_UNKNOW_ERROR": "Docker - Unknown Error: {}", "SHELL_COULD_NOT_PARSE": "Shell Command - Could not parse: {}", "TERRAFORM_COULD_NOT_PARSE": "Terraform - Could not parse file: {}", + "DOCKER_SWARM_COULD_NOT_PARSE": "Swarm - Could not parse file: {}", + "NOMAD_COULD_NOT_PARSE": "Nomad - Could not parse file: {}", } From 0098d97752fbce1fd0faf49ccc51bea1d6aac0b8 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 11 Apr 2025 23:59:11 +0100 Subject: [PATCH 02/65] add Nomad parser --- glitch/parsers/nomad.py | 473 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 473 insertions(+) create mode 100644 glitch/parsers/nomad.py diff --git a/glitch/parsers/nomad.py b/glitch/parsers/nomad.py new file mode 100644 index 00000000..b7b697bd --- /dev/null +++ b/glitch/parsers/nomad.py @@ -0,0 +1,473 @@ +# type: ignore +# Built upon the work done by Nuno Saveedra and João Gonçalves +import os + +from hcl2.parser import hcl2 +import glitch.parsers.parser as p + +from glitch.exceptions import EXCEPTIONS, throw_exception +from glitch.repr.inter import * +from typing import List, Dict, Any + +from lark.tree import Meta, Tree +from lark.lexer import Token +from lark.visitors import Transformer, v_args, Discard + + +class GLITCHTransformer(Transformer): + """Takes a syntax tree generated by the parser and + transforms it to a dict. + """ + + def __init__(self, code: List[str]): + self.code = code + self.comments = [] + super().__init__() + + def new_line_or_comment(self, args: List) -> List: + for arg in args: + if isinstance(arg, Token): + if arg.value.startswith(("//", "#", "/*")): + comment = Comment(arg.value) + comment.line, comment.column = arg.line, arg.column + comment.end_line, comment.end_column = arg.end_line, arg.end_column + self.comments.append(comment) + return Discard + + def new_line_and_or_comma(self, args: List) -> List: + return Discard + + def __get_element_code( + self, start_line: int, start_col: int, end_line: int, end_col: int + ) -> str: + if start_line == end_line: + res = self.code[start_line - 1][max(0, start_col - 1) : end_col - 1] + else: + res = self.code[start_line - 1][max(0, start_col - 1) :] + + for line in range(start_line, end_line - 1): + res += self.code[line] + + if start_line != end_line: + res += self.code[end_line - 1][: end_col - 1] + + return res + + def __get_element_info(self, meta: Meta | Token) -> ElementInfo: + return ElementInfo( + meta.line, + meta.column, + meta.end_line, + meta.end_column, + self.__get_element_code( + meta.line, + meta.column, + meta.end_line, + meta.end_column, + ), + ) + + def __get_element_info_from_tokens(self, start: Token, end: Token) -> ElementInfo: + return ElementInfo( + start.line, + start.column, + end.end_line, + end.end_column, + self.__get_element_code( + start.line, + start.column, + end.end_line, + end.end_column, + ), + ) + + def __parse_heredoc(self, tree: Tree) -> str: + res = "" + for arg in tree.children: + res += arg.value + return "\n".join(res.split("\n")[1:-1]) + + @v_args(meta=True) + def binary_op(self, meta: Meta, args: List) -> Any: + op_to_ir = { + "+": Sum, + "-": Subtract, + "*": Multiply, + "/": Divide, + "%": Modulo, + "&&": And, + "||": Or, + "==": Equal, + "!=": NotEqual, + ">": GreaterThan, + "<": LessThan, + ">=": GreaterThanOrEqual, + "<=": LessThanOrEqual, + } + + if args[1].children[0].children[0] in op_to_ir: + return op_to_ir[args[1].children[0].children[0]]( + self.__get_element_info_from_tokens(args[0], args[1].children[1]), + args[0], + args[1].children[1], + ) + + @v_args(meta=True) + def unary_op(self, meta: Meta, args: List) -> Any: + if args[0] == "-": + return Minus(self.__get_element_info(meta), args[1]) + elif args[0] == "!": + return Not(self.__get_element_info(meta), args[1]) + + @v_args(meta=True) + def get_attr(self, meta: Meta, args: List) -> Any: + return args[0] + + @v_args(meta=True) + def index(self, meta: Meta, args: List) -> Any: + return args[0] + + @v_args(meta=True) + def index_expr_term(self, meta: Meta, args: List) -> Any: + return Access(self.__get_element_info(meta), args[0], args[1]) + + @v_args(meta=True) + def get_attr_expr_term(self, meta: Meta, args: List) -> Any: + return Access(self.__get_element_info(meta), args[0], args[1]) + + @v_args(meta=True) + def int_lit(self, meta: Meta, args: List) -> int: + return Integer(int("".join(args)), self.__get_element_info(meta)) + + @v_args(meta=True) + def float_lit(self, meta: Meta, args: List) -> float: + return Float(float("".join(args)), self.__get_element_info(meta)) + + @v_args(meta=True) + def interpolation_maybe_nested(self, meta: Meta, args: List) -> Any: + return args[0] + + @v_args(meta=True) + def string_with_interpolation(self, meta: Meta, args: List) -> str: + if len(args) == 1: + if isinstance(args[0], Token): + return String( + args[0].value, + self.__get_element_info(meta), + ) + return args[0] + else: + for i in range(len(args)): + if isinstance(args[i], Token): + args[i] = String( + args[i].value, + self.__get_element_info(args[i]), + ) + + res = Sum( + ElementInfo( + args[0].line, + args[0].column, + args[1].end_line, + args[1].end_column, + self.__get_element_code( + args[0].line, + args[0].column, + args[1].end_line, + args[1].end_column, + ), + ), + args[0], + args[1], + ) + for i in range(2, len(args)): + res = Sum( + ElementInfo( + res.line, + res.column, + args[i].end_line, + args[i].end_column, + self.__get_element_code( + res.line, + res.column, + args[i].end_line, + args[i].end_column, + ), + ), + res, + args[i], + ) + res.line, res.column = meta.line, meta.column + res.end_line, res.end_column = meta.end_line, meta.end_column + + return res + + @v_args(meta=True) + def expr_term(self, meta: Meta, args: List) -> Expr: + if len(args) == 0: + return Null(self.__get_element_info(meta)) + elif len(args) == 1: + if isinstance(args[0], Tree) and args[0].data == "heredoc_template": + return String( + self.__parse_heredoc(args[0]), + self.__get_element_info(meta), + ) + if isinstance(args[0], Expr): + return args[0] + if args[0].type == "STRING_LIT": + return String( + args[0].value[1:-1], + self.__get_element_info(args[0]), + ) + return args[0] + return args + + def object_elem(self, args: List) -> Expr: + if len(args) == 2: + return (args[0], args[1]) + else: + return (args[0], args[2]) + + @v_args(meta=True) + def object(self, meta: Meta, args: List) -> Any: + object_elems = {} + for k, v in args: + object_elems[k] = v + res = Hash(object_elems, self.__get_element_info(meta)) + return res + + @v_args(meta=True) + def tuple(self, meta: Meta, args: List) -> Any: + return Array(args, self.__get_element_info(meta)) + + @v_args(meta=True) + def block(self, meta: Meta, args: List) -> Any: + + if args[0].value == "task": + def get_task_type(atts:List[Any]) -> str: + for elem in atts: + if isinstance(elem,(Attribute,UnitBlock)): + if elem.name == "driver": + return elem.value.value + + name:String = String(args[1].value,self.__get_element_info(args[1])) + au = AtomicUnit(name,f"task.{get_task_type(args[-1])}") + for elem in args[-1]: + if isinstance(elem,Attribute): + au.add_attribute(elem) + else: + print(f"#####ERROR PARSING ELEMENT:{elem} IN A TASK#####") + return au + + elif args[0].value == "group": + ub:UnitBlock = UnitBlock(args[1].value,UnitBlockType.block) + + for elem in args[-1]: + if isinstance(elem,AtomicUnit): + ub.add_atomic_unit(elem) + elif isinstance(elem,Attribute): + ub.add_attribute(elem) + else: + print(f"#####ERROR PARSING ELEMENT:{elem} IN A GROUP#####") + return ub + + elif args[0].value not in ["job","group","task","port"]: + + subatts: Dict[String, Expr] = {} + + for elem in args[-1]: + if isinstance(elem, Attribute): + if isinstance(elem.value, VariableReference): + # Attribute(VariableReference()) + # Review this decision, I might be throwing away important info/expressivity of the IR by doing VariableReference -> String + subatts[ + String(elem.name, ElementInfo.from_code_element(elem)) + ] = String( + elem.value.value, ElementInfo.from_code_element(elem.value) + ) + else: + if isinstance(elem.name, VariableReference): + subatts[elem.name] = elem.value + else: + subatts[ + String(elem.name, ElementInfo.from_code_element(elem)) + ] = elem.value + else: + print(f"#####ERROR PARSING ELEMENT:{elem} IN A BLOCK#####") + + info: ElementInfo = self.__get_element_info(meta) + val_hash: Hash = Hash(subatts, info) + + return Attribute(args[0].value, val_hash, info) + + elif args[0].value == "port": + # port block inside network block inside resource blocks | args[1] is the string literal that has the port name + atts: Dict[String, Expr] = {} + + for elem in args[-1]: + atts[String(elem.name, ElementInfo.from_code_element(elem))] = ( + elem.value + ) + + val_hash: Hash = Hash( + { + String("port_name", self.__get_element_info(args[1])): String( + args[1].value, self.__get_element_info(args[1]) + ), + **atts, + }, + self.__get_element_info(meta), + ) + + return Attribute(args[0], val_hash, self.__get_element_info(meta)) + + else: + #job + ub = UnitBlock(args[0].value, UnitBlockType.script) + for arg in args[-1]: + if isinstance(arg, AtomicUnit): + ub.add_atomic_unit(arg) + elif isinstance(arg, UnitBlock): + ub.add_unit_block(arg) + elif isinstance(arg, Attribute): + ub.add_attribute(arg) + + ub.set_element_info(self.__get_element_info(meta)) + return ub + + def body(self, args: List) -> Any: + return args + + @v_args(meta=True) + def conditional(self, meta: Meta, args: List) -> Any: + condition = ConditionalStatement( + args[0], + ConditionalStatement.ConditionType.IF, + ) + condition.line, condition.column = meta.line, meta.column + condition.end_line, condition.end_column = meta.end_line, meta.end_column + condition.add_statement(args[1]) + condition.else_statement = ConditionalStatement( + Null(), + ConditionalStatement.ConditionType.IF, + ) + condition.else_statement.add_statement(args[2]) + return condition + + @v_args(meta=True) + def attribute(self, meta: Meta, args: List) -> Attribute: + return Attribute(args[0].value, args[2], self.__get_element_info(meta)) + + @v_args(meta=True) + def identifier(self, meta: Meta, value: Any) -> Expr: + if value[0] == "null": + return Null(self.__get_element_info(meta)) + elif value[0] in ["true", "false"]: + return Boolean(value[0] == "true", self.__get_element_info(meta)) + name = value[0] + if isinstance(name, Token): + name = name.value + return VariableReference(name, self.__get_element_info(meta)) + + @v_args(meta=True) + def attr_splat_expr_term(self, meta: Meta, args: List) -> Any: + # TODO: Not supported yet + return Null(self.__get_element_info(meta)) + + @v_args(meta=True) + def full_splat_expr_term(self, meta: Meta, args: List) -> Any: + # TODO: Not supported yet + return Null(self.__get_element_info(meta)) + + @v_args(meta=True) + def for_tuple_expr(self, meta: Meta, args: List) -> Any: + # TODO: Not supported yet + return Null(self.__get_element_info(meta)) + + @v_args(meta=True) + def for_object_expr(self, meta: Meta, args: List) -> Any: + # TODO: Not supported yet + return Null(self.__get_element_info(meta)) + + @v_args(meta=True) + def function_call(self, meta: Meta, args: List) -> Any: + if len(args) == 1: + return FunctionCall( + args[0], + [], + self.__get_element_info(meta), + ) + return FunctionCall( + args[0], + args[1], + self.__get_element_info(meta), + ) + + def arguments(self, args: List) -> Any: + return args + + def start(self, args: List): + return args[0] + + +class NomadParser(p.Parser): + def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: + + try: + with open(path) as f: + unit_block = None + tree = hcl2.parse(f.read() + "\n") + f.seek(0, 0) + code = f.readlines() + transformer = GLITCHTransformer(code) + elements = transformer.transform(tree) + + if elements and isinstance(elements[0], UnitBlock): + unit_block = elements[0] + unit_block.path = path + else: + throw_exception(EXCEPTIONS["HASHICORP_NOMAD_COULD_NOT_PARSE"], path) + return None + + for c in transformer.comments: + unit_block.add_comment(c) + except: + throw_exception(EXCEPTIONS["HASHICORP_NOMAD_COULD_NOT_PARSE"], path) + return None + + return unit_block + + def parse_module(self, path: str) -> Module: + res: Module = Module(os.path.basename(os.path.normpath(path)), path) + super().parse_file_structure(res.folder, path) + + files = [ + f.path for f in os.scandir(f"{path}") if f.is_file() and not f.is_symlink() + ] + for f in files: + unit_block = self.parse_file(f, UnitBlockType.unknown) + res.add_block(unit_block) + + return res + + def parse_folder(self, path: str) -> Project: + res: Project = Project(os.path.basename(os.path.normpath(path))) + res.add_module(self.parse_module(path)) + + subfolders = [ + f.path for f in os.scandir(f"{path}") if f.is_dir() and not f.is_symlink() + ] + for d in subfolders: + aux = self.parse_folder(d) + res.blocks += aux.blocks + res.modules += aux.modules + + return res + + + + + + + + From ae6b6b18177dc98bc1a9e1e45ad40b73bae3db07 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 11 Apr 2025 23:59:29 +0100 Subject: [PATCH 03/65] add Swarm parser --- glitch/parsers/swarm.py | 223 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 glitch/parsers/swarm.py diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py new file mode 100644 index 00000000..26cf7a96 --- /dev/null +++ b/glitch/parsers/swarm.py @@ -0,0 +1,223 @@ +# type: ignore #TODO + +import os +from typing import Any, List, Optional + +from ruamel.yaml.main import YAML +from ruamel.yaml.nodes import ( + MappingNode, + Node, +) +from ruamel.yaml.tokens import Token + +from glitch.exceptions import EXCEPTIONS, throw_exception +from glitch.parsers.yaml import YamlParser +from glitch.repr.inter import ( + AtomicUnit, + Attribute, + Comment, + ElementInfo, + Expr, + Module, + Project, + String, + UnitBlock, + UnitBlockType, +) + + +class SwarmParser(YamlParser): + """ + Stack/Compose YAML files parser + """ + + def parse_atomic_unit( + self, type: str, unit_block: UnitBlock, dict: tuple[Any, Any], code: List[str] + ) -> None: + """ + Parses and creates AtomicUnits + """ + def create_atomic_unit( + start_line: Token | Node, + end_line: Token | Node, + type: str, + name: str, + code: List[str], + ) -> AtomicUnit: + name_info = ElementInfo( + start_line.start_mark.line + 1, + start_line.start_mark.column + 1, + start_line.end_mark.line + 1, + start_line.end_mark.column + 1, + self._get_code(start_line, start_line, code), + ) + str_name = String(name, name_info) + + au = AtomicUnit(str_name, type) + au.line = start_line.start_mark.line + 1 + au.end_line = end_line.end_mark.line + 1 + au.end_column = end_line.end_mark.column + 1 + au.column = start_line.start_mark.column + 1 + au.code = self._get_code(start_line, end_line, code) + return au + + au: AtomicUnit = create_atomic_unit( + dict[0], dict[1], type[:-1], dict[0].value, code + ) + au.attributes += self.__parse_attributes(dict[1], code) + + unit_block.add_atomic_unit(au) + + def __parse_attributes(self, val: Any, code: List[str]) -> List[Attribute]: + """ + Parses the Attributes of an AtomicUnit + """ + + def create_attribute(token: Token | Node, name: str, value: Any) -> Attribute: + info: ElementInfo = ElementInfo( + token.start_mark.line + 1, + token.start_mark.column + 1, + token.end_mark.line + 1, + token.end_mark.column + 1, + self._get_code(token, token, code), + ) + a: Attribute = Attribute(name, value, info) + attributes.append(a) + return a + + attributes: List[Attribute] = [] + + if isinstance(val, MappingNode): + for att in val.value: + if isinstance(att, tuple): + att_value: Expr = self.get_value(att[1], code) + create_attribute(att[0], att[0].value, att_value) + + return attributes + + def parse_file( + self, path: str, type: UnitBlockType = UnitBlockType.script + ) -> Optional[UnitBlock]: + """ + Parses a stack/compose file into a UnitBlock each with its respective + AtomicUnits (each of the services,networks,volumes,configs and secrets) + and their Attributes + """ + try: + with open(path, "r") as f: + try: + parsed_file = YAML().compose(f) + f.seek(0, 0) + code: List[str] = f.readlines() + code.append("") + f.seek(0, 0) + except: + throw_exception(EXCEPTIONS["DOCKER_SWARM_COULD_NOT_PARSE"], path) + return None + if isinstance(parsed_file, MappingNode): + file_unit_block: UnitBlock = UnitBlock( + os.path.basename(os.path.normpath(path)), type + ) + file_unit_block.path = path + if isinstance(parsed_file.value, list): + for field in parsed_file.value: + if field[0].value == "version": + expr: Expr = self.get_value(field[1], code) + info: ElementInfo = ElementInfo( + field[0].start_mark.line + 1, + field[0].start_mark.column + 1, + field[1].end_mark.line + 1, + field[1].end_mark.column + 1, + self._get_code(field[0], field[1], code), + ) + att: Attribute = Attribute(field[0].value, expr, info) + file_unit_block.add_attribute(att) + elif field[0].value in [ + "services", + "networks", + "volumes", + "configs", + "secrets", + ]: + unit_block = UnitBlock( + field[0].value, UnitBlockType.block + ) + unit_block.line = field[0].start_mark.line + unit_block.column = field[0].start_mark.column + unit_block.end_line = field[0].end_mark.line + unit_block.end_column = field[0].end_mark.column + + for unit in field[1].value: + self.parse_atomic_unit( + field[0].value, unit_block, unit, code + ) + file_unit_block.add_unit_block(unit_block) + + else: + throw_exception( + EXCEPTIONS["DOCKER_SWARM_COULD_NOT_PARSE"], path + ) + + for comment in self._get_comments(parsed_file, f): + c = Comment(comment[1]) + c.line = comment[0] + c.code = code[c.line - 1] + file_unit_block.add_comment(c) + file_unit_block.code = "".join(code) + return file_unit_block + except: + throw_exception(EXCEPTIONS["DOCKER_SWARM_COULD_NOT_PARSE"], path) + + def parse_folder(self, path: str, root: bool = True) -> Optional[Project]: + """ + Swarm doesn't have a standard/sample directory layout, + but normally the stack/compose files are either at the root of + a projects folder, all in a specific folder or a stack for + different parts of the system are in each part subfolder + we consider each subfolder a Module + """ + if root: + res: Project = Project(os.path.basename(os.path.normpath(path))) + + subfolders = [ + f.path + for f in os.scandir(f"{path}") + if f.is_dir() and not f.is_symlink() + ] + + for d in subfolders: + res.add_module(self.parse_module(d)) + + files = [ + f.path + for f in os.scandir(f"{path}") + if f.is_file() + and not f.is_symlink() + and f.path.endswith((".yml", ".yaml")) + ] + + for fi in files: + res.add_block(self.parse_file(fi)) + return res + else: + return None + + def parse_module(self, path) -> Module: + """ + We consider each subfolder of the Project folder a Module + + TODO:Think if it is worth considering searching for modules recursively + as done for other languagues supported by GLITCH + """ + res: Module = Module(os.path.basename(os.path.normpath(path)), path) + + files = [ + f.path + for f in os.scandir(f"{path}") + if f.is_file() and not f.is_symlink() and f.path.endswith((".yml", ".yaml")) + ] + + for fi in files: + res.add_block(self.parse_file(fi)) + + return res From 5a1e69b2016be522c47ea026b02c1903c075ba25 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 12 Apr 2025 00:05:42 +0100 Subject: [PATCH 04/65] fix incorrect exception name --- glitch/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/exceptions.py b/glitch/exceptions.py index 6cde3483..ce89b842 100644 --- a/glitch/exceptions.py +++ b/glitch/exceptions.py @@ -17,7 +17,7 @@ "SHELL_COULD_NOT_PARSE": "Shell Command - Could not parse: {}", "TERRAFORM_COULD_NOT_PARSE": "Terraform - Could not parse file: {}", "DOCKER_SWARM_COULD_NOT_PARSE": "Swarm - Could not parse file: {}", - "NOMAD_COULD_NOT_PARSE": "Nomad - Could not parse file: {}", + "HASHICORP_NOMAD_COULD_NOT_PARSE": "Nomad - Could not parse file: {}", } From 4c0178ceb7c90ca58f685d7b91d24c71c3df2bc3 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:52:00 +0100 Subject: [PATCH 05/65] format swarm.py --- glitch/parsers/swarm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 26cf7a96..5d1d4ce2 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -35,8 +35,9 @@ def parse_atomic_unit( self, type: str, unit_block: UnitBlock, dict: tuple[Any, Any], code: List[str] ) -> None: """ - Parses and creates AtomicUnits + Parses and creates AtomicUnits """ + def create_atomic_unit( start_line: Token | Node, end_line: Token | Node, @@ -206,7 +207,7 @@ def parse_module(self, path) -> Module: """ We consider each subfolder of the Project folder a Module - TODO:Think if it is worth considering searching for modules recursively + TODO:Think if it is worth considering searching for modules recursively as done for other languagues supported by GLITCH """ res: Module = Module(os.path.basename(os.path.normpath(path)), path) From 10dd8e63d040618938da83056b9be341a19db3d0 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:56:40 +0100 Subject: [PATCH 06/65] change Terraform parser get_element_info method visibility to protected so it can be accessible to subclasses --- glitch/parsers/terraform.py | 54 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/glitch/parsers/terraform.py b/glitch/parsers/terraform.py index e10f1f3e..eefddec7 100644 --- a/glitch/parsers/terraform.py +++ b/glitch/parsers/terraform.py @@ -51,7 +51,7 @@ def __get_element_code( return res - def __get_element_info(self, meta: Meta | Token) -> ElementInfo: + def _get_element_info(self, meta: Meta | Token) -> ElementInfo: return ElementInfo( meta.line, meta.column, @@ -113,9 +113,9 @@ def binary_op(self, meta: Meta, args: List) -> Any: @v_args(meta=True) def unary_op(self, meta: Meta, args: List) -> Any: if args[0] == "-": - return Minus(self.__get_element_info(meta), args[1]) + return Minus(self._get_element_info(meta), args[1]) elif args[0] == "!": - return Not(self.__get_element_info(meta), args[1]) + return Not(self._get_element_info(meta), args[1]) @v_args(meta=True) def get_attr(self, meta: Meta, args: List) -> Any: @@ -127,19 +127,19 @@ def index(self, meta: Meta, args: List) -> Any: @v_args(meta=True) def index_expr_term(self, meta: Meta, args: List) -> Any: - return Access(self.__get_element_info(meta), args[0], args[1]) + return Access(self._get_element_info(meta), args[0], args[1]) @v_args(meta=True) def get_attr_expr_term(self, meta: Meta, args: List) -> Any: - return Access(self.__get_element_info(meta), args[0], args[1]) + return Access(self._get_element_info(meta), args[0], args[1]) @v_args(meta=True) def int_lit(self, meta: Meta, args: List) -> int: - return Integer(int("".join(args)), self.__get_element_info(meta)) + return Integer(int("".join(args)), self._get_element_info(meta)) @v_args(meta=True) def float_lit(self, meta: Meta, args: List) -> float: - return Float(float("".join(args)), self.__get_element_info(meta)) + return Float(float("".join(args)), self._get_element_info(meta)) @v_args(meta=True) def interpolation_maybe_nested(self, meta: Meta, args: List) -> Any: @@ -151,7 +151,7 @@ def string_with_interpolation(self, meta: Meta, args: List) -> str: if isinstance(args[0], Token): return String( args[0].value, - self.__get_element_info(meta), + self._get_element_info(meta), ) return args[0] else: @@ -159,7 +159,7 @@ def string_with_interpolation(self, meta: Meta, args: List) -> str: if isinstance(args[i], Token): args[i] = String( args[i].value, - self.__get_element_info(args[i]), + self._get_element_info(args[i]), ) res = Sum( @@ -203,19 +203,19 @@ def string_with_interpolation(self, meta: Meta, args: List) -> str: @v_args(meta=True) def expr_term(self, meta: Meta, args: List) -> Expr: if len(args) == 0: - return Null(self.__get_element_info(meta)) + return Null(self._get_element_info(meta)) elif len(args) == 1: if isinstance(args[0], Tree) and args[0].data == "heredoc_template": return String( self.__parse_heredoc(args[0]), - self.__get_element_info(meta), + self._get_element_info(meta), ) if isinstance(args[0], Expr): return args[0] if args[0].type == "STRING_LIT": return String( args[0].value[1:-1], - self.__get_element_info(args[0]), + self._get_element_info(args[0]), ) return args[0] return args @@ -231,12 +231,12 @@ def object(self, meta: Meta, args: List) -> Any: object_elems = {} for k, v in args: object_elems[k] = v - res = Hash(object_elems, self.__get_element_info(meta)) + res = Hash(object_elems, self._get_element_info(meta)) return res @v_args(meta=True) def tuple(self, meta: Meta, args: List) -> Any: - return Array(args, self.__get_element_info(meta)) + return Array(args, self._get_element_info(meta)) @v_args(meta=True) def block(self, meta: Meta, args: List) -> Any: @@ -244,12 +244,12 @@ def block(self, meta: Meta, args: List) -> Any: au = AtomicUnit( String( args[2].value[1:-1], # Remove quotes - self.__get_element_info(args[2]), + self._get_element_info(args[2]), ), args[1].value[1:-1], ) au.attributes = [] - au.set_element_info(self.__get_element_info(meta)) + au.set_element_info(self._get_element_info(meta)) for arg in args[-1]: if isinstance(arg, Attribute): au.attributes.append(arg) @@ -275,7 +275,7 @@ def block(self, meta: Meta, args: List) -> Any: else: ub.add_attribute(arg) - ub.set_element_info(self.__get_element_info(meta)) + ub.set_element_info(self._get_element_info(meta)) return ub def body(self, args: List) -> Any: @@ -299,38 +299,38 @@ def conditional(self, meta: Meta, args: List) -> Any: @v_args(meta=True) def attribute(self, meta: Meta, args: List) -> Attribute: - return Attribute(args[0].value, args[2], self.__get_element_info(meta)) + return Attribute(args[0].value, args[2], self._get_element_info(meta)) @v_args(meta=True) def identifier(self, meta: Meta, value: Any) -> Expr: if value[0] == "null": - return Null(self.__get_element_info(meta)) + return Null(self._get_element_info(meta)) elif value[0] in ["true", "false"]: - return Boolean(value[0] == "true", self.__get_element_info(meta)) + return Boolean(value[0] == "true", self._get_element_info(meta)) name = value[0] if isinstance(name, Token): name = name.value - return VariableReference(name, self.__get_element_info(meta)) + return VariableReference(name, self._get_element_info(meta)) @v_args(meta=True) def attr_splat_expr_term(self, meta: Meta, args: List) -> Any: # TODO: Not supported yet - return Null(self.__get_element_info(meta)) + return Null(self._get_element_info(meta)) @v_args(meta=True) def full_splat_expr_term(self, meta: Meta, args: List) -> Any: # TODO: Not supported yet - return Null(self.__get_element_info(meta)) + return Null(self._get_element_info(meta)) @v_args(meta=True) def for_tuple_expr(self, meta: Meta, args: List) -> Any: # TODO: Not supported yet - return Null(self.__get_element_info(meta)) + return Null(self._get_element_info(meta)) @v_args(meta=True) def for_object_expr(self, meta: Meta, args: List) -> Any: # TODO: Not supported yet - return Null(self.__get_element_info(meta)) + return Null(self._get_element_info(meta)) @v_args(meta=True) def function_call(self, meta: Meta, args: List) -> Any: @@ -338,12 +338,12 @@ def function_call(self, meta: Meta, args: List) -> Any: return FunctionCall( args[0], [], - self.__get_element_info(meta), + self._get_element_info(meta), ) return FunctionCall( args[0], args[1], - self.__get_element_info(meta), + self._get_element_info(meta), ) def arguments(self, args: List) -> Any: From 602191ffc5216194c8f8aed0d05b3241ade20e4c Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 25 Apr 2025 17:26:37 +0100 Subject: [PATCH 07/65] changed the Transformer for Nomad to be a new class that extends the GLITCHTransformer used in the Terraform parser --- glitch/parsers/nomad.py | 361 ++++------------------------------------ 1 file changed, 30 insertions(+), 331 deletions(-) diff --git a/glitch/parsers/nomad.py b/glitch/parsers/nomad.py index b7b697bd..8f3370ae 100644 --- a/glitch/parsers/nomad.py +++ b/glitch/parsers/nomad.py @@ -1,278 +1,59 @@ # type: ignore -# Built upon the work done by Nuno Saveedra and João Gonçalves import os from hcl2.parser import hcl2 import glitch.parsers.parser as p - +from glitch.parsers.terraform import GLITCHTransformer from glitch.exceptions import EXCEPTIONS, throw_exception from glitch.repr.inter import * from typing import List, Dict, Any -from lark.tree import Meta, Tree -from lark.lexer import Token -from lark.visitors import Transformer, v_args, Discard +from lark.tree import Meta +from lark.visitors import v_args -class GLITCHTransformer(Transformer): +class NomadTransformer(GLITCHTransformer): """Takes a syntax tree generated by the parser and - transforms it to a dict. + transforms it into a dict. """ def __init__(self, code: List[str]): - self.code = code - self.comments = [] - super().__init__() - - def new_line_or_comment(self, args: List) -> List: - for arg in args: - if isinstance(arg, Token): - if arg.value.startswith(("//", "#", "/*")): - comment = Comment(arg.value) - comment.line, comment.column = arg.line, arg.column - comment.end_line, comment.end_column = arg.end_line, arg.end_column - self.comments.append(comment) - return Discard - - def new_line_and_or_comma(self, args: List) -> List: - return Discard - - def __get_element_code( - self, start_line: int, start_col: int, end_line: int, end_col: int - ) -> str: - if start_line == end_line: - res = self.code[start_line - 1][max(0, start_col - 1) : end_col - 1] - else: - res = self.code[start_line - 1][max(0, start_col - 1) :] - - for line in range(start_line, end_line - 1): - res += self.code[line] - - if start_line != end_line: - res += self.code[end_line - 1][: end_col - 1] - - return res - - def __get_element_info(self, meta: Meta | Token) -> ElementInfo: - return ElementInfo( - meta.line, - meta.column, - meta.end_line, - meta.end_column, - self.__get_element_code( - meta.line, - meta.column, - meta.end_line, - meta.end_column, - ), - ) - - def __get_element_info_from_tokens(self, start: Token, end: Token) -> ElementInfo: - return ElementInfo( - start.line, - start.column, - end.end_line, - end.end_column, - self.__get_element_code( - start.line, - start.column, - end.end_line, - end.end_column, - ), - ) - - def __parse_heredoc(self, tree: Tree) -> str: - res = "" - for arg in tree.children: - res += arg.value - return "\n".join(res.split("\n")[1:-1]) - - @v_args(meta=True) - def binary_op(self, meta: Meta, args: List) -> Any: - op_to_ir = { - "+": Sum, - "-": Subtract, - "*": Multiply, - "/": Divide, - "%": Modulo, - "&&": And, - "||": Or, - "==": Equal, - "!=": NotEqual, - ">": GreaterThan, - "<": LessThan, - ">=": GreaterThanOrEqual, - "<=": LessThanOrEqual, - } - - if args[1].children[0].children[0] in op_to_ir: - return op_to_ir[args[1].children[0].children[0]]( - self.__get_element_info_from_tokens(args[0], args[1].children[1]), - args[0], - args[1].children[1], - ) - - @v_args(meta=True) - def unary_op(self, meta: Meta, args: List) -> Any: - if args[0] == "-": - return Minus(self.__get_element_info(meta), args[1]) - elif args[0] == "!": - return Not(self.__get_element_info(meta), args[1]) - - @v_args(meta=True) - def get_attr(self, meta: Meta, args: List) -> Any: - return args[0] - - @v_args(meta=True) - def index(self, meta: Meta, args: List) -> Any: - return args[0] - - @v_args(meta=True) - def index_expr_term(self, meta: Meta, args: List) -> Any: - return Access(self.__get_element_info(meta), args[0], args[1]) - - @v_args(meta=True) - def get_attr_expr_term(self, meta: Meta, args: List) -> Any: - return Access(self.__get_element_info(meta), args[0], args[1]) - - @v_args(meta=True) - def int_lit(self, meta: Meta, args: List) -> int: - return Integer(int("".join(args)), self.__get_element_info(meta)) - - @v_args(meta=True) - def float_lit(self, meta: Meta, args: List) -> float: - return Float(float("".join(args)), self.__get_element_info(meta)) - - @v_args(meta=True) - def interpolation_maybe_nested(self, meta: Meta, args: List) -> Any: - return args[0] - - @v_args(meta=True) - def string_with_interpolation(self, meta: Meta, args: List) -> str: - if len(args) == 1: - if isinstance(args[0], Token): - return String( - args[0].value, - self.__get_element_info(meta), - ) - return args[0] - else: - for i in range(len(args)): - if isinstance(args[i], Token): - args[i] = String( - args[i].value, - self.__get_element_info(args[i]), - ) - - res = Sum( - ElementInfo( - args[0].line, - args[0].column, - args[1].end_line, - args[1].end_column, - self.__get_element_code( - args[0].line, - args[0].column, - args[1].end_line, - args[1].end_column, - ), - ), - args[0], - args[1], - ) - for i in range(2, len(args)): - res = Sum( - ElementInfo( - res.line, - res.column, - args[i].end_line, - args[i].end_column, - self.__get_element_code( - res.line, - res.column, - args[i].end_line, - args[i].end_column, - ), - ), - res, - args[i], - ) - res.line, res.column = meta.line, meta.column - res.end_line, res.end_column = meta.end_line, meta.end_column - - return res - - @v_args(meta=True) - def expr_term(self, meta: Meta, args: List) -> Expr: - if len(args) == 0: - return Null(self.__get_element_info(meta)) - elif len(args) == 1: - if isinstance(args[0], Tree) and args[0].data == "heredoc_template": - return String( - self.__parse_heredoc(args[0]), - self.__get_element_info(meta), - ) - if isinstance(args[0], Expr): - return args[0] - if args[0].type == "STRING_LIT": - return String( - args[0].value[1:-1], - self.__get_element_info(args[0]), - ) - return args[0] - return args - - def object_elem(self, args: List) -> Expr: - if len(args) == 2: - return (args[0], args[1]) - else: - return (args[0], args[2]) - - @v_args(meta=True) - def object(self, meta: Meta, args: List) -> Any: - object_elems = {} - for k, v in args: - object_elems[k] = v - res = Hash(object_elems, self.__get_element_info(meta)) - return res - - @v_args(meta=True) - def tuple(self, meta: Meta, args: List) -> Any: - return Array(args, self.__get_element_info(meta)) + super().__init__(code) @v_args(meta=True) def block(self, meta: Meta, args: List) -> Any: if args[0].value == "task": - def get_task_type(atts:List[Any]) -> str: + + def get_task_type(atts: List[Any]) -> str: for elem in atts: - if isinstance(elem,(Attribute,UnitBlock)): + if isinstance(elem, (Attribute, UnitBlock)): if elem.name == "driver": return elem.value.value - - name:String = String(args[1].value,self.__get_element_info(args[1])) - au = AtomicUnit(name,f"task.{get_task_type(args[-1])}") + + name: String = String(args[1].value, self._get_element_info(args[1])) + au = AtomicUnit(name, f"task.{get_task_type(args[-1])}") for elem in args[-1]: - if isinstance(elem,Attribute): + if isinstance(elem, Attribute): au.add_attribute(elem) else: print(f"#####ERROR PARSING ELEMENT:{elem} IN A TASK#####") return au elif args[0].value == "group": - ub:UnitBlock = UnitBlock(args[1].value,UnitBlockType.block) + ub: UnitBlock = UnitBlock(args[1].value, UnitBlockType.block) for elem in args[-1]: - if isinstance(elem,AtomicUnit): + if isinstance(elem, AtomicUnit): ub.add_atomic_unit(elem) - elif isinstance(elem,Attribute): + elif isinstance(elem, Attribute): ub.add_attribute(elem) else: print(f"#####ERROR PARSING ELEMENT:{elem} IN A GROUP#####") return ub - - elif args[0].value not in ["job","group","task","port"]: - + + elif args[0].value not in ["job", "group", "task", "port"]: + subatts: Dict[String, Expr] = {} for elem in args[-1]: @@ -295,11 +76,11 @@ def get_task_type(atts:List[Any]) -> str: else: print(f"#####ERROR PARSING ELEMENT:{elem} IN A BLOCK#####") - info: ElementInfo = self.__get_element_info(meta) + info: ElementInfo = self._get_element_info(meta) val_hash: Hash = Hash(subatts, info) return Attribute(args[0].value, val_hash, info) - + elif args[0].value == "port": # port block inside network block inside resource blocks | args[1] is the string literal that has the port name atts: Dict[String, Expr] = {} @@ -311,18 +92,18 @@ def get_task_type(atts:List[Any]) -> str: val_hash: Hash = Hash( { - String("port_name", self.__get_element_info(args[1])): String( - args[1].value, self.__get_element_info(args[1]) + String("port_name", self._get_element_info(args[1])): String( + args[1].value, self._get_element_info(args[1]) ), **atts, }, - self.__get_element_info(meta), + self._get_element_info(meta), ) - return Attribute(args[0], val_hash, self.__get_element_info(meta)) + return Attribute(args[0], val_hash, self._get_element_info(meta)) else: - #job + # job ub = UnitBlock(args[0].value, UnitBlockType.script) for arg in args[-1]: if isinstance(arg, AtomicUnit): @@ -332,94 +113,19 @@ def get_task_type(atts:List[Any]) -> str: elif isinstance(arg, Attribute): ub.add_attribute(arg) - ub.set_element_info(self.__get_element_info(meta)) + ub.set_element_info(self._get_element_info(meta)) return ub - def body(self, args: List) -> Any: - return args - - @v_args(meta=True) - def conditional(self, meta: Meta, args: List) -> Any: - condition = ConditionalStatement( - args[0], - ConditionalStatement.ConditionType.IF, - ) - condition.line, condition.column = meta.line, meta.column - condition.end_line, condition.end_column = meta.end_line, meta.end_column - condition.add_statement(args[1]) - condition.else_statement = ConditionalStatement( - Null(), - ConditionalStatement.ConditionType.IF, - ) - condition.else_statement.add_statement(args[2]) - return condition - - @v_args(meta=True) - def attribute(self, meta: Meta, args: List) -> Attribute: - return Attribute(args[0].value, args[2], self.__get_element_info(meta)) - - @v_args(meta=True) - def identifier(self, meta: Meta, value: Any) -> Expr: - if value[0] == "null": - return Null(self.__get_element_info(meta)) - elif value[0] in ["true", "false"]: - return Boolean(value[0] == "true", self.__get_element_info(meta)) - name = value[0] - if isinstance(name, Token): - name = name.value - return VariableReference(name, self.__get_element_info(meta)) - - @v_args(meta=True) - def attr_splat_expr_term(self, meta: Meta, args: List) -> Any: - # TODO: Not supported yet - return Null(self.__get_element_info(meta)) - - @v_args(meta=True) - def full_splat_expr_term(self, meta: Meta, args: List) -> Any: - # TODO: Not supported yet - return Null(self.__get_element_info(meta)) - - @v_args(meta=True) - def for_tuple_expr(self, meta: Meta, args: List) -> Any: - # TODO: Not supported yet - return Null(self.__get_element_info(meta)) - - @v_args(meta=True) - def for_object_expr(self, meta: Meta, args: List) -> Any: - # TODO: Not supported yet - return Null(self.__get_element_info(meta)) - - @v_args(meta=True) - def function_call(self, meta: Meta, args: List) -> Any: - if len(args) == 1: - return FunctionCall( - args[0], - [], - self.__get_element_info(meta), - ) - return FunctionCall( - args[0], - args[1], - self.__get_element_info(meta), - ) - - def arguments(self, args: List) -> Any: - return args - - def start(self, args: List): - return args[0] - - class NomadParser(p.Parser): def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: - + try: with open(path) as f: unit_block = None tree = hcl2.parse(f.read() + "\n") f.seek(0, 0) code = f.readlines() - transformer = GLITCHTransformer(code) + transformer = NomadTransformer(code) elements = transformer.transform(tree) if elements and isinstance(elements[0], UnitBlock): @@ -434,7 +140,7 @@ def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: except: throw_exception(EXCEPTIONS["HASHICORP_NOMAD_COULD_NOT_PARSE"], path) return None - + return unit_block def parse_module(self, path: str) -> Module: @@ -464,10 +170,3 @@ def parse_folder(self, path: str) -> Project: return res - - - - - - - From 0adf8ae30b2983df8e19bb59af10f3f40456b31e Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 25 Apr 2025 17:29:40 +0100 Subject: [PATCH 08/65] format nomad.py --- glitch/parsers/nomad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/parsers/nomad.py b/glitch/parsers/nomad.py index 8f3370ae..9be4a791 100644 --- a/glitch/parsers/nomad.py +++ b/glitch/parsers/nomad.py @@ -116,6 +116,7 @@ def get_task_type(atts: List[Any]) -> str: ub.set_element_info(self._get_element_info(meta)) return ub + class NomadParser(p.Parser): def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: @@ -169,4 +170,3 @@ def parse_folder(self, path: str) -> Project: res.modules += aux.modules return res - From f5089f4db32f25445b1c1303936d300d61fcd964 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 27 Jun 2025 23:56:53 +0100 Subject: [PATCH 09/65] format nomad.py and fix missing abs file path issue --- glitch/parsers/nomad.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/glitch/parsers/nomad.py b/glitch/parsers/nomad.py index 9be4a791..cf02aa31 100644 --- a/glitch/parsers/nomad.py +++ b/glitch/parsers/nomad.py @@ -19,10 +19,10 @@ class NomadTransformer(GLITCHTransformer): def __init__(self, code: List[str]): super().__init__(code) + self.path = "" @v_args(meta=True) def block(self, meta: Meta, args: List) -> Any: - if args[0].value == "task": def get_task_type(atts: List[Any]) -> str: @@ -42,7 +42,7 @@ def get_task_type(atts: List[Any]) -> str: elif args[0].value == "group": ub: UnitBlock = UnitBlock(args[1].value, UnitBlockType.block) - + ub.path = self.path for elem in args[-1]: if isinstance(elem, AtomicUnit): ub.add_atomic_unit(elem) @@ -53,7 +53,6 @@ def get_task_type(atts: List[Any]) -> str: return ub elif args[0].value not in ["job", "group", "task", "port"]: - subatts: Dict[String, Expr] = {} for elem in args[-1]: @@ -119,7 +118,6 @@ def get_task_type(atts: List[Any]) -> str: class NomadParser(p.Parser): def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: - try: with open(path) as f: unit_block = None @@ -127,11 +125,12 @@ def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: f.seek(0, 0) code = f.readlines() transformer = NomadTransformer(code) + transformer.path = os.path.abspath(path) elements = transformer.transform(tree) if elements and isinstance(elements[0], UnitBlock): unit_block = elements[0] - unit_block.path = path + unit_block.path = os.path.abspath(path) else: throw_exception(EXCEPTIONS["HASHICORP_NOMAD_COULD_NOT_PARSE"], path) return None From 759ff54b409ba2919aa8da31dc030deb5213f24c Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 28 Jun 2025 00:01:37 +0100 Subject: [PATCH 10/65] fixes swarm.py adds support for 'extends' and 'includes' extension --- glitch/parsers/swarm.py | 219 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 208 insertions(+), 11 deletions(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 5d1d4ce2..8588153d 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -6,6 +6,8 @@ from ruamel.yaml.main import YAML from ruamel.yaml.nodes import ( MappingNode, + ScalarNode, + SequenceNode, Node, ) from ruamel.yaml.tokens import Token @@ -18,11 +20,14 @@ Comment, ElementInfo, Expr, + Hash, Module, Project, String, UnitBlock, UnitBlockType, + Array, + Dependency, ) @@ -74,14 +79,21 @@ def __parse_attributes(self, val: Any, code: List[str]) -> List[Attribute]: Parses the Attributes of an AtomicUnit """ - def create_attribute(token: Token | Node, name: str, value: Any) -> Attribute: - info: ElementInfo = ElementInfo( - token.start_mark.line + 1, - token.start_mark.column + 1, - token.end_mark.line + 1, - token.end_mark.column + 1, - self._get_code(token, token, code), - ) + def create_attribute( + token: Token | Node | None, name: str, value: Any, _info: ElementInfo = None + ) -> Attribute: + if _info is not None and token is None: + # HACK: (Part of) Handling transforming attributes coming from ">>" inserts to normal attributes + info = _info + else: + info: ElementInfo = ElementInfo( + token.start_mark.line + 1, + token.start_mark.column + 1, + token.end_mark.line + 1, + token.end_mark.column + 1, + self._get_code(token, token, code), + ) + a: Attribute = Attribute(name, value, info) attributes.append(a) return a @@ -92,6 +104,56 @@ def create_attribute(token: Token | Node, name: str, value: Any) -> Attribute: for att in val.value: if isinstance(att, tuple): att_value: Expr = self.get_value(att[1], code) + if att[0].value == "environment" and isinstance( + att[1], SequenceNode + ): + """ + HACK: Converts all Sequence/Arrays environments to Hash + environment: + - VAR1=123 + - VAR2=456 + vs + environment: + VAR1 : 123 + VAR2 : 456 + + """ + fixed_env = {} + for elem in att_value.value: + elem_info: ElementInfo = ElementInfo.from_code_element(elem) + curr_str = elem.value + split_str = curr_str.split("=") + assert len(split_str) == 2 + key, n_val = split_str + + key_s = String(key, elem_info) + val_s = String(n_val, elem_info) + fixed_env[key_s] = val_s + att_info = ElementInfo.from_code_element(att_value) + att_value = Hash(fixed_env, att_info) + if isinstance(att[1], MappingNode): + # HACK: Handle transforming attributes coming from ">>" inserts to normal attributes + if isinstance(att_value, Hash): + affected_keys = [] + temp_store = {} + + for k, v in att_value.value.items(): + if k.value == "<<": + affected_keys.append(k) + for _k, _v in v.value.items(): + temp_store[_k] = _v + + att_value.value.update(temp_store) + + for elem in affected_keys: + att_value.value.pop(elem) + if att[0].value == "<<" and isinstance(att_value, Hash): + # HACK: Handle transforming attributes coming from ">>" inserts to normal attributes + for k, v in att_value.value.items(): + create_attribute( + None, k.value, v, ElementInfo.from_code_element(v) + ) + else: create_attribute(att[0], att[0].value, att_value) return attributes @@ -106,6 +168,7 @@ def parse_file( """ try: with open(path, "r") as f: + includes = [] try: parsed_file = YAML().compose(f) f.seek(0, 0) @@ -119,7 +182,7 @@ def parse_file( file_unit_block: UnitBlock = UnitBlock( os.path.basename(os.path.normpath(path)), type ) - file_unit_block.path = path + file_unit_block.path = os.path.abspath(path) if isinstance(parsed_file.value, list): for field in parsed_file.value: if field[0].value == "version": @@ -133,6 +196,19 @@ def parse_file( ) att: Attribute = Attribute(field[0].value, expr, info) file_unit_block.add_attribute(att) + elif field[0].value == "include": + includes_temp = self.get_value(field[1], code).value + + for elem in includes_temp: + if isinstance(elem, String): + includes.append(elem.value) + elif isinstance(elem, Hash): + for k, v in elem.value.items(): + if k.value == "path": + includes.append(v.value) + for elem in includes: + file_unit_block.add_dependency(Dependency([elem])) + elif field[0].value in [ "services", "networks", @@ -143,6 +219,7 @@ def parse_file( unit_block = UnitBlock( field[0].value, UnitBlockType.block ) + unit_block.path = os.path.abspath(path) unit_block.line = field[0].start_mark.line unit_block.column = field[0].start_mark.column unit_block.end_line = field[0].end_mark.line @@ -154,6 +231,10 @@ def parse_file( ) file_unit_block.add_unit_block(unit_block) + elif isinstance(field[0], ScalarNode) and isinstance( + field[1], MappingNode + ): + continue else: throw_exception( EXCEPTIONS["DOCKER_SWARM_COULD_NOT_PARSE"], path @@ -165,10 +246,124 @@ def parse_file( c.code = code[c.line - 1] file_unit_block.add_comment(c) file_unit_block.code = "".join(code) + + # FIXME: Handling the includes, might not be the best way + for inc in includes: + curr_path = os.path.split(path)[0] + joint_path = os.path.normpath(os.path.join(curr_path, inc)) + if os.path.exists(joint_path): + include_file_unit_block = self.parse_file(joint_path) + + print(f"HEY{include_file_unit_block}") + for ub in include_file_unit_block.unit_blocks: + for ub_curr in file_unit_block.unit_blocks: + if ub.name == ub_curr.name: + for au in ub.atomic_units: + ub_curr.add_atomic_unit(au) + else: + print( + f'Failed to parse include file expected at "{joint_path}". File not found.' + ) + + to_extend: List[List[AtomicUnit, Attribute]] = [] + + services = [] + for ub in file_unit_block.unit_blocks: + if ub.name == "services": + services = ub.atomic_units + # FIXME: Handling the extends from the same file or from other files, might not be the best way + for service in services: + for attribute in service.attributes: + if attribute.name == "extends": + deps = [] + # if isinstance(attribute.value,String): + # deps.append(attribute.value.value) + if isinstance(attribute.value, Hash): + # adds the name of file as a dependency + for k, v in attribute.value.value: + if k.value == "file": + deps.append(v.value) + break + + file_unit_block.add_dependency(Dependency(deps)) + to_extend.append([service, attribute]) + break + + for service_to, attribute in to_extend: + att: Attribute = attribute + service_from_list = [] + service_from = "" + + if isinstance(att.value, String): + service_from = att.value.value + service_from_list = services + + elif isinstance(att.value, Hash): + hash_dict = att.value.value + file = "" + service_from = "" + for k, v in hash_dict.items(): + if k.value == "file": + file = v.value + elif k.value == "service": + service_from = v.value + curr_path = os.path.split(path)[0] + joint_path = os.path.normpath(os.path.join(curr_path, file)) + if os.path.exists(joint_path): + service_from_file_unit_block = self.parse_file(joint_path) + service_from_list = ( + service_from_file_unit_block.atomic_units + ) + else: + print( + f'Failed to parse extends file expected at "{joint_path}". File not found.' + ) + + for s in service_from_list: + if s.name.value == service_from: + att_names = [x.name for x in service_to.attributes] + + for s_att in s.attributes: + if s_att.name in ["depends_on", "volumes_from"]: + continue + elif s_att.name not in att_names: + service_to.add_attribute(s_att) + elif s_att.name in att_names: + for to_att in service_to.attributes: + if to_att.name == s_att.name: + if isinstance(to_att.value, Array): + self.__handle_array( + s_att.value, to_att.value + ) + elif isinstance(to_att.value, Hash): + self.__handle_hash( + s_att.value, to_att.value + ) + else: + continue + break + break + return file_unit_block except: throw_exception(EXCEPTIONS["DOCKER_SWARM_COULD_NOT_PARSE"], path) + def __handle_array(self, src: Array, dst: Array) -> None: + temp = [elem for elem in src.value if elem not in dst.value] + for elem in dst.value: + temp.append(elem) + dst.value = temp + + def __handle_hash(self, src: Hash, dst: Hash) -> None: + for k, v in src.value.items(): + if k not in dst.value: + dst.value[k] = v + else: + if isinstance(v, Array): + self.__handle_array(v, dst.value[k]) + elif isinstance(v, Hash): + self.__handle_hash(v, dst.value[k]) + def parse_folder(self, path: str, root: bool = True) -> Optional[Project]: """ Swarm doesn't have a standard/sample directory layout, @@ -206,8 +401,10 @@ def parse_folder(self, path: str, root: bool = True) -> Optional[Project]: def parse_module(self, path) -> Module: """ We consider each subfolder of the Project folder a Module - - TODO:Think if it is worth considering searching for modules recursively + + TODO: Think if it is worth considering searching for modules recursively + especially now since includes and extends from other YAMLs are now implemented + as done for other languagues supported by GLITCH """ res: Module = Module(os.path.basename(os.path.normpath(path)), path) From 899368c8bfeab0567980c3eb7c5cecba0058561a Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 28 Jun 2025 00:29:16 +0100 Subject: [PATCH 11/65] fix docker_images_scraper.py: add support for the new search api and add new filtering for official images that are deprecated --- scripts/docker_images_scraper.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/docker_images_scraper.py b/scripts/docker_images_scraper.py index 4e006df3..ce1057e6 100644 --- a/scripts/docker_images_scraper.py +++ b/scripts/docker_images_scraper.py @@ -1,15 +1,28 @@ import requests -next_url = "https://hub.docker.com/api/content/v1/products/search?image_filter=official&page=1&page_size=100&q=&type=image" -headers = {"Accept": "application/json", "Search-Version": "v3"} +next_url = "https://hub.docker.com/api/search/v4?badges=official&size=100&query=&type=image" +headers = {"Accept": "application/json"} +ses = requests.Session() images_list = [] +deprecated = [] +res = ses.get(next_url, headers=headers).json() +total = res["total"] +current = 0 -while next_url: - res = requests.get(next_url, headers=headers).json() - next_url = res["next"] - images = [i["name"] for i in res["summaries"]] - images_list += images +while current < total: + for elem in res["results"]: + if "deprecate" in elem["short_description"].lower(): + deprecated.append(elem["name"]) + else: + images_list.append(elem["name"]) + current += len(res["results"]) + if current < total: + res = ses.get(next_url + f"&from={current}", headers=headers).json() with open("official_images", "w") as f: f.write("\n".join(images_list)) + +with open("official_deprecated_images", "w") as f: + f.write("\n".join(deprecated)) + From 9680335071971a6616605581e3b9fefda57ac97c Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 17 Jul 2025 19:45:13 +0100 Subject: [PATCH 12/65] =?UTF-8?q?remove=20forgotten=20print=20=F0=9F=98=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- glitch/parsers/swarm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 8588153d..e64b640a 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -254,7 +254,6 @@ def parse_file( if os.path.exists(joint_path): include_file_unit_block = self.parse_file(joint_path) - print(f"HEY{include_file_unit_block}") for ub in include_file_unit_block.unit_blocks: for ub_curr in file_unit_block.unit_blocks: if ub.name == ub_curr.name: From 70314127c1a0b8701092beead08eb78dacd232bd Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 17 Jul 2025 19:48:06 +0100 Subject: [PATCH 13/65] fix boolean parsing --- glitch/parsers/swarm.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index e64b640a..b152c09a 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -2,7 +2,7 @@ import os from typing import Any, List, Optional - +import re from ruamel.yaml.main import YAML from ruamel.yaml.nodes import ( MappingNode, @@ -103,6 +103,20 @@ def create_attribute( if isinstance(val, MappingNode): for att in val.value: if isinstance(att, tuple): + if isinstance(att[1], ScalarNode) and att[1].tag.endswith("bool"): + # HACK: turn boolean scalar node values strings into + # real booleans values for get_value method, + # taking into account yaml 1.1 using the spec provided regexp (used by compose) + + if re.match( + "y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON", att[1].value + ): + att[1].value = True + elif re.match( + "n|N|no|No|NO|false|False|FALSE|off|Off|OFF", att[1].value + ): + att[1].value = False + att_value: Expr = self.get_value(att[1], code) if att[0].value == "environment" and isinstance( att[1], SequenceNode From c32defac733d7463a0f38c2512be7db4dd401d90 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:07:37 +0100 Subject: [PATCH 14/65] swarm and nomad tech introduction --- glitch/__main__.py | 6 ++++++ glitch/tech.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/glitch/__main__.py b/glitch/__main__.py index 0d8cb9d5..9f9ca501 100644 --- a/glitch/__main__.py +++ b/glitch/__main__.py @@ -18,6 +18,8 @@ from glitch.parsers.puppet import PuppetParser from glitch.parsers.terraform import TerraformParser from glitch.parsers.gha import GithubActionsParser +from glitch.parsers.swarm import SwarmParser +from glitch.parsers.nomad import NomadParser from glitch.exceptions import throw_exception from glitch.repair.interactive.main import run_infrafix from pkg_resources import resource_filename @@ -90,6 +92,10 @@ def __get_parser(tech: Tech) -> Parser: return TerraformParser() elif tech == Tech.gha: return GithubActionsParser() + elif tech == Tech.swarm: + return SwarmParser() + elif tech == Tech.nomad: + return NomadParser() else: raise ValueError(f"Invalid tech: {tech}") diff --git a/glitch/tech.py b/glitch/tech.py index ad321319..167e719f 100644 --- a/glitch/tech.py +++ b/glitch/tech.py @@ -13,3 +13,5 @@ def __init__(self, tech: str, extensions: List[str]): terraform = "terraform", ["tf"] docker = "docker", ["Dockerfile"] gha = "github-actions", ["yml", "yaml"] + swarm = "swarm", ["yml", "yaml"] + nomad = "nomad", ["hcl", "nomad", "job"] From 4cd85740fdd2c0fd9b5e049c8ac6a3846f668735 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:09:23 +0100 Subject: [PATCH 15/65] fix add more common tar compressed with bzip2 extension, also for zstd compressed tar --- glitch/configs/default.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/configs/default.ini b/glitch/configs/default.ini index f2efba19..f5755b29 100644 --- a/glitch/configs/default.ini +++ b/glitch/configs/default.ini @@ -14,7 +14,7 @@ secrets = ["auth_token", "authetication_token","auth-token", "authentication-tok "ssh-key-public-content", "ssh-key-private-content"] misc_secrets = ["key", "cert"] roles = [] -download_extensions = ["iso", "tar", "tar.gz", "tar.bzip2", "zip", +download_extensions = ["iso", "tar", "tar.gz", "tar.bzip2", "tar.bz2","tar.zst", "zip", "rar", "gzip", "gzip2", "deb", "rpm", "sh", "run", "bin", "tgz"] ssh_dirs = ["source", "destination", "path", "directory", "src", "dest", "file"] admin = ["admin", "root"] From 602602bc666e50b2af9fc716bdeb113a3acb7150 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:10:20 +0100 Subject: [PATCH 16/65] adding the smells error messages --- glitch/analysis/rules.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/glitch/analysis/rules.py b/glitch/analysis/rules.py index 2d2087d5..8243f08b 100644 --- a/glitch/analysis/rules.py +++ b/glitch/analysis/rules.py @@ -26,6 +26,31 @@ class Error: Tech.docker: { "sec_non_official_image": "Use of non-official Docker image - Use of non-official images should be avoided or taken into careful consideration. (CWE-829)", }, + Tech.swarm: { + "sec_non_official_image": "Use of non-official Docker image - Use of non-official images should be avoided or taken into careful consideration. (CWE-829)", + "sec_image_integrity": "Use of image is not tagged with digest - The images downloaded from the internet should be checked. (CWE-353)", + "sec_unstable_tag": "Unstable version/release image tag - Prefer specifying a release version or better yet a specific version digest (CWE-353)", + "sec_no_image_tag": "The image is not tagged - Prefer specifying a release version or better yet specific version digest", + "arc_no_apig": "No API Gateway - If following a microservices architecture you should use in front of your services an API Gateway instead of directly exposing them.", + "arc_no_logging": "Log Collection not found - It is advised to setup logging for your services.", + "arc_wobbly": "Missing Healthchecks - You should setup healthchecks for your services, or check if the images used already have default ones.", + "sec_mounted_docker_socket": "Docker socket mounted to a container - Avoid mounting the Docker socket to container, if the container is compromised its access to the Docker socket allows control of all other containers, and even acquiring control of the host machine.", + "sec_privileged_containers": "Use of Privileged Containers - Developers should always try to give and use the least privileges possible. Use of privileged containers severely thins out the security and isolation provided by container runtimes, its use should be avoided as much as possible (CWE-250)", + "sec_depr_off_imgs" : "Use of deprecated official Docker images - Use of official deprecated images should be avoided as it makes you open to vulnerabilities, quality issues and unfixed bugs (CWE-1104)", + }, + Tech.nomad: { + "sec_non_official_image": "Use of non-official Docker image - Use of non-official images should be avoided or taken into careful consideration. (CWE-829)", + "sec_image_integrity": "Container image is not tagged with digest - The images downloaded from the internet should be checked. (CWE-353)", + "sec_unstable_tag": "Unstable version/release image tag - Prefer specifying a release version or better yet a specific version digest (CWE-353)", + "sec_no_image_tag": "Container Image is not tagged - Prefer specifying a release version or better yet specific version digest (CWE-353)", + "arc_no_apig": "No API Gateway - If following a microservices architecture you should use in front of your services an API Gateway instead of directly exposing them.", + "arc_no_logging": "Log Collection not found - It is advised to setup logging for your services.", + "arc_wobbly": "Missing Healthchecks - You should setup healthchecks for your services, or check if the images used already have default ones.", + "sec_mounted_docker_socket": "Docker socket mounted to a container - Avoid mounting the Docker socket to container, if the container is compromised its access to the Docker socket allows control of all other containers, and even acquiring control of the host machine.", + "sec_privileged_containers": "Use of Privileged Containers - Developers should always try to give and use the least privileges possible. Use of privileged containers severely thins out the security and isolation provided by container runtimes, its use should be avoided as much as possible (CWE-250)", + "arc_multiple_services": "Multiple Services per Deployment Unit - If you are following a Microservices architecture you are violating the idependent deployability rule by deploying multiple microservices in the same group.", + "sec_depr_off_imgs" : "Use of deprecated official Docker images - Use of official deprecated images should be avoided as it makes you open to vulnerabilities, quality issues and unfixed bugs (CWE-1104)", + }, Tech.terraform: { "sec_integrity_policy": "Integrity Policy - Image tag is prone to be mutable or integrity monitoring is disabled. (CWE-471)", "sec_ssl_tls_policy": "SSL/TLS/mTLS Policy - Developers should use SSL/TLS/mTLS protocols and their secure versions. (CWE-326)", From 808439c14274680f5f882f9de83fd434f7db726e Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:11:38 +0100 Subject: [PATCH 17/65] swarm and nomad parsers fixes --- glitch/parsers/nomad.py | 7 ++-- glitch/parsers/swarm.py | 72 ++++++++++++++++++++--------------------- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/glitch/parsers/nomad.py b/glitch/parsers/nomad.py index cf02aa31..be625b59 100644 --- a/glitch/parsers/nomad.py +++ b/glitch/parsers/nomad.py @@ -33,11 +33,14 @@ def get_task_type(atts: List[Any]) -> str: name: String = String(args[1].value, self._get_element_info(args[1])) au = AtomicUnit(name, f"task.{get_task_type(args[-1])}") + + au.set_element_info(self._get_element_info(meta)) for elem in args[-1]: if isinstance(elem, Attribute): au.add_attribute(elem) else: print(f"#####ERROR PARSING ELEMENT:{elem} IN A TASK#####") + return au elif args[0].value == "group": @@ -91,7 +94,7 @@ def get_task_type(atts: List[Any]) -> str: val_hash: Hash = Hash( { - String("port_name", self._get_element_info(args[1])): String( + String("port", self._get_element_info(args[1])): String( args[1].value, self._get_element_info(args[1]) ), **atts, @@ -140,7 +143,7 @@ def parse_file(self, path: str, type: UnitBlockType) -> UnitBlock: except: throw_exception(EXCEPTIONS["HASHICORP_NOMAD_COULD_NOT_PARSE"], path) return None - + return unit_block def parse_module(self, path: str) -> Module: diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index b152c09a..d651cd87 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -19,6 +19,7 @@ Attribute, Comment, ElementInfo, + Null, Expr, Hash, Module, @@ -136,12 +137,15 @@ def create_attribute( for elem in att_value.value: elem_info: ElementInfo = ElementInfo.from_code_element(elem) curr_str = elem.value - split_str = curr_str.split("=") - assert len(split_str) == 2 - key, n_val = split_str + split_str = curr_str.split("=",1) + if len(split_str) == 2: + key, n_val = split_str + val_s = String(n_val, elem_info) + else: + key = curr_str + val_s = Null(elem_info) key_s = String(key, elem_info) - val_s = String(n_val, elem_info) fixed_env[key_s] = val_s att_info = ElementInfo.from_code_element(att_value) att_value = Hash(fixed_env, att_info) @@ -293,7 +297,7 @@ def parse_file( # deps.append(attribute.value.value) if isinstance(attribute.value, Hash): # adds the name of file as a dependency - for k, v in attribute.value.value: + for k, v in attribute.value.value.items(): if k.value == "file": deps.append(v.value) break @@ -357,7 +361,7 @@ def parse_file( break break - return file_unit_block + return file_unit_block except: throw_exception(EXCEPTIONS["DOCKER_SWARM_COULD_NOT_PARSE"], path) @@ -385,42 +389,38 @@ def parse_folder(self, path: str, root: bool = True) -> Optional[Project]: different parts of the system are in each part subfolder we consider each subfolder a Module """ - if root: - res: Project = Project(os.path.basename(os.path.normpath(path))) - - subfolders = [ - f.path - for f in os.scandir(f"{path}") - if f.is_dir() and not f.is_symlink() - ] - - for d in subfolders: - res.add_module(self.parse_module(d)) - - files = [ - f.path - for f in os.scandir(f"{path}") - if f.is_file() - and not f.is_symlink() - and f.path.endswith((".yml", ".yaml")) - ] - - for fi in files: - res.add_block(self.parse_file(fi)) - return res - else: - return None + + res: Project = Project(os.path.basename(os.path.normpath(path))) - def parse_module(self, path) -> Module: - """ - We consider each subfolder of the Project folder a Module + subfolders = [ + f.path + for f in os.scandir(f"{path}") + if f.is_dir() and not f.is_symlink() + ] - TODO: Think if it is worth considering searching for modules recursively - especially now since includes and extends from other YAMLs are now implemented + for d in subfolders: + res.add_module(self.parse_module(d)) + files = [ + f.path + for f in os.scandir(f"{path}") + if f.is_file() + and not f.is_symlink() + and f.path.endswith((".yml", ".yaml")) + ] + + for fi in files: + res.add_block(self.parse_file(fi)) + + return res + + def parse_module(self, path) -> Module: + """ + We consider each subfolder of the Project folder a Module as done for other languagues supported by GLITCH """ res: Module = Module(os.path.basename(os.path.normpath(path)), path) + super().parse_file_structure(res.folder,path) files = [ f.path From 4d2fbd98c36655ae103b88739a53600a385879ea Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:12:44 +0100 Subject: [PATCH 18/65] correctly escape the '%' character for copying into latex --- glitch/stats/print.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/stats/print.py b/glitch/stats/print.py index dc0f6dab..a36cb969 100644 --- a/glitch/stats/print.py +++ b/glitch/stats/print.py @@ -100,7 +100,7 @@ def print_stats( "\\textbf{Smell}", "\\textbf{Occurrences}", "\\textbf{Smell density (Smell/KLoC)}", - "\\textbf{Proportion of scripts (%)}", + "\\textbf{Proportion of scripts (\\%)}", ], ) latex = ( # type: ignore From 5c67131e681d8652f4968c70060ba264ac96b1df Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:20:17 +0100 Subject: [PATCH 19/65] adding implementation of Log Collection and some of container image smells, re-enable detection of weak crypto and https missing smells --- glitch/analysis/security/hard_secr.py | 10 +- glitch/analysis/security/visitor.py | 246 ++++++++++++++++++++++++++ 2 files changed, 254 insertions(+), 2 deletions(-) diff --git a/glitch/analysis/security/hard_secr.py b/glitch/analysis/security/hard_secr.py index 0088cf7b..65e9420a 100644 --- a/glitch/analysis/security/hard_secr.py +++ b/glitch/analysis/security/hard_secr.py @@ -28,10 +28,13 @@ def __check_pair( SecurityVisitor.PASSWORDS + SecurityVisitor.SECRETS + SecurityVisitor.USERS ): secr_checker = StringChecker( - lambda s: re.match( + lambda s: (re.match( r"[_A-Za-z0-9$\/\.\[\]-]*{text}\b".format(text=item), s ) - is not None + is not None) or (re.match( + r"[_A-Za-z0-9$\/\.\[\]-]*{text}\b".format(text=item.upper()), s + ) + is not None) ) if secr_checker.check(name) and not whitelist_checker.check(name): if not var_checker.check(value): @@ -77,6 +80,9 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) elif isinstance(element, KeyValue) and isinstance(element.value, Hash): for key, value in element.value.value.items(): + #HACK: ignore cases where values are obtained via environment variables + if isinstance(value,String) and value.value.startswith("${") and value.value.endswith("}"): + continue errors += self.__check_pair(element, key, value, file) return errors diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index ae81dbc1..530dc47f 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -18,6 +18,29 @@ class SecurityVisitor(RuleVisitor): __URL_REGEX = r"^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([_\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$" + @staticmethod + def image_parser(img_name: str): + image, tag, digest = "", "", "" + + if "@" in img_name: + parts_dig = img_name.split("@") + digest = parts_dig[-1] + if ":" in parts_dig[0]: + parts_tag = parts_dig[0].split(":") + image = parts_tag[0] + tag = parts_tag[1] + else: + image = parts_dig[0] + + elif ":" in img_name: + parts_tag = img_name.split(":") + image = parts_tag[0] + tag = parts_tag[1] + else: + image = img_name + + return image, tag, digest + class NonOfficialImageSmell(SmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: return [] @@ -35,6 +58,175 @@ def check(self, element: CodeElement, file: str) -> List[Error]: return [Error("sec_non_official_image", element, file, repr(element))] return [] + class OrchestratorNonOfficialImageSmell(SmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + image = "" + if isinstance(element, KeyValue) and element.name == "image": + if isinstance(element.value, String): + image = element.value.value + elif ( + isinstance(element, KeyValue) + and element.name == "config" + and isinstance(element.value, Hash) + ): + for k, v in element.value.value.items(): + if isinstance(k, String) and k.value == "image": + image = v.value + break + + img_name, _, _ = SecurityVisitor.image_parser(image) + + if img_name != "": + for off_img in SecurityVisitor.DOCKER_OFFICIAL_IMAGES: + if img_name.startswith(off_img): + return [] + + return [Error("sec_non_official_image", element, file, repr(element))] + + return [] + + class ImageTagsSmell(SmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + return [] + + class OrchestratorImageTagsSmell(SmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + errors: List[Error] = [] + + if isinstance(element, KeyValue) and ( + (element.name == "image" and isinstance(element.value, String)) + or (isinstance(element.value, Hash) and element.name == "config") + ): + image = "" + + if isinstance(element.value, String): + image = element.value.value + else: + for k, v in element.value.value.items(): + if isinstance(k, String) and k.value == "image": + image = v.value + break + + has_digest, has_tag = False, False + _, tag, digest = SecurityVisitor.image_parser(image) + + if tag != "": + has_tag = True + if digest != "": + has_digest = True + + if image != "" and has_digest: # image tagged with digest + checksum_s = digest.split(":") + checksum = checksum_s[-1] + if ( + checksum_s[0] == "sha256" and len(checksum) != 64 + ): # sha256 256 digest -> 64 hexadecimal digits + errors.append( + Error("sec_image_integrity", element, file, repr(element)) + ) + + elif image != "" and has_tag: + errors.append( + Error("sec_image_integrity", element, file, repr(element)) + ) + dangerous_tags: List[str] = SecurityVisitor.DANGEROUS_IMAGE_TAGS + + for dt in dangerous_tags: + if dt in tag: + errors.append( + Error("sec_unstable_tag", element, file, repr(element)) + ) + break + elif ( + image != "" and not has_digest and not has_tag + ): # Image not tagged, avoids mistakenely nomad tasks without images (non-docker or non-podman) + errors.append( + Error("sec_no_image_tag", element, file, repr(element)) + ) + + return errors + + class LogAggregatorAbsenceSmell(SmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + return [] + + class OrchestratorLogAggregatorAbsenceSmell(SmellChecker): + # By default Docker uses the + def check(self, element: CodeElement, file: str) -> List[Error]: + errors: List[Error] = [] + if isinstance(element, UnitBlock): + # HACK: Besides the ones we are explicitly stating a registry we assume the default, normally Docker hub + log_collectors: List[str] = ( + SecurityVisitor.LOG_AGGREGATORS_AND_COLLECTORS + ) + + log_drivers: List[str] = SecurityVisitor.DOCKER_LOG_DRIVERS + + has_log_collector = False + + for au in element.atomic_units: + if au.type != "service" and not au.type.startswith("task."): + # Don't analyze Unit Blocks which aren't tasks or services + return [] + + for att in au.attributes: + image = "" + if att.name == "config" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if isinstance(k, String) and k.value == "image": + image = v.value + break + + elif att.name == "image" and isinstance(att.value, String): + image = att.value.value + + img_name, _, _ = SecurityVisitor.image_parser(image) + + if image != "": + for lc in log_collectors: + if img_name.startswith(lc): + return [] + break + + # if it doesn't have a log collector/aggregator in the deployment + if not has_log_collector: + for au in element.atomic_units: + has_logging = False + for att in au.attributes: + if has_logging: + break + if att.name == "logging" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if ( + k.value == "driver" + and isinstance(v, String) + and v.value in log_drivers + ): + has_logging = True + break + elif att.name == "config" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if ( + isinstance(k, String) + and k.value == "logging" + and isinstance(v, Hash) + ): + for _k, _v in v.value.items(): + if ( + _k.value == "driver" + and isinstance(_v, String) + and _v.value in log_drivers + ): + has_logging = True + break + if has_logging: + break + + if not has_logging: + errors.append(Error("arc_no_logging", au, file, repr(au))) + + return errors + def __init__(self, tech: Tech) -> None: super().__init__(tech) @@ -46,8 +238,14 @@ def __init__(self, tech: Tech) -> None: for child in TerraformSmellChecker.__subclasses__(): self.checkers.append(child()) + self.image_tag_smells = SecurityVisitor.ImageTagsSmell() + self.log_agg_smell = SecurityVisitor.LogAggregatorAbsenceSmell() if tech == Tech.docker: self.non_off_img = SecurityVisitor.DockerNonOfficialImageSmell() + elif tech in [Tech.swarm, Tech.nomad]: + self.non_off_img = SecurityVisitor.OrchestratorNonOfficialImageSmell() + self.image_tag_smells = SecurityVisitor.OrchestratorImageTagsSmell() + self.log_agg_smell = SecurityVisitor.OrchestratorLogAggregatorAbsenceSmell() else: self.non_off_img = SecurityVisitor.NonOfficialImageSmell() @@ -173,6 +371,27 @@ def config(self, config_path: str) -> None: "official_docker_images" ) + if self.tech in [Tech.swarm, Tech.nomad]: + SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES = self._load_data_file( + "deprecated_official_docker_images" + ) + SecurityVisitor.LOG_AGGREGATORS_AND_COLLECTORS = self._load_data_file( + "log_collectors_and_aggregators" + ) + SecurityVisitor.DANGEROUS_IMAGE_TAGS = self._load_data_file( + "dangerous_image_tags" + ) + SecurityVisitor.DOCKER_LOG_DRIVERS = self._load_data_file( + "docker_log_drivers" + ) + SecurityVisitor.API_GATEWAYS = self._load_data_file("api_gateways") + SecurityVisitor.DATABASES_AND_KVS = self._load_data_file( + "databases_and_kvs" + ) + SecurityVisitor.MESSAGE_QUEUES_AND_EVENT_BROKERS = self._load_data_file( + "message_queues_and_event_brokers" + ) + @staticmethod def _load_data_file(file: str) -> List[str]: folder_path = os.path.dirname(os.path.realpath(glitch.__file__)) @@ -232,6 +451,27 @@ def check_dependency(self, d: Dependency, file: str) -> List[Error]: def __check_keyvalue(self, c: KeyValue, file: str) -> List[Error]: errors: List[Error] = [] + + # official docker image for swarm and nomad + + errors += self.non_off_img.check(c, file) + # image tags smells for swarm and nomad + errors += self.image_tag_smells.check(c, file) + + # check https/tls/ssl in Hash values + if isinstance(c.value, Hash): + pairs_to_check = [c.value.value] + + while pairs_to_check: + for k, v in pairs_to_check[0].items(): + if isinstance(v, String) and self.__is_http_url(v.value): + errors.append(Error("sec_https", v, file, repr(v))) + if isinstance(v, String) and self.__is_weak_crypt(v.value, k.value): + errors.append(Error("sec_weak_crypt", v, file, repr(v))) + if isinstance(v, Hash): + pairs_to_check.append(v.value) + pairs_to_check.pop(0) + c.name = c.name.strip().lower() # if isinstance(c.value, type(None)): @@ -393,6 +633,11 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: errors += missing_integrity_checks.values() errors += self.non_off_img.check(u, file) + errors += self.log_agg_smell.check(u, file) + + for checker in self.checkers: + checker.code = self.code + errors += checker.check(u, file) return errors @@ -429,6 +674,7 @@ def check_integrity_check(au: AtomicUnit, path: str) -> Optional[Tuple[str, Erro return os.path.basename(a.value), Error( # type: ignore "sec_no_int_check", au, path, repr(a) ) # type: ignore + return None @staticmethod From 253dc45001f0eccd2fae6e019646792d9c713b5c Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:20:56 +0100 Subject: [PATCH 20/65] add deprecated official images smell --- .../security/deprecated_official_images.py | 35 ++++++++++++++++++ .../files/deprecated_official_docker_images | 36 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 glitch/analysis/security/deprecated_official_images.py create mode 100644 glitch/files/deprecated_official_docker_images diff --git a/glitch/analysis/security/deprecated_official_images.py b/glitch/analysis/security/deprecated_official_images.py new file mode 100644 index 00000000..c98d4140 --- /dev/null +++ b/glitch/analysis/security/deprecated_official_images.py @@ -0,0 +1,35 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import CodeElement, KeyValue, Hash, String +from typing import List + + +class DeprecatedOfficialDockerImages(SecuritySmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + errors: List[Error] = [] + image = "" + bad_element = element + if isinstance(element, KeyValue) and element.name == "image": + if isinstance(element.value, String): + image = element.value.value + elif ( + isinstance(element, KeyValue) + and element.name == "config" + and isinstance(element.value, Hash) + ): + for k, v in element.value.value.items(): + if isinstance(k, String) and k.value == "image": + image = v.value + bad_element = v + break + if image != "": + img_name, _, _ = SecurityVisitor.image_parser(image) + for obsolete_img in SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES: + if img_name == obsolete_img: + errors.append( + Error("sec_depr_off_imgs", bad_element, file, repr(bad_element)) + ) + break + + return errors diff --git a/glitch/files/deprecated_official_docker_images b/glitch/files/deprecated_official_docker_images new file mode 100644 index 00000000..f0dd4413 --- /dev/null +++ b/glitch/files/deprecated_official_docker_images @@ -0,0 +1,36 @@ +centos +jenkins +java +sentry +nats-streaming +adoptopenjdk +swarm +owncloud +mono +django +piwik +iojs +opensuse +rails +ubuntu-debootstrap +clearlinux +nuxeo +php-zendserver +emqx +celery +thrift +fsharp +rapidoid +docker-dev +express-gateway +kaazing-gateway +ubuntu-upstart +known +glassfish +crux +clefos +euleros +jobber +sourcemage +sl +hipache \ No newline at end of file From e178edc83010ca9e04dab179518140296dd2b18d Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:21:39 +0100 Subject: [PATCH 21/65] added files needed for log collector/log aggregator smell --- glitch/files/docker_log_drivers | 10 ++++ glitch/files/log_collectors_and_aggregators | 52 +++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 glitch/files/docker_log_drivers create mode 100644 glitch/files/log_collectors_and_aggregators diff --git a/glitch/files/docker_log_drivers b/glitch/files/docker_log_drivers new file mode 100644 index 00000000..3f1e0070 --- /dev/null +++ b/glitch/files/docker_log_drivers @@ -0,0 +1,10 @@ +elastic/elastic-logging-plugin +logzio/logzio-logging-plugin +grafana/loki-docker-driver +sumologic/docker-logging-driver +syslog +gelf +fluentd +awslogs +splunk +gcplogs diff --git a/glitch/files/log_collectors_and_aggregators b/glitch/files/log_collectors_and_aggregators new file mode 100644 index 00000000..0f736106 --- /dev/null +++ b/glitch/files/log_collectors_and_aggregators @@ -0,0 +1,52 @@ +amazon/aws-for-fluent-bit +amazon/aws-otel-collector +amazon/cloudwatch-agent +balabit/syslog-ng +bitnami/fluent-bit +bitnami/fluentd +bitnami/grafana-alloy +bitnami/grafana-loki +bitnami/logstash +bitnami/promtail +bitnami/telegraf +chainguard/opentelemetry-collector-contrib +datadoghq/agent +datalust/seq +docker.elastic.co/beats/filebeat +docker.elastic.co/beats/filebeat-wolfi +dynatrace/oneagent +elastic/filebeat +fluent/fluent-bit +fluent/fluentd +fluentd +ghcr.io/mr-karan/nomad-vector-logger +gliderlabs/logspout +grafana/agent +grafana/alloy +grafana/loki +grafana/promtail +graylog/graylog +graylog/graylog-forwarder +linuxserver/syslog-ng +logstash +logzio/docker-collector-logs +netdata/netdata +otel/opentelemetry-collector-contrib +outcoldsolutions/collectorforopenshift +public.ecr.aws/aws-observability/aws-for-fluent-bit +public.ecr.aws/zinclabs/openobserve +sematext/agent +signoz/signoz-otel-collector +sofixa/nomad_follower +splunk/splunk +sumologic/collector +sumologic/sumologic-otel-collector +supabase/logflare +telegraf +timberio/vector +ubuntu/grafana-agent +ubuntu/telegraf +umputun/docker-logger +voxxit/rsyslog +victoriametrics/victoria-logs +zabbix/zabbix-agent From 7ad6a8da68b8f67875b8383bcbdacd5e5545a586 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:22:32 +0100 Subject: [PATCH 22/65] currently useless file, will remove later --- glitch/files/databases_and_kvs | 70 ++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 glitch/files/databases_and_kvs diff --git a/glitch/files/databases_and_kvs b/glitch/files/databases_and_kvs new file mode 100644 index 00000000..edae8b61 --- /dev/null +++ b/glitch/files/databases_and_kvs @@ -0,0 +1,70 @@ +redis +mysql +mongo +chainguard/mongodb +mongodb/mongodb-community-server +mongodb/mongodb-enterprise-server +bitnami/mongodb +bitnami/postgresql +bitnami/redis +bitnami/mariadb +bitnami/mysql +bitnami/memcached +bitnami/etcd +bitnami/cassandra +bitnami/neo4j +bitnami/influxdb +bitnami/clickhouse +bitnami/valkey +bitnami/couchdb +bitnami/milvus +bitnami/keydb +eqalpha/keydb +bitnami/scylladb +bitnami/percona-mysql +bitnami/cloudnative-pg +bitnami/arangodb +chainguard/mariadb +chainguard/valkey +chainguard/postgres +chainguard/redis +postgres +ubuntu/postgres +valkey/valkey +valkey +apache/kvrocks +memcached +ubuntu/memcached +mariadb +linuxserver/mariadb +influxdb +cassandra +scylladb/scylla +couchdb +apache/couchdb +couchbase +percona +percona/percona-server +rethinkdb +arangodb +arangodb/arangodb +arangodb/enterprise +crate +crate/crate +orientdb +clickhouse +clickhouse/clickhouse-server +cockroachdb/cockroach +neo4j +milvusdb/milvus +semitechnologies/weaviate +ravendb/ravendb +pgvector/pgvector +tensorchord/pgvecto-rs +tensorchord/vchord-postgres +qdrant/qdrant +chromadb/chroma +pingcap/tidb +pingcap/tikv +mcr.microsoft.com/mssql/server +ferretdb/ferretdb From d7177d45bfd07566d88abc618509ed77bdfef5f4 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:22:49 +0100 Subject: [PATCH 23/65] currently useless file, will remove later --- glitch/files/message_queues_and_event_brokers | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 glitch/files/message_queues_and_event_brokers diff --git a/glitch/files/message_queues_and_event_brokers b/glitch/files/message_queues_and_event_brokers new file mode 100644 index 00000000..349c03b4 --- /dev/null +++ b/glitch/files/message_queues_and_event_brokers @@ -0,0 +1,18 @@ +rabbitmq +bitnami/rabbitmq +nats +bitnami/nats +bitnami/kafka +ubuntu/kafka +apache/kafka +apachepulsar/pulsar +apachepulsar/pulsar-all +apache/activemq-classic +apache/activemq-artemis +redpandadata/redpanda +redis +bitnami/redis +chainguard/redis +valkey/valkey +bitnami/valkey +chainguard/valkey From 1e19b3d9f6d1d2460ec39a476bbb2453f447c433 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:23:22 +0100 Subject: [PATCH 24/65] added file needed for the unstable image smell --- glitch/files/dangerous_image_tags | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 glitch/files/dangerous_image_tags diff --git a/glitch/files/dangerous_image_tags b/glitch/files/dangerous_image_tags new file mode 100644 index 00000000..e9723d34 --- /dev/null +++ b/glitch/files/dangerous_image_tags @@ -0,0 +1,20 @@ +stable +unstable +nightly +main +mainline +master +trunk +latest +beta +alpha +current +dev +devel +develop +development +next +staging +test +testing +preview From 3a49007dc15b9466e4ac0ad47c7a2841747dc150 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:24:32 +0100 Subject: [PATCH 25/65] adding no api gateway smell files --- glitch/analysis/security/no_api_gateway.py | 127 +++++++++++++++++++++ glitch/files/api_gateways | 29 +++++ 2 files changed, 156 insertions(+) create mode 100644 glitch/analysis/security/no_api_gateway.py create mode 100644 glitch/files/api_gateways diff --git a/glitch/analysis/security/no_api_gateway.py b/glitch/analysis/security/no_api_gateway.py new file mode 100644 index 00000000..bc04437f --- /dev/null +++ b/glitch/analysis/security/no_api_gateway.py @@ -0,0 +1,127 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import ( + CodeElement, + Hash, + Array, + VariableReference, + String, + UnitBlock, + UnitBlockType, +) +from typing import List, Dict, Any + + +class NoAPIGatewayCheck(SecuritySmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + errors: List[Error] = [] + # Tries to follow an logic similar to the one presented for Kubernetes pods ond doi: 10.5220/0011845500003488 + + if isinstance(element, UnitBlock) and element.type == UnitBlockType.block: + has_api_gateway = False + + network_info: Dict[str, Any] = { + "mode": "bridge", # default network mode + "ports": [], + } + + network_mode_element = None + for att in element.attributes: + if att.name == "network" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if isinstance(k, String) and k.value == "mode": + if v.value == "host": + network_mode_element = v + network_info["mode"] = v.value + elif ( + isinstance(k, String) or isinstance(k, VariableReference) + ) and k.value == "port": + port_info: Dict[str, Any] = { + "name": "", + } + for _k, _v in v.value.items(): + if isinstance(_k, String) and _k.value == "port": + port_info["name"] = _v.value + elif isinstance(_k, String) and _k.value in [ + "static", + "to", + ]: + port_info[_k.value] = _v.value + network_info["ports"].append(port_info) + + for au in element.atomic_units: + for att in au.attributes: + if att.name == "config" and isinstance(att.value, Hash): + temp_errors: List[Error] = [] + is_api_gateway = False + + if ( + isinstance(au.name, String) + and "gateway" in au.name.value.strip().lower() + ): + is_api_gateway = True + has_api_gateway = True + + for k, v in att.value.value.items(): + if ( + isinstance(k, String) + and k.value == "ports" + and isinstance(v, Array) + and not is_api_gateway + ): + for port in v.value: + if isinstance(port, String): + for exp_port in network_info["ports"]: + if exp_port["name"] == port.value: + if network_info["mode"] == "host": + temp_errors.append( + Error( + "arc_no_apig", + port, + file, + repr(port), + ) + ) + + elif network_info[ + "mode" + ] == "bridge" and ( + "to" in exp_port.keys() + or "static" in exp_port.keys() + ): + temp_errors.append( + Error( + "arc_no_apig", + port, + file, + repr(port), + ) + ) + + if isinstance(k, String) and k.value == "image": + image_name, _, _ = SecurityVisitor.image_parser(v.value) + if ( + image_name in SecurityVisitor.API_GATEWAYS + or is_api_gateway + ): + is_api_gateway = True + has_api_gateway = True + else: + errors += temp_errors + temp_errors = [] + + if not is_api_gateway: + errors += temp_errors + + if not has_api_gateway and network_info["mode"] == "host": + errors.append( + Error( + "arc_no_apig", + network_mode_element, + file, + repr(network_mode_element), + ) + ) + + return errors diff --git a/glitch/files/api_gateways b/glitch/files/api_gateways new file mode 100644 index 00000000..9683f5c5 --- /dev/null +++ b/glitch/files/api_gateways @@ -0,0 +1,29 @@ +apache/apisix +bitnami/apisix +kong +kong/kong-gateway +tykio/tyk-gateway +docker.tyk.io/tyk-gateway/tyk-gateway +apache/shenyu-bootstrap +apache/shenyu-admin +krakend +hashicorp/consul +envoyproxy/envoy +bitnami/envoy +express-gateway +nrel/api-umbrella +fusio/fusio +bitnami/nginx +nginx +traefik +linuxserver/nginx +chainguard/nginx +nginxinc/nginx-unprivileged +ubuntu/nginx +kasmweb/nginx +predic8/membrane +traefik/traefik +ubuntu/traefik +kong/kong +openresty/openresty +openwhisk/apigateway From 94712dc9e9612571ba08e4874628ff8777f3b3b6 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:25:46 +0100 Subject: [PATCH 26/65] adding priveleged containers, mounted docker socket and Nomad integrity check smells --- .../security/docker_socket_mounted.py | 46 +++++++++++++++++++ .../security/nomad_integrity_check.py | 37 +++++++++++++++ .../security/privileged_containers.py | 22 +++++++++ 3 files changed, 105 insertions(+) create mode 100644 glitch/analysis/security/docker_socket_mounted.py create mode 100644 glitch/analysis/security/nomad_integrity_check.py create mode 100644 glitch/analysis/security/privileged_containers.py diff --git a/glitch/analysis/security/docker_socket_mounted.py b/glitch/analysis/security/docker_socket_mounted.py new file mode 100644 index 00000000..d65e5b8f --- /dev/null +++ b/glitch/analysis/security/docker_socket_mounted.py @@ -0,0 +1,46 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.repr.inter import CodeElement, KeyValue, Hash, String,Array +from typing import List + + +class DockerSocketMountedInsideContainerUse(SecuritySmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + errors: List[Error] = [] + if isinstance(element, KeyValue): + if element.name == "volumes" and isinstance(element.value, Array): + for volume in element.value.value: + if isinstance(volume, String) and volume.value.split(":")[ + 0 + ].startswith("/var/run/docker.sock"): + errors.append( + Error( + "sec_mounted_docker_socket", volume, file, repr(volume) + ) + ) + break + elif element.name == "config" and isinstance(element.value, Hash): + found_socket_exposed = False + for k, v in element.value.value.items(): + if ( + isinstance(k, String) + and k.value == "volumes" + and isinstance(v, Array) + ): + for volume in v.value: + if isinstance(volume, String) and volume.value.split(":")[ + 0 + ].startswith("/var/run/docker.sock"): + errors.append( + Error( + "sec_mounted_docker_socket", + volume, + file, + repr(volume), + ) + ) + found_socket_exposed = True + break + if found_socket_exposed: + break + return errors diff --git a/glitch/analysis/security/nomad_integrity_check.py b/glitch/analysis/security/nomad_integrity_check.py new file mode 100644 index 00000000..7df5123a --- /dev/null +++ b/glitch/analysis/security/nomad_integrity_check.py @@ -0,0 +1,37 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.repr.inter import CodeElement, KeyValue, Hash, String +from typing import List + + +class NomadNoIntegrityCheck(SecuritySmellChecker): + # FIXME Nomad integrity check (its split from the other integrity checks) + def check(self, element: CodeElement, file: str) -> List[Error]: + if isinstance(element, KeyValue): + if element.name == "artifact" and isinstance(element.value, Hash): + # Nomad integrity check + found_checksum = False + for k, v in element.value.value.items(): + if ( + isinstance(k, String) + and k.value == "options" + and isinstance(v, Hash) + ): + for _k, _ in v.value.items(): + if isinstance(_k, String) and _k.value == "checksum": + found_checksum = True + break + if not found_checksum: + return [ + Error( # type: ignore + "sec_no_int_check", element, file, repr(element) + ) + ] # type: ignore + + if not found_checksum: + return [ + Error( # type: ignore + "sec_no_int_check", element, file, repr(element) + ) + ] # type: ignore + return [] diff --git a/glitch/analysis/security/privileged_containers.py b/glitch/analysis/security/privileged_containers.py new file mode 100644 index 00000000..3d47f746 --- /dev/null +++ b/glitch/analysis/security/privileged_containers.py @@ -0,0 +1,22 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.repr.inter import CodeElement, KeyValue, Boolean, Hash, String +from typing import List + + +class PrivigedContainerUse(SecuritySmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + if isinstance(element, KeyValue): + + if ( + element.name == "privileged" + and isinstance(element.value, Boolean) + and element.value.value + ): + return [Error("sec_privileged_containers", element, file, repr(element))] + elif element.name == "config" and isinstance(element.value, Hash): + for k, v in element.value.value.items(): + if isinstance(k, String) and k.value == "privileged" and v.value: + return [Error("sec_privileged_containers", k, file, repr(k))] + + return [] From 8a241538792ed5324d524f6f77c79c50e0c22a60 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:26:59 +0100 Subject: [PATCH 27/65] adding multiple services per deployment unit and missing healthcheck smells --- .../multiple_services_per_deplyment_unit.py | 43 +++++++++ .../security/wobbly_service_interaction.py | 88 +++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 glitch/analysis/security/multiple_services_per_deplyment_unit.py create mode 100644 glitch/analysis/security/wobbly_service_interaction.py diff --git a/glitch/analysis/security/multiple_services_per_deplyment_unit.py b/glitch/analysis/security/multiple_services_per_deplyment_unit.py new file mode 100644 index 00000000..fc5f16d6 --- /dev/null +++ b/glitch/analysis/security/multiple_services_per_deplyment_unit.py @@ -0,0 +1,43 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import CodeElement, Hash, String, UnitBlock, UnitBlockType +from typing import List + + +class MultipleServicesPerDeploymentUnitCheck(SecuritySmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + # FIXME: Besides log collectors, there are other types of agents/sidecars for observability (some of which are also on the log collector list) + # and proxies which should also be allowed besides the main microservice + errors: List[Error] = [] + if isinstance(element, UnitBlock) and element.type == UnitBlockType.block: + main_service_found = False + for au in element.atomic_units: + if au.type in ["task.docker", "task.podman"]: + image_name = "" + for att in au.attributes: + if att.name == "config" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if isinstance(k, String) and k.value == "image": + image_name, _, _ = SecurityVisitor.image_parser( + v.value + ) + break + if image_name != "": + break + + if image_name in SecurityVisitor.LOG_AGGREGATORS_AND_COLLECTORS: + continue + + elif main_service_found: + errors.append( + Error("arc_multiple_services", au, file, repr(au)) + ) + else: + main_service_found = True + elif main_service_found: + # when there are other types of tasks that aren't docker or podman based + # and one that is likely the main microservice has already been found + errors.append(Error("arc_multiple_services", au, file, repr(au))) + + return errors diff --git a/glitch/analysis/security/wobbly_service_interaction.py b/glitch/analysis/security/wobbly_service_interaction.py new file mode 100644 index 00000000..99a71401 --- /dev/null +++ b/glitch/analysis/security/wobbly_service_interaction.py @@ -0,0 +1,88 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.repr.inter import CodeElement, Hash, AtomicUnit, Array, Boolean +from typing import List + + +class WobblyServiceInteractionCheck(SecuritySmellChecker): + #FIXME: The class is currently wrongly named + def check(self, element: CodeElement, file: str) -> List[Error]: + errors: List[Error] = [] + + if isinstance(element, AtomicUnit): + au = element + found_healthcheck = False + has_disable_nomad = False + + for att in au.attributes: + if found_healthcheck: + break + if att.name == "healthcheck": + found_healthcheck = True + if isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if k.value == "disable" and ( + v.value or v.value.lower() == "true" + ): + errors.append(Error("arc_wobbly", au, file, repr(au))) + break + elif k.value == "test": + if isinstance(v.value, Array): + if ( + len(v.value.value) >= 1 + and v.value.value[0] == "NONE" + ): + errors.append( + Error("arc_wobbly", au, file, repr(au)) + ) + break + break + elif att.name == "config" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if k.value == "healthchecks" and isinstance(v, Hash): + for _k, _v in v.value.items(): + if ( + _k.value == "disable" + and isinstance(_v, Boolean) + and _v.value + ): + has_disable_nomad = True + break + elif has_disable_nomad: + break + if has_disable_nomad: + break + + elif att.name == "service" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if k.value == "check" and isinstance(v, Hash): + found_healthcheck = True + break + elif k.value == "connect" and isinstance(v, Hash): + for _k, _v in v.value.items(): + if _k.value == "sidecar_service" and isinstance( + _v, Hash + ): + # Checks for use of Consul service mesh, sidecar proxy that + # provides timeouts and circuit breaks that also avoid this smell + # technically the original smell is detectable in Nomad if this is not present + found_healthcheck = True + break + if found_healthcheck: + break + + if not found_healthcheck: + errors.append(Error("arc_wobbly", au, file, repr(au))) + + if ( + att.name in ["config", "service"] + and isinstance(att.value, Hash) + and not found_healthcheck + and has_disable_nomad + ): + errors.append(Error("arc_wobbly", au, file, repr(au))) + + if element.type == "service" and not found_healthcheck: + errors.append(Error("arc_wobbly",au,file,repr(au))) + + return errors From f8042679116136ca995afd14b660e717289a4b5a Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sun, 24 Aug 2025 00:59:28 +0100 Subject: [PATCH 28/65] fix wrong version of non official image smell detection --- glitch/analysis/security/visitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index 530dc47f..e393fd3e 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -78,7 +78,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if img_name != "": for off_img in SecurityVisitor.DOCKER_OFFICIAL_IMAGES: - if img_name.startswith(off_img): + if img_name == off_img: return [] return [Error("sec_non_official_image", element, file, repr(element))] From 757d8c8b15a3f0340bf19a2fc93805110818b8ca Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sun, 24 Aug 2025 17:16:04 +0100 Subject: [PATCH 29/65] fix alternative ways of referring to an official image from Docker Hub --- .../analysis/security/deprecated_official_images.py | 13 ++++++++++++- glitch/analysis/security/visitor.py | 11 ++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/glitch/analysis/security/deprecated_official_images.py b/glitch/analysis/security/deprecated_official_images.py index c98d4140..77ec4fd3 100644 --- a/glitch/analysis/security/deprecated_official_images.py +++ b/glitch/analysis/security/deprecated_official_images.py @@ -26,7 +26,18 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if image != "": img_name, _, _ = SecurityVisitor.image_parser(image) for obsolete_img in SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES: - if img_name == obsolete_img: + obsolete_img_dockerio = f"docker.io/library/{obsolete_img}" + obsolete_img_library = f"library/{obsolete_img}" + obsolete_img_complete_link = ( + f"registry.hub.docker.com/library/{obsolete_img}" + ) + + if ( + img_name == obsolete_img + or img_name == obsolete_img_dockerio + or img_name == obsolete_img_library + or img_name == obsolete_img_complete_link + ): errors.append( Error("sec_depr_off_imgs", bad_element, file, repr(bad_element)) ) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index e393fd3e..a4a21b1b 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -78,7 +78,16 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if img_name != "": for off_img in SecurityVisitor.DOCKER_OFFICIAL_IMAGES: - if img_name == off_img: + off_img_dockerio = f"docker.io/library/{off_img}" + off_img_library = f"library/{off_img}" + off_img_complete_link = f"registry.hub.docker.com/library/{off_img}" + + if ( + img_name == off_img + or img_name == off_img_dockerio + or img_name == off_img_library + or img_name == off_img_complete_link + ): return [] return [Error("sec_non_official_image", element, file, repr(element))] From f8765a3a07bb6a1e30f511b641933a6e2cd5c8f0 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sun, 24 Aug 2025 17:43:04 +0100 Subject: [PATCH 30/65] fix logic error and always make tag lower, since it can be mixed or complete uppercase --- glitch/analysis/security/visitor.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index a4a21b1b..699a201b 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -134,10 +134,13 @@ def check(self, element: CodeElement, file: str) -> List[Error]: Error("sec_image_integrity", element, file, repr(element)) ) - elif image != "" and has_tag: - errors.append( - Error("sec_image_integrity", element, file, repr(element)) - ) + if image != "" and has_tag: + tag = tag.lower() + if not has_digest: + errors.append( + Error("sec_image_integrity", element, file, repr(element)) + ) + dangerous_tags: List[str] = SecurityVisitor.DANGEROUS_IMAGE_TAGS for dt in dangerous_tags: @@ -146,7 +149,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: Error("sec_unstable_tag", element, file, repr(element)) ) break - elif ( + if ( image != "" and not has_digest and not has_tag ): # Image not tagged, avoids mistakenely nomad tasks without images (non-docker or non-podman) errors.append( From e9671bd9269ceca6778d270757d816c421b04ad9 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sun, 24 Aug 2025 17:53:53 +0100 Subject: [PATCH 31/65] fix small parser mistake --- glitch/parsers/swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index d651cd87..095eef0b 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -203,7 +203,7 @@ def parse_file( file_unit_block.path = os.path.abspath(path) if isinstance(parsed_file.value, list): for field in parsed_file.value: - if field[0].value == "version": + if field[0].value == "version" or field[0].value == "name": expr: Expr = self.get_value(field[1], code) info: ElementInfo = ElementInfo( field[0].start_mark.line + 1, From 85ee60aa390beb713e5666c8341c26c629cc1445 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sun, 24 Aug 2025 19:05:01 +0100 Subject: [PATCH 32/65] try to fix self include when extending from another file --- glitch/parsers/swarm.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 095eef0b..807ef12b 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -326,15 +326,31 @@ def parse_file( service_from = v.value curr_path = os.path.split(path)[0] joint_path = os.path.normpath(os.path.join(curr_path, file)) - if os.path.exists(joint_path): - service_from_file_unit_block = self.parse_file(joint_path) - service_from_list = ( - service_from_file_unit_block.atomic_units - ) + if os.path.normpath(path) != joint_path: + if os.path.exists(joint_path): + service_from_file_unit_block = self.parse_file(joint_path) + if service_from_file_unit_block is not None: + for u_block in service_from_file_unit_block.unit_blocks: + if u_block.type == UnitBlockType.block and u_block.name == "services": + service_from_list += u_block.atomic_units + break + + else: + print( + f'Failed to parse extends file expected at "{joint_path}". File not found.' + ) + else: + print( + f'Failed to parse extends file expected at "{joint_path}". File not found.' + ) else: + # Avoid infinite recursion from same file + #service_from_list = services + # something else missing print( - f'Failed to parse extends file expected at "{joint_path}". File not found.' + f'Failed to parse extends file expected at "{joint_path}". File not found.' ) + # for s in service_from_list: if s.name.value == service_from: From 3f57889d084273bf046b4cef8e55a51b252116af Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sun, 24 Aug 2025 22:05:01 +0100 Subject: [PATCH 33/65] format + extends should be fixed now... --- glitch/parsers/swarm.py | 199 +++++++++++++++++++++------------------- 1 file changed, 103 insertions(+), 96 deletions(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 807ef12b..1a3561bd 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -137,7 +137,7 @@ def create_attribute( for elem in att_value.value: elem_info: ElementInfo = ElementInfo.from_code_element(elem) curr_str = elem.value - split_str = curr_str.split("=",1) + split_str = curr_str.split("=", 1) if len(split_str) == 2: key, n_val = split_str val_s = String(n_val, elem_info) @@ -177,7 +177,10 @@ def create_attribute( return attributes def parse_file( - self, path: str, type: UnitBlockType = UnitBlockType.script + self, + path: str, + type: UnitBlockType = UnitBlockType.script, + extends: bool = True, ) -> Optional[UnitBlock]: """ Parses a stack/compose file into a UnitBlock each with its respective @@ -269,7 +272,7 @@ def parse_file( for inc in includes: curr_path = os.path.split(path)[0] joint_path = os.path.normpath(os.path.join(curr_path, inc)) - if os.path.exists(joint_path): + if os.path.exists(joint_path) and os.path.isfile(joint_path): include_file_unit_block = self.parse_file(joint_path) for ub in include_file_unit_block.unit_blocks: @@ -289,93 +292,101 @@ def parse_file( if ub.name == "services": services = ub.atomic_units # FIXME: Handling the extends from the same file or from other files, might not be the best way - for service in services: - for attribute in service.attributes: - if attribute.name == "extends": - deps = [] - # if isinstance(attribute.value,String): - # deps.append(attribute.value.value) - if isinstance(attribute.value, Hash): - # adds the name of file as a dependency - for k, v in attribute.value.value.items(): - if k.value == "file": - deps.append(v.value) - break - - file_unit_block.add_dependency(Dependency(deps)) - to_extend.append([service, attribute]) - break - - for service_to, attribute in to_extend: - att: Attribute = attribute - service_from_list = [] - service_from = "" - - if isinstance(att.value, String): - service_from = att.value.value - service_from_list = services - - elif isinstance(att.value, Hash): - hash_dict = att.value.value - file = "" - service_from = "" - for k, v in hash_dict.items(): - if k.value == "file": - file = v.value - elif k.value == "service": - service_from = v.value - curr_path = os.path.split(path)[0] - joint_path = os.path.normpath(os.path.join(curr_path, file)) - if os.path.normpath(path) != joint_path: - if os.path.exists(joint_path): - service_from_file_unit_block = self.parse_file(joint_path) - if service_from_file_unit_block is not None: - for u_block in service_from_file_unit_block.unit_blocks: - if u_block.type == UnitBlockType.block and u_block.name == "services": - service_from_list += u_block.atomic_units + if extends: + for service in services: + for attribute in service.attributes: + if attribute.name == "extends": + deps = [] + # if isinstance(attribute.value,String): + # deps.append(attribute.value.value) + if isinstance(attribute.value, Hash): + # adds the name of file as a dependency + for k, v in attribute.value.value.items(): + if k.value == "file": + deps.append(v.value) break - + file_unit_block.add_dependency(Dependency(deps)) + to_extend.append([service, attribute]) + break + + for service_to, attribute in to_extend: + att: Attribute = attribute + service_from_list = [] + service_from = "" + + if isinstance(att.value, String): + service_from = att.value.value + service_from_list = services + + elif isinstance(att.value, Hash): + hash_dict = att.value.value + file = "" + service_from = "" + for k, v in hash_dict.items(): + if k.value == "file": + file = v.value + elif k.value == "service": + service_from = v.value + curr_path = os.path.split(path)[0] + joint_path = os.path.normpath(os.path.join(curr_path, file)) + if os.path.normpath(path) != joint_path: + if os.path.exists(joint_path): + service_from_file_unit_block = self.parse_file( + joint_path, extends=False + ) + if service_from_file_unit_block is not None: + for ( + u_block + ) in service_from_file_unit_block.unit_blocks: + if ( + u_block.type == UnitBlockType.block + and u_block.name == "services" + ): + service_from_list += ( + u_block.atomic_units + ) + break + + else: + print( + f'Failed to parse extends file expected at "{joint_path}". File not found.' + ) else: print( - f'Failed to parse extends file expected at "{joint_path}". File not found.' - ) + f'Failed to parse extends file expected at "{joint_path}". File not found.' + ) else: - print( - f'Failed to parse extends file expected at "{joint_path}". File not found.' - ) - else: - # Avoid infinite recursion from same file - #service_from_list = services - # something else missing - print( - f'Failed to parse extends file expected at "{joint_path}". File not found.' - ) - # - - for s in service_from_list: - if s.name.value == service_from: - att_names = [x.name for x in service_to.attributes] - - for s_att in s.attributes: - if s_att.name in ["depends_on", "volumes_from"]: - continue - elif s_att.name not in att_names: - service_to.add_attribute(s_att) - elif s_att.name in att_names: - for to_att in service_to.attributes: - if to_att.name == s_att.name: - if isinstance(to_att.value, Array): - self.__handle_array( - s_att.value, to_att.value - ) - elif isinstance(to_att.value, Hash): - self.__handle_hash( - s_att.value, to_att.value - ) - else: - continue - break - break + service_from_list = services + + # print( + # f'Failed to parse extends file expected at "{joint_path}". File not found.' + # ) + # + + for s in service_from_list: + if s.name.value == service_from: + att_names = [x.name for x in service_to.attributes] + + for s_att in s.attributes: + if s_att.name in ["depends_on", "volumes_from"]: + continue + elif s_att.name not in att_names: + service_to.add_attribute(s_att) + elif s_att.name in att_names: + for to_att in service_to.attributes: + if to_att.name == s_att.name: + if isinstance(to_att.value, Array): + self.__handle_array( + s_att.value, to_att.value + ) + elif isinstance(to_att.value, Hash): + self.__handle_hash( + s_att.value, to_att.value + ) + else: + continue + break + break return file_unit_block except: @@ -405,13 +416,11 @@ def parse_folder(self, path: str, root: bool = True) -> Optional[Project]: different parts of the system are in each part subfolder we consider each subfolder a Module """ - + res: Project = Project(os.path.basename(os.path.normpath(path))) subfolders = [ - f.path - for f in os.scandir(f"{path}") - if f.is_dir() and not f.is_symlink() + f.path for f in os.scandir(f"{path}") if f.is_dir() and not f.is_symlink() ] for d in subfolders: @@ -420,23 +429,21 @@ def parse_folder(self, path: str, root: bool = True) -> Optional[Project]: files = [ f.path for f in os.scandir(f"{path}") - if f.is_file() - and not f.is_symlink() - and f.path.endswith((".yml", ".yaml")) + if f.is_file() and not f.is_symlink() and f.path.endswith((".yml", ".yaml")) ] for fi in files: res.add_block(self.parse_file(fi)) return res - + def parse_module(self, path) -> Module: """ - We consider each subfolder of the Project folder a Module + We consider each subfolder of the Project folder a Module as done for other languagues supported by GLITCH """ res: Module = Module(os.path.basename(os.path.normpath(path)), path) - super().parse_file_structure(res.folder,path) + super().parse_file_structure(res.folder, path) files = [ f.path From 27b9f6b1c55e42726e14cad339e5743d63e485c1 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 12:08:21 +0100 Subject: [PATCH 34/65] remove unused stuff --- glitch/analysis/security/visitor.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index 699a201b..4a90634a 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -397,12 +397,6 @@ def config(self, config_path: str) -> None: "docker_log_drivers" ) SecurityVisitor.API_GATEWAYS = self._load_data_file("api_gateways") - SecurityVisitor.DATABASES_AND_KVS = self._load_data_file( - "databases_and_kvs" - ) - SecurityVisitor.MESSAGE_QUEUES_AND_EVENT_BROKERS = self._load_data_file( - "message_queues_and_event_brokers" - ) @staticmethod def _load_data_file(file: str) -> List[str]: From 9a683b98aac6a9c17c92dfb4559810ce5c529169 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 13:57:43 +0100 Subject: [PATCH 35/65] fix uniformize use of container image string element instead of whole block (to get correct line numbers) --- glitch/analysis/security/visitor.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index 4a90634a..b11c2b86 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -61,6 +61,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: class OrchestratorNonOfficialImageSmell(SmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: image = "" + bad_element = element if isinstance(element, KeyValue) and element.name == "image": if isinstance(element.value, String): image = element.value.value @@ -72,6 +73,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: for k, v in element.value.value.items(): if isinstance(k, String) and k.value == "image": image = v.value + bad_element = v break img_name, _, _ = SecurityVisitor.image_parser(image) @@ -89,8 +91,8 @@ def check(self, element: CodeElement, file: str) -> List[Error]: or img_name == off_img_complete_link ): return [] - - return [Error("sec_non_official_image", element, file, repr(element))] + + return [Error("sec_non_official_image", bad_element, file, repr(bad_element))] return [] @@ -101,19 +103,20 @@ def check(self, element: CodeElement, file: str) -> List[Error]: class OrchestratorImageTagsSmell(SmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] - + bad_element = element if isinstance(element, KeyValue) and ( (element.name == "image" and isinstance(element.value, String)) or (isinstance(element.value, Hash) and element.name == "config") ): image = "" - + if isinstance(element.value, String): image = element.value.value else: for k, v in element.value.value.items(): if isinstance(k, String) and k.value == "image": image = v.value + bad_element = v break has_digest, has_tag = False, False @@ -131,14 +134,14 @@ def check(self, element: CodeElement, file: str) -> List[Error]: checksum_s[0] == "sha256" and len(checksum) != 64 ): # sha256 256 digest -> 64 hexadecimal digits errors.append( - Error("sec_image_integrity", element, file, repr(element)) + Error("sec_image_integrity", bad_element, file, repr(bad_element)) ) if image != "" and has_tag: tag = tag.lower() if not has_digest: errors.append( - Error("sec_image_integrity", element, file, repr(element)) + Error("sec_image_integrity", bad_element, file, repr(bad_element)) ) dangerous_tags: List[str] = SecurityVisitor.DANGEROUS_IMAGE_TAGS @@ -146,14 +149,14 @@ def check(self, element: CodeElement, file: str) -> List[Error]: for dt in dangerous_tags: if dt in tag: errors.append( - Error("sec_unstable_tag", element, file, repr(element)) + Error("sec_unstable_tag", bad_element, file, repr(bad_element)) ) break if ( image != "" and not has_digest and not has_tag ): # Image not tagged, avoids mistakenely nomad tasks without images (non-docker or non-podman) errors.append( - Error("sec_no_image_tag", element, file, repr(element)) + Error("sec_no_image_tag", bad_element, file, repr(bad_element)) ) return errors From 060f0e52bb8df4197eacef82e1522fa68879c48c Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 14:11:38 +0100 Subject: [PATCH 36/65] fix rename error key --- glitch/analysis/rules.py | 4 ++-- .../security/wobbly_service_interaction.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/glitch/analysis/rules.py b/glitch/analysis/rules.py index 8243f08b..277504b5 100644 --- a/glitch/analysis/rules.py +++ b/glitch/analysis/rules.py @@ -33,7 +33,7 @@ class Error: "sec_no_image_tag": "The image is not tagged - Prefer specifying a release version or better yet specific version digest", "arc_no_apig": "No API Gateway - If following a microservices architecture you should use in front of your services an API Gateway instead of directly exposing them.", "arc_no_logging": "Log Collection not found - It is advised to setup logging for your services.", - "arc_wobbly": "Missing Healthchecks - You should setup healthchecks for your services, or check if the images used already have default ones.", + "arc_missing_healthchecks": "Missing Healthchecks - You should setup healthchecks for your services, or check if the images used already have default ones.", "sec_mounted_docker_socket": "Docker socket mounted to a container - Avoid mounting the Docker socket to container, if the container is compromised its access to the Docker socket allows control of all other containers, and even acquiring control of the host machine.", "sec_privileged_containers": "Use of Privileged Containers - Developers should always try to give and use the least privileges possible. Use of privileged containers severely thins out the security and isolation provided by container runtimes, its use should be avoided as much as possible (CWE-250)", "sec_depr_off_imgs" : "Use of deprecated official Docker images - Use of official deprecated images should be avoided as it makes you open to vulnerabilities, quality issues and unfixed bugs (CWE-1104)", @@ -45,7 +45,7 @@ class Error: "sec_no_image_tag": "Container Image is not tagged - Prefer specifying a release version or better yet specific version digest (CWE-353)", "arc_no_apig": "No API Gateway - If following a microservices architecture you should use in front of your services an API Gateway instead of directly exposing them.", "arc_no_logging": "Log Collection not found - It is advised to setup logging for your services.", - "arc_wobbly": "Missing Healthchecks - You should setup healthchecks for your services, or check if the images used already have default ones.", + "arc_missing_healthchecks": "Missing Healthchecks - You should setup healthchecks for your services, or check if the images used already have default ones.", "sec_mounted_docker_socket": "Docker socket mounted to a container - Avoid mounting the Docker socket to container, if the container is compromised its access to the Docker socket allows control of all other containers, and even acquiring control of the host machine.", "sec_privileged_containers": "Use of Privileged Containers - Developers should always try to give and use the least privileges possible. Use of privileged containers severely thins out the security and isolation provided by container runtimes, its use should be avoided as much as possible (CWE-250)", "arc_multiple_services": "Multiple Services per Deployment Unit - If you are following a Microservices architecture you are violating the idependent deployability rule by deploying multiple microservices in the same group.", diff --git a/glitch/analysis/security/wobbly_service_interaction.py b/glitch/analysis/security/wobbly_service_interaction.py index 99a71401..13ffce37 100644 --- a/glitch/analysis/security/wobbly_service_interaction.py +++ b/glitch/analysis/security/wobbly_service_interaction.py @@ -5,7 +5,9 @@ class WobblyServiceInteractionCheck(SecuritySmellChecker): - #FIXME: The class is currently wrongly named + #FIXME: The class is currently wrongly named should + # be missinghealthchecks, although in part it is checking for the WobblyServiceInteraction + # in the case of Nomad (See Consul Comment below) so the checks need to be split def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] @@ -24,7 +26,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if k.value == "disable" and ( v.value or v.value.lower() == "true" ): - errors.append(Error("arc_wobbly", au, file, repr(au))) + errors.append(Error("arc_missing_healthchecks", au, file, repr(au))) break elif k.value == "test": if isinstance(v.value, Array): @@ -33,7 +35,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: and v.value.value[0] == "NONE" ): errors.append( - Error("arc_wobbly", au, file, repr(au)) + Error("arc_missing_healthchecks", au, file, repr(au)) ) break break @@ -72,7 +74,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: break if not found_healthcheck: - errors.append(Error("arc_wobbly", au, file, repr(au))) + errors.append(Error("arc_missing_healthchecks", au, file, repr(au))) if ( att.name in ["config", "service"] @@ -80,9 +82,9 @@ def check(self, element: CodeElement, file: str) -> List[Error]: and not found_healthcheck and has_disable_nomad ): - errors.append(Error("arc_wobbly", au, file, repr(au))) + errors.append(Error("arc_missing_healthchecks", au, file, repr(au))) if element.type == "service" and not found_healthcheck: - errors.append(Error("arc_wobbly",au,file,repr(au))) + errors.append(Error("arc_missing_healthchecks",au,file,repr(au))) return errors From e7cfd7c898291a5c20136bac723e5f8975b56031 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:34:07 +0100 Subject: [PATCH 37/65] removed unused commented code --- glitch/parsers/swarm.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 1a3561bd..cb4c122d 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -297,8 +297,7 @@ def parse_file( for attribute in service.attributes: if attribute.name == "extends": deps = [] - # if isinstance(attribute.value,String): - # deps.append(attribute.value.value) + if isinstance(attribute.value, Hash): # adds the name of file as a dependency for k, v in attribute.value.value.items(): @@ -358,11 +357,6 @@ def parse_file( else: service_from_list = services - # print( - # f'Failed to parse extends file expected at "{joint_path}". File not found.' - # ) - # - for s in service_from_list: if s.name.value == service_from: att_names = [x.name for x in service_to.attributes] From 948e3747096dd1f65179c35392c4c8a1c2142727 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 16:02:39 +0100 Subject: [PATCH 38/65] removed parsing of included files, was not a correct approach for scanning them --- glitch/parsers/swarm.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index cb4c122d..2e7e61a2 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -268,23 +268,6 @@ def parse_file( file_unit_block.add_comment(c) file_unit_block.code = "".join(code) - # FIXME: Handling the includes, might not be the best way - for inc in includes: - curr_path = os.path.split(path)[0] - joint_path = os.path.normpath(os.path.join(curr_path, inc)) - if os.path.exists(joint_path) and os.path.isfile(joint_path): - include_file_unit_block = self.parse_file(joint_path) - - for ub in include_file_unit_block.unit_blocks: - for ub_curr in file_unit_block.unit_blocks: - if ub.name == ub_curr.name: - for au in ub.atomic_units: - ub_curr.add_atomic_unit(au) - else: - print( - f'Failed to parse include file expected at "{joint_path}". File not found.' - ) - to_extend: List[List[AtomicUnit, Attribute]] = [] services = [] From 663556dfd7501e4f3188f41d006d3f9fbe73e415 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 16:57:20 +0100 Subject: [PATCH 39/65] rename class and add missing detection --- glitch/analysis/rules.py | 1 + .../security/wobbly_service_interaction.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/glitch/analysis/rules.py b/glitch/analysis/rules.py index 277504b5..459f2326 100644 --- a/glitch/analysis/rules.py +++ b/glitch/analysis/rules.py @@ -50,6 +50,7 @@ class Error: "sec_privileged_containers": "Use of Privileged Containers - Developers should always try to give and use the least privileges possible. Use of privileged containers severely thins out the security and isolation provided by container runtimes, its use should be avoided as much as possible (CWE-250)", "arc_multiple_services": "Multiple Services per Deployment Unit - If you are following a Microservices architecture you are violating the idependent deployability rule by deploying multiple microservices in the same group.", "sec_depr_off_imgs" : "Use of deprecated official Docker images - Use of official deprecated images should be avoided as it makes you open to vulnerabilities, quality issues and unfixed bugs (CWE-1104)", + "arc_wobbly_service_interaction" : "Wobbly Service Interaction - If you are following a Microservices you are likely compromising the principle of isolation of failures of microservice. Using a Consul sidecar proxy allows having Circuit Breakers and Timeouts that avoid cascading failures due to wobbly interactions between microservices", }, Tech.terraform: { "sec_integrity_policy": "Integrity Policy - Image tag is prone to be mutable or integrity monitoring is disabled. (CWE-471)", diff --git a/glitch/analysis/security/wobbly_service_interaction.py b/glitch/analysis/security/wobbly_service_interaction.py index 13ffce37..ed71016e 100644 --- a/glitch/analysis/security/wobbly_service_interaction.py +++ b/glitch/analysis/security/wobbly_service_interaction.py @@ -4,10 +4,9 @@ from typing import List -class WobblyServiceInteractionCheck(SecuritySmellChecker): - #FIXME: The class is currently wrongly named should - # be missinghealthchecks, although in part it is checking for the WobblyServiceInteraction - # in the case of Nomad (See Consul Comment below) so the checks need to be split +class MissingHealthchecksCheck(SecuritySmellChecker): + #NOTE: This class checks for Missing Healthchecks smell in Nomad and Swarm + # But it is checking for the WobblyServiceInteraction in Nomad def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] @@ -15,6 +14,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: au = element found_healthcheck = False has_disable_nomad = False + found_sidecar = False for att in au.attributes: if found_healthcheck: @@ -66,9 +66,9 @@ def check(self, element: CodeElement, file: str) -> List[Error]: _v, Hash ): # Checks for use of Consul service mesh, sidecar proxy that - # provides timeouts and circuit breaks that also avoid this smell - # technically the original smell is detectable in Nomad if this is not present - found_healthcheck = True + # provides Timeouts and Circuit Breaker mechanisms that avoid the Wobbly Service Interaction smell + # the smell is detectable in Nomad if this is not present + found_sidecar = True break if found_healthcheck: break @@ -86,5 +86,6 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if element.type == "service" and not found_healthcheck: errors.append(Error("arc_missing_healthchecks",au,file,repr(au))) - + if element.type.startswith("task") and not found_sidecar: + errors.append(Error("arc_wobbly_service_interaction",au,file,repr(au))) return errors From 52d225cd15b95ae092ad3ab5fe635641f99e9cf7 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 16:58:24 +0100 Subject: [PATCH 40/65] rename file --- .../{wobbly_service_interaction.py => missing_healthchecks.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename glitch/analysis/security/{wobbly_service_interaction.py => missing_healthchecks.py} (100%) diff --git a/glitch/analysis/security/wobbly_service_interaction.py b/glitch/analysis/security/missing_healthchecks.py similarity index 100% rename from glitch/analysis/security/wobbly_service_interaction.py rename to glitch/analysis/security/missing_healthchecks.py From 85a8fe9763c39696abfa2589fc66799bd4c415fb Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 16:59:12 +0100 Subject: [PATCH 41/65] fix typo in class name --- glitch/analysis/security/privileged_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/analysis/security/privileged_containers.py b/glitch/analysis/security/privileged_containers.py index 3d47f746..1bb97462 100644 --- a/glitch/analysis/security/privileged_containers.py +++ b/glitch/analysis/security/privileged_containers.py @@ -4,7 +4,7 @@ from typing import List -class PrivigedContainerUse(SecuritySmellChecker): +class PrivilegedContainerUse(SecuritySmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: if isinstance(element, KeyValue): From ded778e81f6b159c4e2cd770bf0583fcaa17f96e Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 17:38:52 +0100 Subject: [PATCH 42/65] moved the detection of nomad no integrity check together with the others --- .../security/nomad_integrity_check.py | 37 ----------- glitch/analysis/security/visitor.py | 64 ++++++++++++++++--- 2 files changed, 56 insertions(+), 45 deletions(-) delete mode 100644 glitch/analysis/security/nomad_integrity_check.py diff --git a/glitch/analysis/security/nomad_integrity_check.py b/glitch/analysis/security/nomad_integrity_check.py deleted file mode 100644 index 7df5123a..00000000 --- a/glitch/analysis/security/nomad_integrity_check.py +++ /dev/null @@ -1,37 +0,0 @@ -from glitch.analysis.rules import Error -from glitch.analysis.security.smell_checker import SecuritySmellChecker -from glitch.repr.inter import CodeElement, KeyValue, Hash, String -from typing import List - - -class NomadNoIntegrityCheck(SecuritySmellChecker): - # FIXME Nomad integrity check (its split from the other integrity checks) - def check(self, element: CodeElement, file: str) -> List[Error]: - if isinstance(element, KeyValue): - if element.name == "artifact" and isinstance(element.value, Hash): - # Nomad integrity check - found_checksum = False - for k, v in element.value.value.items(): - if ( - isinstance(k, String) - and k.value == "options" - and isinstance(v, Hash) - ): - for _k, _ in v.value.items(): - if isinstance(_k, String) and _k.value == "checksum": - found_checksum = True - break - if not found_checksum: - return [ - Error( # type: ignore - "sec_no_int_check", element, file, repr(element) - ) - ] # type: ignore - - if not found_checksum: - return [ - Error( # type: ignore - "sec_no_int_check", element, file, repr(element) - ) - ] # type: ignore - return [] diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index b11c2b86..5fe05235 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -3,7 +3,7 @@ import json import glitch import configparser -from urllib.parse import urlparse +from urllib.parse import urlparse, parse_qs from glitch.analysis.rules import Error, RuleVisitor, SmellChecker from nltk.tokenize import WordPunctTokenizer # type: ignore from typing import Tuple, List, Optional @@ -91,8 +91,12 @@ def check(self, element: CodeElement, file: str) -> List[Error]: or img_name == off_img_complete_link ): return [] - - return [Error("sec_non_official_image", bad_element, file, repr(bad_element))] + + return [ + Error( + "sec_non_official_image", bad_element, file, repr(bad_element) + ) + ] return [] @@ -109,7 +113,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: or (isinstance(element.value, Hash) and element.name == "config") ): image = "" - + if isinstance(element.value, String): image = element.value.value else: @@ -134,14 +138,24 @@ def check(self, element: CodeElement, file: str) -> List[Error]: checksum_s[0] == "sha256" and len(checksum) != 64 ): # sha256 256 digest -> 64 hexadecimal digits errors.append( - Error("sec_image_integrity", bad_element, file, repr(bad_element)) + Error( + "sec_image_integrity", + bad_element, + file, + repr(bad_element), + ) ) if image != "" and has_tag: tag = tag.lower() if not has_digest: errors.append( - Error("sec_image_integrity", bad_element, file, repr(bad_element)) + Error( + "sec_image_integrity", + bad_element, + file, + repr(bad_element), + ) ) dangerous_tags: List[str] = SecurityVisitor.DANGEROUS_IMAGE_TAGS @@ -149,7 +163,12 @@ def check(self, element: CodeElement, file: str) -> List[Error]: for dt in dangerous_tags: if dt in tag: errors.append( - Error("sec_unstable_tag", bad_element, file, repr(bad_element)) + Error( + "sec_unstable_tag", + bad_element, + file, + repr(bad_element), + ) ) break if ( @@ -632,6 +651,9 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: missing_integrity_checks = {} for au in u.atomic_units: result = self.check_integrity_check(au, file) + if result is not None and result[0] is None: + errors.append(result[1]) + continue if result is not None: missing_integrity_checks[result[0]] = result[1] continue @@ -651,7 +673,7 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: return errors @staticmethod - def check_integrity_check(au: AtomicUnit, path: str) -> Optional[Tuple[str, Error]]: + def check_integrity_check(au: AtomicUnit, path: str) -> Optional[Tuple[str|None, Error]]: for item in SecurityVisitor.DOWNLOAD: if not isinstance(au.name, str): continue @@ -673,6 +695,32 @@ def check_integrity_check(au: AtomicUnit, path: str) -> Optional[Tuple[str, Erro else repr(a.value).strip().lower() ) + # Nomad integrity check + if a.name == "artifact" and isinstance(a.value, Hash): + found_checksum = False + for k, v in a.value.value.items(): + if ( + isinstance(k, String) + and k.value == "options" + and isinstance(v, Hash) + ): + for _k, _ in v.value.items(): + if isinstance(_k, String) and _k.value == "checksum": + found_checksum = True + break + elif ( + isinstance(k, String) + and k.value == "source" + and isinstance(v, String) + ): + # artifact uses https://github.com/hashicorp/go-getter + parsed_source = urlparse(v.value) # type: ignore + checksum = parse_qs(parsed_source.query).get("checksum", []) # type: ignore + if checksum: + found_checksum = True + if not found_checksum: + return (None, Error("sec_no_int_check", a, path, repr(a))) # type: ignore + for item in SecurityVisitor.DOWNLOAD: if not re.search( r"(http|https|www)[^ ,]*\.{text}".format(text=item), value From 05945629c5dc2604dd8af7d70c883243a799a538 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 18:22:09 +0100 Subject: [PATCH 43/65] fix: move smell checkers out of visitor.py --- .../security/container_image_tags_smells.py | 84 +++++++ .../security/deprecated_official_images.py | 3 +- .../security/docker_socket_mounted.py | 2 +- glitch/analysis/security/log_aggregation.py | 82 +++++++ .../analysis/security/missing_healthchecks.py | 2 +- .../multiple_services_per_deplyment_unit.py | 5 +- glitch/analysis/security/no_api_gateway.py | 5 +- .../security/non_official_container_images.py | 49 ++++ .../security/privileged_containers.py | 7 +- glitch/analysis/security/visitor.py | 220 +----------------- glitch/analysis/utils.py | 24 ++ 11 files changed, 256 insertions(+), 227 deletions(-) create mode 100644 glitch/analysis/security/container_image_tags_smells.py create mode 100644 glitch/analysis/security/log_aggregation.py create mode 100644 glitch/analysis/security/non_official_container_images.py create mode 100644 glitch/analysis/utils.py diff --git a/glitch/analysis/security/container_image_tags_smells.py b/glitch/analysis/security/container_image_tags_smells.py new file mode 100644 index 00000000..f7852953 --- /dev/null +++ b/glitch/analysis/security/container_image_tags_smells.py @@ -0,0 +1,84 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import CodeElement, KeyValue, Hash, String +from glitch.analysis.utils import parse_container_image_name +from typing import List + + +class ContainerImageTagsSmells(SecuritySmellChecker): + # NOTE: This is the implementation for Nomad and Swarm + def check(self, element: CodeElement, file: str) -> List[Error]: + errors: List[Error] = [] + bad_element = element + if isinstance(element, KeyValue) and ( + (element.name == "image" and isinstance(element.value, String)) + or (isinstance(element.value, Hash) and element.name == "config") + ): + image = "" + + if isinstance(element.value, String): + image = element.value.value + else: + for k, v in element.value.value.items(): + if isinstance(k, String) and k.value == "image": + image = v.value + bad_element = v + break + + has_digest, has_tag = False, False + _, tag, digest = parse_container_image_name(image) + + if tag != "": + has_tag = True + if digest != "": + has_digest = True + + if image != "" and has_digest: # image tagged with digest + checksum_s = digest.split(":") + checksum = checksum_s[-1] + if ( + checksum_s[0] == "sha256" and len(checksum) != 64 + ): # sha256 256 digest -> 64 hexadecimal digits + errors.append( + Error( + "sec_image_integrity", + bad_element, + file, + repr(bad_element), + ) + ) + + if image != "" and has_tag: + tag = tag.lower() + if not has_digest: + errors.append( + Error( + "sec_image_integrity", + bad_element, + file, + repr(bad_element), + ) + ) + + dangerous_tags: List[str] = SecurityVisitor.DANGEROUS_IMAGE_TAGS + + for dt in dangerous_tags: + if dt in tag: + errors.append( + Error( + "sec_unstable_tag", + bad_element, + file, + repr(bad_element), + ) + ) + break + if ( + image != "" and not has_digest and not has_tag + ): # Image not tagged, avoids mistakenely nomad tasks without images (non-docker or non-podman) + errors.append( + Error("sec_no_image_tag", bad_element, file, repr(bad_element)) + ) + + return errors diff --git a/glitch/analysis/security/deprecated_official_images.py b/glitch/analysis/security/deprecated_official_images.py index 77ec4fd3..9a493c9d 100644 --- a/glitch/analysis/security/deprecated_official_images.py +++ b/glitch/analysis/security/deprecated_official_images.py @@ -2,6 +2,7 @@ from glitch.analysis.security.smell_checker import SecuritySmellChecker from glitch.analysis.security.visitor import SecurityVisitor from glitch.repr.inter import CodeElement, KeyValue, Hash, String +from glitch.analysis.utils import parse_container_image_name from typing import List @@ -24,7 +25,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: bad_element = v break if image != "": - img_name, _, _ = SecurityVisitor.image_parser(image) + img_name, _, _ = parse_container_image_name(image) for obsolete_img in SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES: obsolete_img_dockerio = f"docker.io/library/{obsolete_img}" obsolete_img_library = f"library/{obsolete_img}" diff --git a/glitch/analysis/security/docker_socket_mounted.py b/glitch/analysis/security/docker_socket_mounted.py index d65e5b8f..66f83a93 100644 --- a/glitch/analysis/security/docker_socket_mounted.py +++ b/glitch/analysis/security/docker_socket_mounted.py @@ -4,7 +4,7 @@ from typing import List -class DockerSocketMountedInsideContainerUse(SecuritySmellChecker): +class DockerSocketMountedInsideContainer(SecuritySmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] if isinstance(element, KeyValue): diff --git a/glitch/analysis/security/log_aggregation.py b/glitch/analysis/security/log_aggregation.py new file mode 100644 index 00000000..c310161b --- /dev/null +++ b/glitch/analysis/security/log_aggregation.py @@ -0,0 +1,82 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import CodeElement, Hash, String, UnitBlock +from glitch.analysis.utils import parse_container_image_name +from typing import List + + +class LogAggregatorAbsence(SecuritySmellChecker): + def check(self, element: CodeElement, file: str) -> List[Error]: + errors: List[Error] = [] + + if isinstance(element, UnitBlock): + # HACK: Besides the ones we are explicitly stating a registry we assume the default, normally Docker hub + log_collectors: List[str] = SecurityVisitor.LOG_AGGREGATORS_AND_COLLECTORS + + log_drivers: List[str] = SecurityVisitor.DOCKER_LOG_DRIVERS + + has_log_collector = False + + for au in element.atomic_units: + if au.type != "service" and not au.type.startswith("task."): + # Don't analyze Unit Blocks which aren't tasks or services + return [] + + for att in au.attributes: + image = "" + if att.name == "config" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if isinstance(k, String) and k.value == "image": + image = v.value + break + + elif att.name == "image" and isinstance(att.value, String): + image = att.value.value + + img_name, _, _ = parse_container_image_name(image) + + if image != "": + for lc in log_collectors: + if img_name.startswith(lc): + return [] + break + + # if it doesn't have a log collector/aggregator in the deployment + if not has_log_collector: + for au in element.atomic_units: + has_logging = False + for att in au.attributes: + if has_logging: + break + if att.name == "logging" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if ( + k.value == "driver" + and isinstance(v, String) + and v.value in log_drivers + ): + has_logging = True + break + elif att.name == "config" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if ( + isinstance(k, String) + and k.value == "logging" + and isinstance(v, Hash) + ): + for _k, _v in v.value.items(): + if ( + _k.value == "driver" + and isinstance(_v, String) + and _v.value in log_drivers + ): + has_logging = True + break + if has_logging: + break + + if not has_logging: + errors.append(Error("arc_no_logging", au, file, repr(au))) + + return errors diff --git a/glitch/analysis/security/missing_healthchecks.py b/glitch/analysis/security/missing_healthchecks.py index ed71016e..35e46011 100644 --- a/glitch/analysis/security/missing_healthchecks.py +++ b/glitch/analysis/security/missing_healthchecks.py @@ -4,7 +4,7 @@ from typing import List -class MissingHealthchecksCheck(SecuritySmellChecker): +class MissingHealthchecks(SecuritySmellChecker): #NOTE: This class checks for Missing Healthchecks smell in Nomad and Swarm # But it is checking for the WobblyServiceInteraction in Nomad def check(self, element: CodeElement, file: str) -> List[Error]: diff --git a/glitch/analysis/security/multiple_services_per_deplyment_unit.py b/glitch/analysis/security/multiple_services_per_deplyment_unit.py index fc5f16d6..bbbed76b 100644 --- a/glitch/analysis/security/multiple_services_per_deplyment_unit.py +++ b/glitch/analysis/security/multiple_services_per_deplyment_unit.py @@ -2,10 +2,11 @@ from glitch.analysis.security.smell_checker import SecuritySmellChecker from glitch.analysis.security.visitor import SecurityVisitor from glitch.repr.inter import CodeElement, Hash, String, UnitBlock, UnitBlockType +from glitch.analysis.utils import parse_container_image_name from typing import List -class MultipleServicesPerDeploymentUnitCheck(SecuritySmellChecker): +class MultipleServicesPerDeploymentUnit(SecuritySmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: # FIXME: Besides log collectors, there are other types of agents/sidecars for observability (some of which are also on the log collector list) # and proxies which should also be allowed besides the main microservice @@ -19,7 +20,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if att.name == "config" and isinstance(att.value, Hash): for k, v in att.value.value.items(): if isinstance(k, String) and k.value == "image": - image_name, _, _ = SecurityVisitor.image_parser( + image_name, _, _ = parse_container_image_name( v.value ) break diff --git a/glitch/analysis/security/no_api_gateway.py b/glitch/analysis/security/no_api_gateway.py index bc04437f..aa72757c 100644 --- a/glitch/analysis/security/no_api_gateway.py +++ b/glitch/analysis/security/no_api_gateway.py @@ -10,10 +10,11 @@ UnitBlock, UnitBlockType, ) +from glitch.analysis.utils import parse_container_image_name from typing import List, Dict, Any -class NoAPIGatewayCheck(SecuritySmellChecker): +class NoAPIGateway(SecuritySmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] # Tries to follow an logic similar to the one presented for Kubernetes pods ond doi: 10.5220/0011845500003488 @@ -100,7 +101,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) if isinstance(k, String) and k.value == "image": - image_name, _, _ = SecurityVisitor.image_parser(v.value) + image_name, _, _ = parse_container_image_name(v.value) if ( image_name in SecurityVisitor.API_GATEWAYS or is_api_gateway diff --git a/glitch/analysis/security/non_official_container_images.py b/glitch/analysis/security/non_official_container_images.py new file mode 100644 index 00000000..20f18704 --- /dev/null +++ b/glitch/analysis/security/non_official_container_images.py @@ -0,0 +1,49 @@ +from glitch.analysis.rules import Error +from glitch.analysis.security.smell_checker import SecuritySmellChecker +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.repr.inter import CodeElement, KeyValue, Hash, String +from glitch.analysis.utils import parse_container_image_name +from typing import List + + +class NonOfficialContainerImage(SecuritySmellChecker): + # NOTE: This is the implementation for Nomad and Swarm + def check(self, element: CodeElement, file: str) -> List[Error]: + image = "" + bad_element = element + + if isinstance(element, KeyValue) and element.name == "image": + if isinstance(element.value, String): + image = element.value.value + elif ( + isinstance(element, KeyValue) + and element.name == "config" + and isinstance(element.value, Hash) + ): + for k, v in element.value.value.items(): + if isinstance(k, String) and k.value == "image": + image = v.value + bad_element = v + break + + img_name, _, _ = parse_container_image_name(image) + + if img_name != "": + for off_img in SecurityVisitor.DOCKER_OFFICIAL_IMAGES: + off_img_dockerio = f"docker.io/library/{off_img}" + off_img_library = f"library/{off_img}" + off_img_complete_link = f"registry.hub.docker.com/library/{off_img}" + + if ( + img_name == off_img + or img_name == off_img_dockerio + or img_name == off_img_library + or img_name == off_img_complete_link + ): + return [] + + return [ + Error("sec_non_official_image", bad_element, file, repr(bad_element)) + ] + + return [] diff --git a/glitch/analysis/security/privileged_containers.py b/glitch/analysis/security/privileged_containers.py index 1bb97462..cd16540b 100644 --- a/glitch/analysis/security/privileged_containers.py +++ b/glitch/analysis/security/privileged_containers.py @@ -4,16 +4,17 @@ from typing import List -class PrivilegedContainerUse(SecuritySmellChecker): +class PrivilegedContainers(SecuritySmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: if isinstance(element, KeyValue): - if ( element.name == "privileged" and isinstance(element.value, Boolean) and element.value.value ): - return [Error("sec_privileged_containers", element, file, repr(element))] + return [ + Error("sec_privileged_containers", element, file, repr(element)) + ] elif element.name == "config" and isinstance(element.value, Hash): for k, v in element.value.value.items(): if isinstance(k, String) and k.value == "privileged" and v.value: diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index 5fe05235..f5467d72 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -58,209 +58,6 @@ def check(self, element: CodeElement, file: str) -> List[Error]: return [Error("sec_non_official_image", element, file, repr(element))] return [] - class OrchestratorNonOfficialImageSmell(SmellChecker): - def check(self, element: CodeElement, file: str) -> List[Error]: - image = "" - bad_element = element - if isinstance(element, KeyValue) and element.name == "image": - if isinstance(element.value, String): - image = element.value.value - elif ( - isinstance(element, KeyValue) - and element.name == "config" - and isinstance(element.value, Hash) - ): - for k, v in element.value.value.items(): - if isinstance(k, String) and k.value == "image": - image = v.value - bad_element = v - break - - img_name, _, _ = SecurityVisitor.image_parser(image) - - if img_name != "": - for off_img in SecurityVisitor.DOCKER_OFFICIAL_IMAGES: - off_img_dockerio = f"docker.io/library/{off_img}" - off_img_library = f"library/{off_img}" - off_img_complete_link = f"registry.hub.docker.com/library/{off_img}" - - if ( - img_name == off_img - or img_name == off_img_dockerio - or img_name == off_img_library - or img_name == off_img_complete_link - ): - return [] - - return [ - Error( - "sec_non_official_image", bad_element, file, repr(bad_element) - ) - ] - - return [] - - class ImageTagsSmell(SmellChecker): - def check(self, element: CodeElement, file: str) -> List[Error]: - return [] - - class OrchestratorImageTagsSmell(SmellChecker): - def check(self, element: CodeElement, file: str) -> List[Error]: - errors: List[Error] = [] - bad_element = element - if isinstance(element, KeyValue) and ( - (element.name == "image" and isinstance(element.value, String)) - or (isinstance(element.value, Hash) and element.name == "config") - ): - image = "" - - if isinstance(element.value, String): - image = element.value.value - else: - for k, v in element.value.value.items(): - if isinstance(k, String) and k.value == "image": - image = v.value - bad_element = v - break - - has_digest, has_tag = False, False - _, tag, digest = SecurityVisitor.image_parser(image) - - if tag != "": - has_tag = True - if digest != "": - has_digest = True - - if image != "" and has_digest: # image tagged with digest - checksum_s = digest.split(":") - checksum = checksum_s[-1] - if ( - checksum_s[0] == "sha256" and len(checksum) != 64 - ): # sha256 256 digest -> 64 hexadecimal digits - errors.append( - Error( - "sec_image_integrity", - bad_element, - file, - repr(bad_element), - ) - ) - - if image != "" and has_tag: - tag = tag.lower() - if not has_digest: - errors.append( - Error( - "sec_image_integrity", - bad_element, - file, - repr(bad_element), - ) - ) - - dangerous_tags: List[str] = SecurityVisitor.DANGEROUS_IMAGE_TAGS - - for dt in dangerous_tags: - if dt in tag: - errors.append( - Error( - "sec_unstable_tag", - bad_element, - file, - repr(bad_element), - ) - ) - break - if ( - image != "" and not has_digest and not has_tag - ): # Image not tagged, avoids mistakenely nomad tasks without images (non-docker or non-podman) - errors.append( - Error("sec_no_image_tag", bad_element, file, repr(bad_element)) - ) - - return errors - - class LogAggregatorAbsenceSmell(SmellChecker): - def check(self, element: CodeElement, file: str) -> List[Error]: - return [] - - class OrchestratorLogAggregatorAbsenceSmell(SmellChecker): - # By default Docker uses the - def check(self, element: CodeElement, file: str) -> List[Error]: - errors: List[Error] = [] - if isinstance(element, UnitBlock): - # HACK: Besides the ones we are explicitly stating a registry we assume the default, normally Docker hub - log_collectors: List[str] = ( - SecurityVisitor.LOG_AGGREGATORS_AND_COLLECTORS - ) - - log_drivers: List[str] = SecurityVisitor.DOCKER_LOG_DRIVERS - - has_log_collector = False - - for au in element.atomic_units: - if au.type != "service" and not au.type.startswith("task."): - # Don't analyze Unit Blocks which aren't tasks or services - return [] - - for att in au.attributes: - image = "" - if att.name == "config" and isinstance(att.value, Hash): - for k, v in att.value.value.items(): - if isinstance(k, String) and k.value == "image": - image = v.value - break - - elif att.name == "image" and isinstance(att.value, String): - image = att.value.value - - img_name, _, _ = SecurityVisitor.image_parser(image) - - if image != "": - for lc in log_collectors: - if img_name.startswith(lc): - return [] - break - - # if it doesn't have a log collector/aggregator in the deployment - if not has_log_collector: - for au in element.atomic_units: - has_logging = False - for att in au.attributes: - if has_logging: - break - if att.name == "logging" and isinstance(att.value, Hash): - for k, v in att.value.value.items(): - if ( - k.value == "driver" - and isinstance(v, String) - and v.value in log_drivers - ): - has_logging = True - break - elif att.name == "config" and isinstance(att.value, Hash): - for k, v in att.value.value.items(): - if ( - isinstance(k, String) - and k.value == "logging" - and isinstance(v, Hash) - ): - for _k, _v in v.value.items(): - if ( - _k.value == "driver" - and isinstance(_v, String) - and _v.value in log_drivers - ): - has_logging = True - break - if has_logging: - break - - if not has_logging: - errors.append(Error("arc_no_logging", au, file, repr(au))) - - return errors - def __init__(self, tech: Tech) -> None: super().__init__(tech) @@ -272,14 +69,8 @@ def __init__(self, tech: Tech) -> None: for child in TerraformSmellChecker.__subclasses__(): self.checkers.append(child()) - self.image_tag_smells = SecurityVisitor.ImageTagsSmell() - self.log_agg_smell = SecurityVisitor.LogAggregatorAbsenceSmell() if tech == Tech.docker: self.non_off_img = SecurityVisitor.DockerNonOfficialImageSmell() - elif tech in [Tech.swarm, Tech.nomad]: - self.non_off_img = SecurityVisitor.OrchestratorNonOfficialImageSmell() - self.image_tag_smells = SecurityVisitor.OrchestratorImageTagsSmell() - self.log_agg_smell = SecurityVisitor.OrchestratorLogAggregatorAbsenceSmell() else: self.non_off_img = SecurityVisitor.NonOfficialImageSmell() @@ -480,12 +271,6 @@ def check_dependency(self, d: Dependency, file: str) -> List[Error]: def __check_keyvalue(self, c: KeyValue, file: str) -> List[Error]: errors: List[Error] = [] - # official docker image for swarm and nomad - - errors += self.non_off_img.check(c, file) - # image tags smells for swarm and nomad - errors += self.image_tag_smells.check(c, file) - # check https/tls/ssl in Hash values if isinstance(c.value, Hash): pairs_to_check = [c.value.value] @@ -664,7 +449,6 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: errors += missing_integrity_checks.values() errors += self.non_off_img.check(u, file) - errors += self.log_agg_smell.check(u, file) for checker in self.checkers: checker.code = self.code @@ -673,7 +457,9 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: return errors @staticmethod - def check_integrity_check(au: AtomicUnit, path: str) -> Optional[Tuple[str|None, Error]]: + def check_integrity_check( + au: AtomicUnit, path: str + ) -> Optional[Tuple[str | None, Error]]: for item in SecurityVisitor.DOWNLOAD: if not isinstance(au.name, str): continue diff --git a/glitch/analysis/utils.py b/glitch/analysis/utils.py new file mode 100644 index 00000000..a05a8e44 --- /dev/null +++ b/glitch/analysis/utils.py @@ -0,0 +1,24 @@ +from typing import Tuple + + +def parse_container_image_name(img_name: str) -> Tuple[str, str, str]: + image, tag, digest = "", "", "" + + if "@" in img_name: + parts_dig = img_name.split("@") + digest = parts_dig[-1] + if ":" in parts_dig[0]: + parts_tag = parts_dig[0].split(":") + image = parts_tag[0] + tag = parts_tag[1] + else: + image = parts_dig[0] + + elif ":" in img_name: + parts_tag = img_name.split(":") + image = parts_tag[0] + tag = parts_tag[1] + else: + image = img_name + + return image, tag, digest From 5b6cd014529e7299c2b8e3129935a91ed9c9d9c7 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 18:23:42 +0100 Subject: [PATCH 44/65] move container image name parser util to separated file --- glitch/analysis/security/visitor.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index f5467d72..5696bf8a 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -18,29 +18,6 @@ class SecurityVisitor(RuleVisitor): __URL_REGEX = r"^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([_\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$" - @staticmethod - def image_parser(img_name: str): - image, tag, digest = "", "", "" - - if "@" in img_name: - parts_dig = img_name.split("@") - digest = parts_dig[-1] - if ":" in parts_dig[0]: - parts_tag = parts_dig[0].split(":") - image = parts_tag[0] - tag = parts_tag[1] - else: - image = parts_dig[0] - - elif ":" in img_name: - parts_tag = img_name.split(":") - image = parts_tag[0] - tag = parts_tag[1] - else: - image = img_name - - return image, tag, digest - class NonOfficialImageSmell(SmellChecker): def check(self, element: CodeElement, file: str) -> List[Error]: return [] From 8bd96328d08609507378fa1ac77bf346bab6c996 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Mon, 25 Aug 2025 18:24:16 +0100 Subject: [PATCH 45/65] format swarm.py --- glitch/parsers/swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 2e7e61a2..797c2a4b 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -280,7 +280,7 @@ def parse_file( for attribute in service.attributes: if attribute.name == "extends": deps = [] - + if isinstance(attribute.value, Hash): # adds the name of file as a dependency for k, v in attribute.value.value.items(): From ae48806de72298595380a486dd41e8e4157d3a88 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:33:19 +0100 Subject: [PATCH 46/65] fix matching invalid ip bindings in complete command invocations, by spliting the args --- glitch/analysis/security/invalid_bind.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/glitch/analysis/security/invalid_bind.py b/glitch/analysis/security/invalid_bind.py index 9f861e7b..10dea455 100644 --- a/glitch/analysis/security/invalid_bind.py +++ b/glitch/analysis/security/invalid_bind.py @@ -5,6 +5,7 @@ from glitch.repr.inter import * from glitch.analysis.expr_checkers.string_checker import StringChecker from typing import List +from shlex import split as shsplit class InvalidBind(SecuritySmellChecker): @@ -27,4 +28,10 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ): return [Error("sec_invalid_bind", element, file, repr(element))] + if isinstance(element, KeyValue) and isinstance(element.value, String): + #HACK: splits a string in command parts as for complete commmands invocations the regex wasn't + # matching on full command invocations that included a reference to "0.0.0.0" or the its http/s variants + for part in shsplit(element.value.value): + if check_invalid.str_check(part): + return [Error("sec_invalid_bind", element, file, repr(element))] return [] From 0ade21c62236785c4a378dfe3a6f9ed1c18a458d Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:36:15 +0100 Subject: [PATCH 47/65] fix typo --- glitch/analysis/rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/analysis/rules.py b/glitch/analysis/rules.py index 459f2326..8110e44a 100644 --- a/glitch/analysis/rules.py +++ b/glitch/analysis/rules.py @@ -18,7 +18,7 @@ class Error: "sec_hard_secr": "Hard-coded secret - Developers should not reveal sensitive information in the source code. (CWE-798)", "sec_hard_pass": "Hard-coded password - Developers should not reveal sensitive information in the source code. (CWE-259)", "sec_hard_user": "Hard-coded user - Developers should not reveal sensitive information in the source code. (CWE-798)", - "sec_invalid_bind": "Invalid IP address binding - Binding to the address 0.0.0.0 allows connections from every possible network which might be a security issues. (CWE-284)", + "sec_invalid_bind": "Invalid IP address binding - Binding to the address 0.0.0.0 allows connections from every possible network which might be a security issue. (CWE-284)", "sec_no_int_check": "No integrity check - The content of files downloaded from the internet should be checked. (CWE-353)", "sec_no_default_switch": "Missing default case statement - Not handling every possible input combination might allow an attacker to trigger an error for an unhandled value. (CWE-478)", "sec_full_permission_filesystem": "Full permission to the filesystem - Files should not have full permissions to every user. (CWE-732)", From f8ce6252894d911e54f176b722c53980cd50ebaa Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:37:37 +0100 Subject: [PATCH 48/65] remove hack, fix variable parsing in swarm --- glitch/analysis/security/hard_secr.py | 16 +++++++------- glitch/parsers/swarm.py | 31 ++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/glitch/analysis/security/hard_secr.py b/glitch/analysis/security/hard_secr.py index 65e9420a..4b5a593c 100644 --- a/glitch/analysis/security/hard_secr.py +++ b/glitch/analysis/security/hard_secr.py @@ -28,13 +28,16 @@ def __check_pair( SecurityVisitor.PASSWORDS + SecurityVisitor.SECRETS + SecurityVisitor.USERS ): secr_checker = StringChecker( - lambda s: (re.match( - r"[_A-Za-z0-9$\/\.\[\]-]*{text}\b".format(text=item), s + lambda s: ( + re.match(r"[_A-Za-z0-9$\/\.\[\]-]*{text}\b".format(text=item), s) + is not None ) - is not None) or (re.match( - r"[_A-Za-z0-9$\/\.\[\]-]*{text}\b".format(text=item.upper()), s + or ( + re.match( + r"[_A-Za-z0-9$\/\.\[\]-]*{text}\b".format(text=item.upper()), s + ) + is not None ) - is not None) ) if secr_checker.check(name) and not whitelist_checker.check(name): if not var_checker.check(value): @@ -80,9 +83,6 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) elif isinstance(element, KeyValue) and isinstance(element.value, Hash): for key, value in element.value.value.items(): - #HACK: ignore cases where values are obtained via environment variables - if isinstance(value,String) and value.value.startswith("${") and value.value.endswith("}"): - continue errors += self.__check_pair(element, key, value, file) return errors diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 797c2a4b..5809a416 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -29,6 +29,7 @@ UnitBlockType, Array, Dependency, + VariableReference, ) @@ -119,6 +120,31 @@ def create_attribute( att[1].value = False att_value: Expr = self.get_value(att[1], code) + + if ( + att[0].value == "environment" + and isinstance(att[1], MappingNode) + and isinstance(att_value, Hash) + ): + temp_value_copy = att_value.value.copy() + modified = False + for k, v in att_value.value.items(): + if ( + isinstance(v, String) + and v.value.startswith("${") + and v.value.endswith("}") + ): + new_val = VariableReference( + v.value, ElementInfo.from_code_element(v) + ) + temp_value_copy[k] = new_val + modified = True + if modified: + att_value = Hash( + temp_value_copy, + ElementInfo.from_code_element(att_value), + ) + if att[0].value == "environment" and isinstance( att[1], SequenceNode ): @@ -140,7 +166,10 @@ def create_attribute( split_str = curr_str.split("=", 1) if len(split_str) == 2: key, n_val = split_str - val_s = String(n_val, elem_info) + if n_val.startswith("${") and n_val.endswith("}"): + val_s = VariableReference(n_val, elem_info) + else: + val_s = String(n_val, elem_info) else: key = curr_str val_s = Null(elem_info) From 7ceb362203362384c9de46f91fa407f31926970f Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:46:58 +0100 Subject: [PATCH 49/65] change to use StringChecker and add handling of arrays and hashes --- glitch/analysis/security/visitor.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index 5696bf8a..955fcf22 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -10,7 +10,7 @@ from glitch.tech import Tech from glitch.repr.inter import * - +from glitch.analysis.expr_checkers.string_checker import StringChecker from glitch.analysis.terraform.smell_checker import TerraformSmellChecker from glitch.analysis.security.smell_checker import SecuritySmellChecker @@ -249,19 +249,37 @@ def __check_keyvalue(self, c: KeyValue, file: str) -> List[Error]: errors: List[Error] = [] # check https/tls/ssl in Hash values + + ssl_checker = StringChecker(lambda x: self.__is_http_url(x)) + weak_crypt_checker = StringChecker(lambda x: self.__is_weak_crypt(x, "")) + if isinstance(c.value, Hash): pairs_to_check = [c.value.value] while pairs_to_check: - for k, v in pairs_to_check[0].items(): - if isinstance(v, String) and self.__is_http_url(v.value): + for _, v in pairs_to_check[0].items(): + if ssl_checker.check(v): errors.append(Error("sec_https", v, file, repr(v))) - if isinstance(v, String) and self.__is_weak_crypt(v.value, k.value): + if weak_crypt_checker.check(v): errors.append(Error("sec_weak_crypt", v, file, repr(v))) if isinstance(v, Hash): pairs_to_check.append(v.value) + pairs_to_check.pop(0) + elif isinstance(c.value, Array): + for x in c.value.value: + if ssl_checker.check(x): + errors.append(Error("sec_https", x, file, repr(x))) + if weak_crypt_checker.check(x): + errors.append(Error("sec_weak_crypt", x, file, repr(x))) + + else: + if ssl_checker.check(c.value): + errors.append(Error("sec_https", c, file, repr(c))) + if weak_crypt_checker.check(c.value): + errors.append(Error("sec_weak_crypt", c, file, repr(c))) + c.name = c.name.strip().lower() # if isinstance(c.value, type(None)): @@ -461,6 +479,7 @@ def check_integrity_check( # Nomad integrity check if a.name == "artifact" and isinstance(a.value, Hash): found_checksum = False + for k, v in a.value.value.items(): if ( isinstance(k, String) From 365c90674f6f44a511da73a86792f0eb784c0890 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 28 Aug 2025 14:07:27 +0100 Subject: [PATCH 50/65] fix empty strings in env var value --- glitch/parsers/swarm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/glitch/parsers/swarm.py b/glitch/parsers/swarm.py index 5809a416..47295339 100644 --- a/glitch/parsers/swarm.py +++ b/glitch/parsers/swarm.py @@ -166,10 +166,12 @@ def create_attribute( split_str = curr_str.split("=", 1) if len(split_str) == 2: key, n_val = split_str + if n_val.strip() in ["\"\"", "''"]: + n_val = "" if n_val.startswith("${") and n_val.endswith("}"): val_s = VariableReference(n_val, elem_info) else: - val_s = String(n_val, elem_info) + val_s = String(n_val.strip(), elem_info) else: key = curr_str val_s = Null(elem_info) From aef512b7e0cea64d561b31938849d615a692d3c3 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 28 Aug 2025 18:42:58 +0100 Subject: [PATCH 51/65] fix some of SecurityVisitor global constants not being initialized for all the techs (as some of the smell detectors that use them were 'made available' to all other techs) --- glitch/analysis/security/visitor.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index 955fcf22..eb338964 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -173,20 +173,17 @@ def config(self, config_path: str) -> None: "official_docker_images" ) - if self.tech in [Tech.swarm, Tech.nomad]: - SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES = self._load_data_file( - "deprecated_official_docker_images" - ) - SecurityVisitor.LOG_AGGREGATORS_AND_COLLECTORS = self._load_data_file( - "log_collectors_and_aggregators" - ) - SecurityVisitor.DANGEROUS_IMAGE_TAGS = self._load_data_file( - "dangerous_image_tags" - ) - SecurityVisitor.DOCKER_LOG_DRIVERS = self._load_data_file( - "docker_log_drivers" - ) - SecurityVisitor.API_GATEWAYS = self._load_data_file("api_gateways") + SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES = self._load_data_file( + "deprecated_official_docker_images" + ) + SecurityVisitor.LOG_AGGREGATORS_AND_COLLECTORS = self._load_data_file( + "log_collectors_and_aggregators" + ) + SecurityVisitor.DANGEROUS_IMAGE_TAGS = self._load_data_file( + "dangerous_image_tags" + ) + SecurityVisitor.DOCKER_LOG_DRIVERS = self._load_data_file("docker_log_drivers") + SecurityVisitor.API_GATEWAYS = self._load_data_file("api_gateways") @staticmethod def _load_data_file(file: str) -> List[str]: From f1ccef5bc774c457d214046c71a5660cbb212d47 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 28 Aug 2025 18:45:54 +0100 Subject: [PATCH 52/65] add: swarm tests --- glitch/tests/security/swarm/files/admin.yml | 12 + .../no_tag_and_digest.yml | 9 + .../no_tag_no_digest.yml | 9 + .../normal_tag_and_digest.yml | 9 + .../normal_tag_and_no_digest.yml | 9 + .../unstable_tag_and_digest.yml | 9 + .../unstable_tag_no_digest.yml | 9 + .../deprecated_docker_official_image.yml | 11 + .../swarm/files/docker_socket_mounted.yml | 11 + .../hard_secr/hard_secr_empty_password.yml | 11 + .../files/hard_secr/hard_secr_password.yml | 11 + .../files/hard_secr/hard_secr_secret.yml | 11 + .../swarm/files/hard_secr/hard_secr_user.yml | 11 + .../tests/security/swarm/files/https_tls.yml | 11 + .../files/invalid_bind/invalid_bind_array.yml | 13 ++ .../invalid_bind/invalid_bind_exec_form.yml | 10 + .../files/invalid_bind/invalid_bind_hash.yml | 11 + .../invalid_bind/invalid_bind_string.yml | 10 + .../swarm/files/missing_healthchecks.yml | 5 + .../swarm/files/no_log_aggregation.yml | 9 + .../swarm/files/non_official_image.yml | 11 + .../security/swarm/files/priv_container.yml | 10 + .../security/swarm/files/susp_comment.yml | 11 + .../swarm/files/template_no_smells.yml | 9 + .../files/weak_crypt/weak_crypt_array.yml | 13 ++ .../weak_crypt/weak_crypt_array_exec_form.yml | 11 + .../files/weak_crypt/weak_crypt_hash.yml | 11 + .../files/weak_crypt/weak_crypt_string.yml | 12 + glitch/tests/security/swarm/test_security.py | 219 ++++++++++++++++++ 29 files changed, 508 insertions(+) create mode 100644 glitch/tests/security/swarm/files/admin.yml create mode 100644 glitch/tests/security/swarm/files/container_image_tag_smells/no_tag_and_digest.yml create mode 100644 glitch/tests/security/swarm/files/container_image_tag_smells/no_tag_no_digest.yml create mode 100644 glitch/tests/security/swarm/files/container_image_tag_smells/normal_tag_and_digest.yml create mode 100644 glitch/tests/security/swarm/files/container_image_tag_smells/normal_tag_and_no_digest.yml create mode 100644 glitch/tests/security/swarm/files/container_image_tag_smells/unstable_tag_and_digest.yml create mode 100644 glitch/tests/security/swarm/files/container_image_tag_smells/unstable_tag_no_digest.yml create mode 100644 glitch/tests/security/swarm/files/deprecated_docker_official_image.yml create mode 100644 glitch/tests/security/swarm/files/docker_socket_mounted.yml create mode 100644 glitch/tests/security/swarm/files/hard_secr/hard_secr_empty_password.yml create mode 100644 glitch/tests/security/swarm/files/hard_secr/hard_secr_password.yml create mode 100644 glitch/tests/security/swarm/files/hard_secr/hard_secr_secret.yml create mode 100644 glitch/tests/security/swarm/files/hard_secr/hard_secr_user.yml create mode 100644 glitch/tests/security/swarm/files/https_tls.yml create mode 100644 glitch/tests/security/swarm/files/invalid_bind/invalid_bind_array.yml create mode 100644 glitch/tests/security/swarm/files/invalid_bind/invalid_bind_exec_form.yml create mode 100644 glitch/tests/security/swarm/files/invalid_bind/invalid_bind_hash.yml create mode 100644 glitch/tests/security/swarm/files/invalid_bind/invalid_bind_string.yml create mode 100644 glitch/tests/security/swarm/files/missing_healthchecks.yml create mode 100644 glitch/tests/security/swarm/files/no_log_aggregation.yml create mode 100644 glitch/tests/security/swarm/files/non_official_image.yml create mode 100644 glitch/tests/security/swarm/files/priv_container.yml create mode 100644 glitch/tests/security/swarm/files/susp_comment.yml create mode 100644 glitch/tests/security/swarm/files/template_no_smells.yml create mode 100644 glitch/tests/security/swarm/files/weak_crypt/weak_crypt_array.yml create mode 100644 glitch/tests/security/swarm/files/weak_crypt/weak_crypt_array_exec_form.yml create mode 100644 glitch/tests/security/swarm/files/weak_crypt/weak_crypt_hash.yml create mode 100644 glitch/tests/security/swarm/files/weak_crypt/weak_crypt_string.yml create mode 100644 glitch/tests/security/swarm/test_security.py diff --git a/glitch/tests/security/swarm/files/admin.yml b/glitch/tests/security/swarm/files/admin.yml new file mode 100644 index 00000000..397fd54a --- /dev/null +++ b/glitch/tests/security/swarm/files/admin.yml @@ -0,0 +1,12 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + user: root + logging: + driver: elastic/elastic-logging-plugin + + diff --git a/glitch/tests/security/swarm/files/container_image_tag_smells/no_tag_and_digest.yml b/glitch/tests/security/swarm/files/container_image_tag_smells/no_tag_and_digest.yml new file mode 100644 index 00000000..e28ee8ef --- /dev/null +++ b/glitch/tests/security/swarm/files/container_image_tag_smells/no_tag_and_digest.yml @@ -0,0 +1,9 @@ +services: + service_example: + image: nginx@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/container_image_tag_smells/no_tag_no_digest.yml b/glitch/tests/security/swarm/files/container_image_tag_smells/no_tag_no_digest.yml new file mode 100644 index 00000000..d3ccf6d9 --- /dev/null +++ b/glitch/tests/security/swarm/files/container_image_tag_smells/no_tag_no_digest.yml @@ -0,0 +1,9 @@ +services: + service_example: + image: nginx + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/container_image_tag_smells/normal_tag_and_digest.yml b/glitch/tests/security/swarm/files/container_image_tag_smells/normal_tag_and_digest.yml new file mode 100644 index 00000000..5a3d7410 --- /dev/null +++ b/glitch/tests/security/swarm/files/container_image_tag_smells/normal_tag_and_digest.yml @@ -0,0 +1,9 @@ +services: + service_example: + image: nginx:1.28.0-alpine@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/container_image_tag_smells/normal_tag_and_no_digest.yml b/glitch/tests/security/swarm/files/container_image_tag_smells/normal_tag_and_no_digest.yml new file mode 100644 index 00000000..ba84a692 --- /dev/null +++ b/glitch/tests/security/swarm/files/container_image_tag_smells/normal_tag_and_no_digest.yml @@ -0,0 +1,9 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/container_image_tag_smells/unstable_tag_and_digest.yml b/glitch/tests/security/swarm/files/container_image_tag_smells/unstable_tag_and_digest.yml new file mode 100644 index 00000000..a7d9ac91 --- /dev/null +++ b/glitch/tests/security/swarm/files/container_image_tag_smells/unstable_tag_and_digest.yml @@ -0,0 +1,9 @@ +services: + service_example: + image: nginx:mainline@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/container_image_tag_smells/unstable_tag_no_digest.yml b/glitch/tests/security/swarm/files/container_image_tag_smells/unstable_tag_no_digest.yml new file mode 100644 index 00000000..379c9d29 --- /dev/null +++ b/glitch/tests/security/swarm/files/container_image_tag_smells/unstable_tag_no_digest.yml @@ -0,0 +1,9 @@ +services: + service_example: + image: nginx:latest + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/deprecated_docker_official_image.yml b/glitch/tests/security/swarm/files/deprecated_docker_official_image.yml new file mode 100644 index 00000000..26f4cf4c --- /dev/null +++ b/glitch/tests/security/swarm/files/deprecated_docker_official_image.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: django:example-version@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin + + diff --git a/glitch/tests/security/swarm/files/docker_socket_mounted.yml b/glitch/tests/security/swarm/files/docker_socket_mounted.yml new file mode 100644 index 00000000..f8a4d066 --- /dev/null +++ b/glitch/tests/security/swarm/files/docker_socket_mounted.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + volumes: + - /var/run/docker.sock:/var/run/docker.sock + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/hard_secr/hard_secr_empty_password.yml b/glitch/tests/security/swarm/files/hard_secr/hard_secr_empty_password.yml new file mode 100644 index 00000000..a2ba8b54 --- /dev/null +++ b/glitch/tests/security/swarm/files/hard_secr/hard_secr_empty_password.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + environment: + - SOME_PASSWORD="" + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/hard_secr/hard_secr_password.yml b/glitch/tests/security/swarm/files/hard_secr/hard_secr_password.yml new file mode 100644 index 00000000..3a3199b5 --- /dev/null +++ b/glitch/tests/security/swarm/files/hard_secr/hard_secr_password.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + environment: + - SOME_PASSWORD=verysecure + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/hard_secr/hard_secr_secret.yml b/glitch/tests/security/swarm/files/hard_secr/hard_secr_secret.yml new file mode 100644 index 00000000..680f435e --- /dev/null +++ b/glitch/tests/security/swarm/files/hard_secr/hard_secr_secret.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + environment: + - SOME_SECRET=verysecure + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/hard_secr/hard_secr_user.yml b/glitch/tests/security/swarm/files/hard_secr/hard_secr_user.yml new file mode 100644 index 00000000..2f1315b7 --- /dev/null +++ b/glitch/tests/security/swarm/files/hard_secr/hard_secr_user.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + environment: + - SOME_USER=someone + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/https_tls.yml b/glitch/tests/security/swarm/files/https_tls.yml new file mode 100644 index 00000000..ce75935f --- /dev/null +++ b/glitch/tests/security/swarm/files/https_tls.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + environment: + - SOME_ENDPOINT_MAYBE=http://1.1.1.1 + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_array.yml b/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_array.yml new file mode 100644 index 00000000..90827903 --- /dev/null +++ b/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_array.yml @@ -0,0 +1,13 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + entrypoint: + - "some_command" + - "--some_flag" + - "0.0.0.0" + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_exec_form.yml b/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_exec_form.yml new file mode 100644 index 00000000..ba3473f5 --- /dev/null +++ b/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_exec_form.yml @@ -0,0 +1,10 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + command: ["example_command","--listen","0.0.0.0"] + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_hash.yml b/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_hash.yml new file mode 100644 index 00000000..89298f5a --- /dev/null +++ b/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_hash.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + environment: + - ENDPOINT=0.0.0.0 + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_string.yml b/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_string.yml new file mode 100644 index 00000000..522cf0f1 --- /dev/null +++ b/glitch/tests/security/swarm/files/invalid_bind/invalid_bind_string.yml @@ -0,0 +1,10 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + command: "example_command --listen 0.0.0.0" + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/missing_healthchecks.yml b/glitch/tests/security/swarm/files/missing_healthchecks.yml new file mode 100644 index 00000000..b65607e0 --- /dev/null +++ b/glitch/tests/security/swarm/files/missing_healthchecks.yml @@ -0,0 +1,5 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/no_log_aggregation.yml b/glitch/tests/security/swarm/files/no_log_aggregation.yml new file mode 100644 index 00000000..f063c68c --- /dev/null +++ b/glitch/tests/security/swarm/files/no_log_aggregation.yml @@ -0,0 +1,9 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + + diff --git a/glitch/tests/security/swarm/files/non_official_image.yml b/glitch/tests/security/swarm/files/non_official_image.yml new file mode 100644 index 00000000..37a616d3 --- /dev/null +++ b/glitch/tests/security/swarm/files/non_official_image.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: example_dockerhub_namespace/nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin + + diff --git a/glitch/tests/security/swarm/files/priv_container.yml b/glitch/tests/security/swarm/files/priv_container.yml new file mode 100644 index 00000000..cc3b1c27 --- /dev/null +++ b/glitch/tests/security/swarm/files/priv_container.yml @@ -0,0 +1,10 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + privileged: true + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/susp_comment.yml b/glitch/tests/security/swarm/files/susp_comment.yml new file mode 100644 index 00000000..5893a533 --- /dev/null +++ b/glitch/tests/security/swarm/files/susp_comment.yml @@ -0,0 +1,11 @@ +services: + service_example: #FIXME TEST BUG DONT FIX + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin + + diff --git a/glitch/tests/security/swarm/files/template_no_smells.yml b/glitch/tests/security/swarm/files/template_no_smells.yml new file mode 100644 index 00000000..8fa23e5d --- /dev/null +++ b/glitch/tests/security/swarm/files/template_no_smells.yml @@ -0,0 +1,9 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_array.yml b/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_array.yml new file mode 100644 index 00000000..a5ee6d14 --- /dev/null +++ b/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_array.yml @@ -0,0 +1,13 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + entrypoint: + - "/bin/sh" + - "-c" + - "wget -O - https://example.com/wow | md5sum > wow.txt" + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_array_exec_form.yml b/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_array_exec_form.yml new file mode 100644 index 00000000..289f393c --- /dev/null +++ b/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_array_exec_form.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + command: + ["/bin/sh", "-c", "wget -O - https://example.com/wow | md5sum > wow.txt"] + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_hash.yml b/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_hash.yml new file mode 100644 index 00000000..1a49f97c --- /dev/null +++ b/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_hash.yml @@ -0,0 +1,11 @@ +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + environment: + - USE_SECURE_HASH=md5 + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_string.yml b/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_string.yml new file mode 100644 index 00000000..1455806e --- /dev/null +++ b/glitch/tests/security/swarm/files/weak_crypt/weak_crypt_string.yml @@ -0,0 +1,12 @@ +version: 3.7 + +services: + service_example: + image: nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5 + healthcheck: + test: "curl --fail http://localhost:80/ || exit 1" + interval: 30s + retries: 5s + command: '/bin/sh -c "wget -O - https://example.com/wow | md5sum > wow.txt"' + logging: + driver: elastic/elastic-logging-plugin diff --git a/glitch/tests/security/swarm/test_security.py b/glitch/tests/security/swarm/test_security.py new file mode 100644 index 00000000..ade87752 --- /dev/null +++ b/glitch/tests/security/swarm/test_security.py @@ -0,0 +1,219 @@ +import os +import unittest + +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.parsers.swarm import SwarmParser +from glitch.repr.inter import UnitBlockType +from glitch.tech import Tech +from typing import List + + +class TestSecurity(unittest.TestCase): + def __help_test( + self, path: str, n_errors: int, codes: List[str], lines: List[int] + ) -> None: + parser = SwarmParser() + inter = parser.parse(path, UnitBlockType.script, False) + analysis = SecurityVisitor(Tech.swarm) + analysis.config("configs/default.ini") + errors = list( + filter( + lambda e: e.code.startswith("sec_") or e.code.startswith("arc_"), + set(analysis.check(inter)), + ) # type: ignore + ) + errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) + self.assertEqual(len(errors), n_errors) + for i in range(n_errors): + self.assertEqual(errors[i].code, codes[i]) + self.assertEqual(errors[i].line, lines[i]) + + def tearDown(self) -> None: + super().tearDown() + if os.path.exists("docker-compose.yml"): + os.remove("docker-compose.yml") + + def test_swarm_admin(self) -> None: + self.__help_test( + "tests/security/swarm/files/admin.yml", + 3, + ["sec_def_admin", "sec_hard_secr", "sec_hard_user"], + [8, 8, 8], + ) + + def test_swarm_empty(self) -> None: + self.__help_test( + "tests/security/swarm/files/hard_secr/hard_secr_empty_password.yml", + 1, + ["sec_empty_pass"], + [8], + ) + + def test_swarm_hard_secret(self) -> None: + self.__help_test( + "tests/security/swarm/files/hard_secr/hard_secr_password.yml", + 2, + ["sec_hard_pass", "sec_hard_secr"], + [8, 8], + ) + self.__help_test( + "tests/security/swarm/files/hard_secr/hard_secr_secret.yml", + 1, + ["sec_hard_secr"], + [8], + ) + self.__help_test( + "tests/security/swarm/files/hard_secr/hard_secr_user.yml", + 2, + ["sec_hard_secr", "sec_hard_user"], + [8, 8], + ) + + def test_swarm_http(self) -> None: + self.__help_test( + "tests/security/swarm/files/https_tls.yml", 1, ["sec_https"], [9] + ) + + def test_swarm_inv_bind(self) -> None: + self.__help_test( + "tests/security/swarm/files/invalid_bind/invalid_bind_array.yml", + 1, + ["sec_invalid_bind"], + [8], + ) + self.__help_test( + "tests/security/swarm/files/invalid_bind/invalid_bind_exec_form.yml", + 1, + ["sec_invalid_bind"], + [8], + ) + self.__help_test( + "tests/security/swarm/files/invalid_bind/invalid_bind_hash.yml", + 1, + ["sec_invalid_bind"], + [8], + ) + self.__help_test( + "tests/security/swarm/files/invalid_bind/invalid_bind_string.yml", + 1, + ["sec_invalid_bind"], + [8], + ) + + def test_swarm_non_official_image(self) -> None: + self.__help_test( + "tests/security/swarm/files/non_official_image.yml", + 1, + ["sec_non_official_image"], + [3], + ) + + def test_swarm_susp(self) -> None: + self.__help_test( + "tests/security/swarm/files/susp_comment.yml", 1, ["sec_susp_comm"], [2] + ) + + def test_swarm_weak_crypt(self) -> None: + self.__help_test( + "tests/security/swarm/files/weak_crypt/weak_crypt_string.yml", + 1, + ["sec_weak_crypt"], + [10], + ) + self.__help_test( + "tests/security/swarm/files/weak_crypt/weak_crypt_hash.yml", + 1, + ["sec_weak_crypt"], + [9], + ) + self.__help_test( + "tests/security/swarm/files/weak_crypt/weak_crypt_array.yml", + 1, + ["sec_weak_crypt"], + [11], + ) + self.__help_test( + "tests/security/swarm/files/weak_crypt/weak_crypt_array_exec_form.yml", + 1, + ["sec_weak_crypt"], + [9], + ) + + def test_swarm_container_image_tag_smells(self) -> None: + self.__help_test( + "tests/security/swarm/files/container_image_tag_smells/normal_tag_and_digest.yml", + 0, + [], + [], + ) + self.__help_test( + "tests/security/swarm/files/container_image_tag_smells/normal_tag_and_no_digest.yml", + 1, + ["sec_image_integrity"], + [3], + ) + self.__help_test( + "tests/security/swarm/files/container_image_tag_smells/no_tag_and_digest.yml", + 0, + [], + [], + ) + self.__help_test( + "tests/security/swarm/files/container_image_tag_smells/no_tag_no_digest.yml", + 1, + ["sec_no_image_tag"], + [3], + ) + + self.__help_test( + "tests/security/swarm/files/container_image_tag_smells/unstable_tag_and_digest.yml", + 1, + ["sec_unstable_tag"], + [3], + ) + self.__help_test( + "tests/security/swarm/files/container_image_tag_smells/unstable_tag_no_digest.yml", + 2, + ["sec_image_integrity", "sec_unstable_tag"], + [3, 3], + ) + + def test_swarm_deprecated_official_img(self) -> None: + self.__help_test( + "tests/security/swarm/files/deprecated_docker_official_image.yml", + 1, + ["sec_depr_off_imgs"], + [3], + ) + + def test_swarm_missing_healthchecks(self) -> None: + self.__help_test( + "tests/security/swarm/files/missing_healthchecks.yml", + 1, + ["arc_missing_healthchecks"], + [2], + ) + + def test_swarm_privileged_container(self) -> None: + self.__help_test( + "tests/security/swarm/files/priv_container.yml", + 1, + ["sec_privileged_containers"], + [8], + ) + + def test_swarm_docker_socket_mounted(self) -> None: + self.__help_test( + "tests/security/swarm/files/docker_socket_mounted.yml", + 1, + ["sec_mounted_docker_socket"], + [9], + ) + + def test_swarm_no_log_aggregation(self) -> None: + self.__help_test( + "tests/security/swarm/files/no_log_aggregation.yml", + 1, + ["arc_no_logging"], + [2], + ) From 578bb59d060ac5ee0e61b2844403b63ec25b70e0 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 28 Aug 2025 18:53:03 +0100 Subject: [PATCH 53/65] add missing __init__.py needed for test detection --- glitch/tests/security/swarm/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 glitch/tests/security/swarm/__init__.py diff --git a/glitch/tests/security/swarm/__init__.py b/glitch/tests/security/swarm/__init__.py new file mode 100644 index 00000000..e69de29b From 9d12ba3d804189016c5a307cd9e413ea48bee697 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 29 Aug 2025 18:22:10 +0100 Subject: [PATCH 54/65] fix filenames of official docker images and deprecated official images in scraper script --- scripts/docker_images_scraper.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/docker_images_scraper.py b/scripts/docker_images_scraper.py index ce1057e6..abb31345 100644 --- a/scripts/docker_images_scraper.py +++ b/scripts/docker_images_scraper.py @@ -20,9 +20,11 @@ if current < total: res = ses.get(next_url + f"&from={current}", headers=headers).json() -with open("official_images", "w") as f: +with open("official_docker_images", "w") as f: + images_list.sort() f.write("\n".join(images_list)) -with open("official_deprecated_images", "w") as f: +with open("deprecated_official_docker_images", "w") as f: + deprecated.sort() f.write("\n".join(deprecated)) From e6150d4495ba672e6ef27e12a246f2b8ebd37794 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 29 Aug 2025 18:22:55 +0100 Subject: [PATCH 55/65] update official docker images list --- .../files/deprecated_official_docker_images | 58 ++-- glitch/files/official_docker_images | 277 ++++++++---------- 2 files changed, 151 insertions(+), 184 deletions(-) diff --git a/glitch/files/deprecated_official_docker_images b/glitch/files/deprecated_official_docker_images index f0dd4413..729d8ade 100644 --- a/glitch/files/deprecated_official_docker_images +++ b/glitch/files/deprecated_official_docker_images @@ -1,36 +1,38 @@ +adoptopenjdk +celery centos -jenkins +clearlinux +clefos +crux +django +docker-dev +emqx +euleros +express-gateway +fsharp +glassfish +hipache +iojs java -sentry +jenkins +jobber +kaazing-gateway +known +mono nats-streaming -adoptopenjdk -swarm +notary +nuxeo +opensuse owncloud -mono -django +php-zendserver piwik -iojs -opensuse +plone rails -ubuntu-debootstrap -clearlinux -nuxeo -php-zendserver -emqx -celery -thrift -fsharp rapidoid -docker-dev -express-gateway -kaazing-gateway -ubuntu-upstart -known -glassfish -crux -clefos -euleros -jobber -sourcemage +sentry sl -hipache \ No newline at end of file +sourcemage +swarm +thrift +ubuntu-debootstrap +ubuntu-upstart \ No newline at end of file diff --git a/glitch/files/official_docker_images b/glitch/files/official_docker_images index d692e385..1789a557 100644 --- a/glitch/files/official_docker_images +++ b/glitch/files/official_docker_images @@ -1,175 +1,140 @@ +adminer +aerospike +almalinux alpine -busybox -nginx -ubuntu -python -redis -postgres -node -httpd -mongo -mysql -memcached -traefik -mariadb -docker -rabbitmq -hello-world -openjdk -golang -registry -wordpress -centos -debian -influxdb -consul -php -nextcloud -sonarqube -haproxy -ruby +alt +amazoncorretto amazonlinux -elasticsearch -tomcat -eclipse-mosquitto -maven -telegraf -vault -caddy -adminer +api-firewall +arangodb +archlinux +backdrop bash -ghost -kong -perl -neo4j -zookeeper +bonita buildpack-deps -mongo-express -gradle -logstash +busybox +caddy cassandra -couchdb -nats chronograf +cirros +clickhouse +clojure +composer +consul +convertigo +couchbase +couchdb +crate +dart +debian +docker drupal -jenkins -kibana -java -solr -percona -teamspeak -sentry -matomo +eclipse-mosquitto +eclipse-temurin +eggdrop +elasticsearch +elixir +erlang fedora -composer -nats-streaming -adoptopenjdk flink -couchbase -swarm -joomla +fluentd +friendica +gazebo +gcc +geonetwork +ghost +golang +gradle groovy -rethinkdb -rocket.chat -redmine -owncloud -rust -kapacitor -erlang -phpmyadmin +haproxy +haskell +haxe +hello-seattle +hello-world +hitch +hola-mundo +httpd +hylang +ibm-semeru-runtimes +ibmjava +influxdb +irssi +jetty +joomla jruby -elixir -amazoncorretto +julia +kapacitor +kibana +kong +krakend +lightstreamer +liquibase +logstash +mageia +mariadb +matomo +maven mediawiki -mono -pypy -jetty -clojure -arangodb +memcached +mongo +mongo-express +monica +mysql +nats +neo4j +neurodebian +nextcloud +nginx +node odoo -eclipse-temurin -xwiki +open-liberty +openjdk oraclelinux -znc -haxe +orientdb +percona +perl +photon +php +phpmyadmin +postfixadmin +postgres +pypy +python +r-base +rabbitmq +rakudo-star +redis +redmine +registry +rethinkdb +rocket.chat +rockylinux ros -hylang -websphere-liberty -django +ruby +rust sapmachine -gcc -archlinux +satosa +scratch +silverpeas +solr +sonarqube +spark +spiped +storm swift +swipl +teamspeak +telegraf +tomcat tomee -piwik -yourls -rockylinux -iojs -crate -aerospike -photon -orientdb -julia +traefik +ubuntu +unit varnish -ibmjava -open-liberty -bonita -monica -neurodebian -opensuse -fluentd -rails -ubuntu-debootstrap -storm -r-base -irssi -haskell -backdrop -clearlinux -plone -notary -cirros -lightstreamer -geonetwork -nuxeo -postfixadmin -gazebo -php-zendserver -convertigo -friendica -hello-seattle -celery -spiped -swipl -fsharp -eggdrop -thrift -rapidoid -almalinux -docker-dev -rakudo-star -express-gateway -Kaazing Gateway -ibm-semeru-runtimes -ubuntu-upstart -silverpeas -mageia -hola-mundo -known -glassfish -dart -crux -euleros -jobber -sourcemage -clefos -alt -sl -hipache -hitch -scratch -satosa -emqx -api-firewall -cheers -dcl2020 \ No newline at end of file +vault +websphere-liberty +wordpress +xwiki +yourls +znc +zookeeper \ No newline at end of file From cb0e621ee5b89056c00cb8630aa58b5505b6f10b Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 29 Aug 2025 18:24:14 +0100 Subject: [PATCH 56/65] remove unnecessary method --- glitch/tests/security/swarm/test_security.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/glitch/tests/security/swarm/test_security.py b/glitch/tests/security/swarm/test_security.py index ade87752..252e0977 100644 --- a/glitch/tests/security/swarm/test_security.py +++ b/glitch/tests/security/swarm/test_security.py @@ -28,11 +28,6 @@ def __help_test( self.assertEqual(errors[i].code, codes[i]) self.assertEqual(errors[i].line, lines[i]) - def tearDown(self) -> None: - super().tearDown() - if os.path.exists("docker-compose.yml"): - os.remove("docker-compose.yml") - def test_swarm_admin(self) -> None: self.__help_test( "tests/security/swarm/files/admin.yml", From 200e9dc12f6b4801a7027169a7f5439d690fc2eb Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Fri, 29 Aug 2025 23:50:23 +0100 Subject: [PATCH 57/65] remove unused import --- glitch/tests/security/swarm/test_security.py | 1 - 1 file changed, 1 deletion(-) diff --git a/glitch/tests/security/swarm/test_security.py b/glitch/tests/security/swarm/test_security.py index 252e0977..bbe55f36 100644 --- a/glitch/tests/security/swarm/test_security.py +++ b/glitch/tests/security/swarm/test_security.py @@ -1,4 +1,3 @@ -import os import unittest from glitch.analysis.security.visitor import SecurityVisitor From febd84dcb338a49cb53132273c5963287d9576c4 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:16:45 +0100 Subject: [PATCH 58/65] fix: consider both list of official images. The newer list of official images (maintained) doesn't include the ones in the deprecated official images one --- glitch/analysis/security/non_official_container_images.py | 3 ++- glitch/analysis/security/visitor.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/glitch/analysis/security/non_official_container_images.py b/glitch/analysis/security/non_official_container_images.py index 20f18704..b9d297dc 100644 --- a/glitch/analysis/security/non_official_container_images.py +++ b/glitch/analysis/security/non_official_container_images.py @@ -29,7 +29,8 @@ def check(self, element: CodeElement, file: str) -> List[Error]: img_name, _, _ = parse_container_image_name(image) if img_name != "": - for off_img in SecurityVisitor.DOCKER_OFFICIAL_IMAGES: + all_official_images:List[str] = SecurityVisitor.DOCKER_OFFICIAL_IMAGES + SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES + for off_img in all_official_images: off_img_dockerio = f"docker.io/library/{off_img}" off_img_library = f"library/{off_img}" off_img_complete_link = f"registry.hub.docker.com/library/{off_img}" diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index eb338964..57d78914 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -31,7 +31,8 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ): return [] image = element.name.split(":") - if image[0] not in SecurityVisitor.DOCKER_OFFICIAL_IMAGES: + all_official_imgs = SecurityVisitor.DOCKER_OFFICIAL_IMAGES + SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES + if image[0] not in all_official_imgs: return [Error("sec_non_official_image", element, file, repr(element))] return [] From 374c39fe36258001d92a6310b7adbeabc2f7b60e Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:19:02 +0100 Subject: [PATCH 59/65] format visitor.py --- glitch/analysis/security/visitor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index 57d78914..bea98230 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -31,7 +31,10 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ): return [] image = element.name.split(":") - all_official_imgs = SecurityVisitor.DOCKER_OFFICIAL_IMAGES + SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES + all_official_imgs = ( + SecurityVisitor.DOCKER_OFFICIAL_IMAGES + + SecurityVisitor.DEPRECATED_OFFICIAL_DOCKER_IMAGES + ) if image[0] not in all_official_imgs: return [Error("sec_non_official_image", element, file, repr(element))] return [] From 8a5f19ecc51f0ad44b02cabda165a392ad5eb474 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:21:33 +0100 Subject: [PATCH 60/65] fix missing healthchecks smell and wobbly service interaction on nomad --- .../analysis/security/missing_healthchecks.py | 180 ++++++++++++------ 1 file changed, 122 insertions(+), 58 deletions(-) diff --git a/glitch/analysis/security/missing_healthchecks.py b/glitch/analysis/security/missing_healthchecks.py index 35e46011..7be36477 100644 --- a/glitch/analysis/security/missing_healthchecks.py +++ b/glitch/analysis/security/missing_healthchecks.py @@ -1,22 +1,117 @@ from glitch.analysis.rules import Error from glitch.analysis.security.smell_checker import SecuritySmellChecker -from glitch.repr.inter import CodeElement, Hash, AtomicUnit, Array, Boolean -from typing import List +from glitch.repr.inter import ( + CodeElement, + Hash, + AtomicUnit, + Array, + UnitBlock, + String, + UnitBlockType, +) +from typing import List, Dict, Any class MissingHealthchecks(SecuritySmellChecker): - #NOTE: This class checks for Missing Healthchecks smell in Nomad and Swarm + # NOTE: This class checks for Missing Healthchecks smell in Nomad and Swarm # But it is checking for the WobblyServiceInteraction in Nomad def check(self, element: CodeElement, file: str) -> List[Error]: errors: List[Error] = [] - + + # nomad group tasks + if isinstance(element, UnitBlock) and element.type == UnitBlockType.block: + services_info: List[Dict[str, bool | str | None]] = [] + + # getting the services and consul sidecars at group level + for att in element.attributes: + if att.name == "service" and isinstance(att.value, Hash): + serv_inf: Dict[str, Any] = { + "has_healthcheck": False, + "has_sidecar": False, + "port": None, + } + + for k, v in att.value.value.items(): + if k.value == "check" and isinstance(v, Hash): + serv_inf["has_healthcheck"] = True + + elif k.value == "connect" and isinstance(v, Hash): + for _k, _v in v.value.items(): + # Checks for use of Consul service mesh, sidecar proxy that + # provides Timeouts and Circuit Breaker mechanisms + # that avoid the Wobbly Service Interaction smell + # the smell is detectable in Nomad if this is not present + if _k.value == "sidecar_service" and isinstance( + _v, Hash + ): + serv_inf["has_sidecar"] = True + break + elif k.value == "port" and isinstance(v, String): + serv_inf["port"] = v.value + services_info.append(serv_inf) + + # checking each task + for au in element.atomic_units: + has_healthcheck = False + has_sidecar = False + + for att in au.attributes: + if att.name == "config" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if k.value == "ports" and isinstance(v, Array): + str_ports: List[str] = [x.value for x in v.value] + for service in services_info: + if ( + service["port"] is not None + and service["port"] in str_ports + ): + if service["has_sidecar"]: + has_sidecar = True + if service["has_healthcheck"]: + has_healthcheck = True + + if has_healthcheck and has_sidecar: + break + + if att.name == "service" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if k.value == "check": + has_healthcheck = True + break + + if not has_sidecar: + errors.append( + Error("arc_wobbly_service_interaction", au, file, repr(au)) + ) + if not has_healthcheck: + errors.append(Error("arc_missing_healthchecks", au, file, repr(au))) + # nomad tasks not in groups + if ( + isinstance(element, UnitBlock) + and element.type == UnitBlockType.script + and element.name == "job" + ): + for au in element.atomic_units: + has_healthcheck = False + for att in au.attributes: + if att.name == "service" and isinstance(att.value, Hash): + for k, v in att.value.value.items(): + if k.value == "check": + has_healthcheck = True + break + if not has_healthcheck: + errors.append(Error("arc_missing_healthchecks", au, file, repr(au))) + + # consul sidecars are only available at group level + errors.append( + Error("arc_wobbly_service_interaction", au, file, repr(au)) + ) + + # swarm if isinstance(element, AtomicUnit): - au = element found_healthcheck = False - has_disable_nomad = False - found_sidecar = False - - for att in au.attributes: + + for att in element.attributes: if found_healthcheck: break if att.name == "healthcheck": @@ -26,7 +121,14 @@ def check(self, element: CodeElement, file: str) -> List[Error]: if k.value == "disable" and ( v.value or v.value.lower() == "true" ): - errors.append(Error("arc_missing_healthchecks", au, file, repr(au))) + errors.append( + Error( + "arc_missing_healthchecks", + element, + file, + repr(element), + ) + ) break elif k.value == "test": if isinstance(v.value, Array): @@ -35,57 +137,19 @@ def check(self, element: CodeElement, file: str) -> List[Error]: and v.value.value[0] == "NONE" ): errors.append( - Error("arc_missing_healthchecks", au, file, repr(au)) + Error( + "arc_missing_healthchecks", + element, + file, + repr(element), + ) ) break break - elif att.name == "config" and isinstance(att.value, Hash): - for k, v in att.value.value.items(): - if k.value == "healthchecks" and isinstance(v, Hash): - for _k, _v in v.value.items(): - if ( - _k.value == "disable" - and isinstance(_v, Boolean) - and _v.value - ): - has_disable_nomad = True - break - elif has_disable_nomad: - break - if has_disable_nomad: - break - - elif att.name == "service" and isinstance(att.value, Hash): - for k, v in att.value.value.items(): - if k.value == "check" and isinstance(v, Hash): - found_healthcheck = True - break - elif k.value == "connect" and isinstance(v, Hash): - for _k, _v in v.value.items(): - if _k.value == "sidecar_service" and isinstance( - _v, Hash - ): - # Checks for use of Consul service mesh, sidecar proxy that - # provides Timeouts and Circuit Breaker mechanisms that avoid the Wobbly Service Interaction smell - # the smell is detectable in Nomad if this is not present - found_sidecar = True - break - if found_healthcheck: - break - if not found_healthcheck: - errors.append(Error("arc_missing_healthchecks", au, file, repr(au))) - - if ( - att.name in ["config", "service"] - and isinstance(att.value, Hash) - and not found_healthcheck - and has_disable_nomad - ): - errors.append(Error("arc_missing_healthchecks", au, file, repr(au))) - if element.type == "service" and not found_healthcheck: - errors.append(Error("arc_missing_healthchecks",au,file,repr(au))) - if element.type.startswith("task") and not found_sidecar: - errors.append(Error("arc_wobbly_service_interaction",au,file,repr(au))) + errors.append( + Error("arc_missing_healthchecks", element, file, repr(element)) + ) + return errors From fa902f51e1f199447080b5a945b1c66b182123ae Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:28:33 +0100 Subject: [PATCH 61/65] add nomad tests --- glitch/tests/security/nomad/__init__.py | 0 glitch/tests/security/nomad/files/admin.nomad | 40 ++++ .../no_tag_and_digest.nomad | 40 ++++ .../no_tag_no_digest.nomad | 40 ++++ .../normal_tag_and_digest.nomad | 40 ++++ .../normal_tag_and_no_digest.nomad | 40 ++++ .../unstable_tag_and_digest.nomad | 40 ++++ .../unstable_tag_no_digest.nomad | 40 ++++ .../deprecated_docker_official_image.nomad | 41 ++++ .../nomad/files/docker_socket_mounted.nomad | 42 ++++ .../hard_secr/hard_secr_empty_password.nomad | 46 ++++ .../files/hard_secr/hard_secr_password.nomad | 46 ++++ .../files/hard_secr/hard_secr_secret.nomad | 46 ++++ .../files/hard_secr/hard_secr_user.nomad | 46 ++++ .../security/nomad/files/https_tls.nomad | 45 ++++ .../security/nomad/files/invalid_bind.nomad | 45 ++++ .../nomad/files/missing_healthchecks.nomad | 34 +++ ...ultiple_services_per_deployment_unit.nomad | 81 +++++++ .../security/nomad/files/no_api_gateway.nomad | 41 ++++ .../nomad/files/no_log_aggregation.nomad | 37 ++++ .../nomad/files/non_official_image.nomad | 40 ++++ .../security/nomad/files/priv_container.nomad | 40 ++++ .../security/nomad/files/susp_comment.nomad | 40 ++++ .../nomad/files/template_no_smells.nomad | 45 ++++ .../security/nomad/files/weak_crypt.nomad | 40 ++++ .../files/wobbly_service_interaction.nomad | 37 ++++ glitch/tests/security/nomad/test_security.py | 206 ++++++++++++++++++ 27 files changed, 1278 insertions(+) create mode 100644 glitch/tests/security/nomad/__init__.py create mode 100644 glitch/tests/security/nomad/files/admin.nomad create mode 100644 glitch/tests/security/nomad/files/container_image_tag_smells/no_tag_and_digest.nomad create mode 100644 glitch/tests/security/nomad/files/container_image_tag_smells/no_tag_no_digest.nomad create mode 100644 glitch/tests/security/nomad/files/container_image_tag_smells/normal_tag_and_digest.nomad create mode 100644 glitch/tests/security/nomad/files/container_image_tag_smells/normal_tag_and_no_digest.nomad create mode 100644 glitch/tests/security/nomad/files/container_image_tag_smells/unstable_tag_and_digest.nomad create mode 100644 glitch/tests/security/nomad/files/container_image_tag_smells/unstable_tag_no_digest.nomad create mode 100644 glitch/tests/security/nomad/files/deprecated_docker_official_image.nomad create mode 100644 glitch/tests/security/nomad/files/docker_socket_mounted.nomad create mode 100644 glitch/tests/security/nomad/files/hard_secr/hard_secr_empty_password.nomad create mode 100644 glitch/tests/security/nomad/files/hard_secr/hard_secr_password.nomad create mode 100644 glitch/tests/security/nomad/files/hard_secr/hard_secr_secret.nomad create mode 100644 glitch/tests/security/nomad/files/hard_secr/hard_secr_user.nomad create mode 100644 glitch/tests/security/nomad/files/https_tls.nomad create mode 100644 glitch/tests/security/nomad/files/invalid_bind.nomad create mode 100644 glitch/tests/security/nomad/files/missing_healthchecks.nomad create mode 100644 glitch/tests/security/nomad/files/multiple_services_per_deployment_unit.nomad create mode 100644 glitch/tests/security/nomad/files/no_api_gateway.nomad create mode 100644 glitch/tests/security/nomad/files/no_log_aggregation.nomad create mode 100644 glitch/tests/security/nomad/files/non_official_image.nomad create mode 100644 glitch/tests/security/nomad/files/priv_container.nomad create mode 100644 glitch/tests/security/nomad/files/susp_comment.nomad create mode 100644 glitch/tests/security/nomad/files/template_no_smells.nomad create mode 100644 glitch/tests/security/nomad/files/weak_crypt.nomad create mode 100644 glitch/tests/security/nomad/files/wobbly_service_interaction.nomad create mode 100644 glitch/tests/security/nomad/test_security.py diff --git a/glitch/tests/security/nomad/__init__.py b/glitch/tests/security/nomad/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/glitch/tests/security/nomad/files/admin.nomad b/glitch/tests/security/nomad/files/admin.nomad new file mode 100644 index 00000000..d19b87ee --- /dev/null +++ b/glitch/tests/security/nomad/files/admin.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + user = "root" + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/container_image_tag_smells/no_tag_and_digest.nomad b/glitch/tests/security/nomad/files/container_image_tag_smells/no_tag_and_digest.nomad new file mode 100644 index 00000000..dc607367 --- /dev/null +++ b/glitch/tests/security/nomad/files/container_image_tag_smells/no_tag_and_digest.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/container_image_tag_smells/no_tag_no_digest.nomad b/glitch/tests/security/nomad/files/container_image_tag_smells/no_tag_no_digest.nomad new file mode 100644 index 00000000..19d537aa --- /dev/null +++ b/glitch/tests/security/nomad/files/container_image_tag_smells/no_tag_no_digest.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/container_image_tag_smells/normal_tag_and_digest.nomad b/glitch/tests/security/nomad/files/container_image_tag_smells/normal_tag_and_digest.nomad new file mode 100644 index 00000000..2611bd69 --- /dev/null +++ b/glitch/tests/security/nomad/files/container_image_tag_smells/normal_tag_and_digest.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/container_image_tag_smells/normal_tag_and_no_digest.nomad b/glitch/tests/security/nomad/files/container_image_tag_smells/normal_tag_and_no_digest.nomad new file mode 100644 index 00000000..773acd83 --- /dev/null +++ b/glitch/tests/security/nomad/files/container_image_tag_smells/normal_tag_and_no_digest.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/container_image_tag_smells/unstable_tag_and_digest.nomad b/glitch/tests/security/nomad/files/container_image_tag_smells/unstable_tag_and_digest.nomad new file mode 100644 index 00000000..e50245d4 --- /dev/null +++ b/glitch/tests/security/nomad/files/container_image_tag_smells/unstable_tag_and_digest.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:mainline@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/container_image_tag_smells/unstable_tag_no_digest.nomad b/glitch/tests/security/nomad/files/container_image_tag_smells/unstable_tag_no_digest.nomad new file mode 100644 index 00000000..23d430c2 --- /dev/null +++ b/glitch/tests/security/nomad/files/container_image_tag_smells/unstable_tag_no_digest.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:mainline" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/deprecated_docker_official_image.nomad b/glitch/tests/security/nomad/files/deprecated_docker_official_image.nomad new file mode 100644 index 00000000..352b09ed --- /dev/null +++ b/glitch/tests/security/nomad/files/deprecated_docker_official_image.nomad @@ -0,0 +1,41 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "rails:1.2.3@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/docker_socket_mounted.nomad b/glitch/tests/security/nomad/files/docker_socket_mounted.nomad new file mode 100644 index 00000000..d80f02fa --- /dev/null +++ b/glitch/tests/security/nomad/files/docker_socket_mounted.nomad @@ -0,0 +1,42 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + volumes = [ + "/var/run/docker.sock:/var/run/docker.sock", + ] + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/hard_secr/hard_secr_empty_password.nomad b/glitch/tests/security/nomad/files/hard_secr/hard_secr_empty_password.nomad new file mode 100644 index 00000000..c5e0e294 --- /dev/null +++ b/glitch/tests/security/nomad/files/hard_secr/hard_secr_empty_password.nomad @@ -0,0 +1,46 @@ + +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + env { + DEV_PASSWORD = "" + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/hard_secr/hard_secr_password.nomad b/glitch/tests/security/nomad/files/hard_secr/hard_secr_password.nomad new file mode 100644 index 00000000..2fc4f089 --- /dev/null +++ b/glitch/tests/security/nomad/files/hard_secr/hard_secr_password.nomad @@ -0,0 +1,46 @@ + +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + env { + DEV_PASSWORD = "hunter2" + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/hard_secr/hard_secr_secret.nomad b/glitch/tests/security/nomad/files/hard_secr/hard_secr_secret.nomad new file mode 100644 index 00000000..7ea74713 --- /dev/null +++ b/glitch/tests/security/nomad/files/hard_secr/hard_secr_secret.nomad @@ -0,0 +1,46 @@ + +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + env { + DEV_SECRET = "iam_superman" + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/hard_secr/hard_secr_user.nomad b/glitch/tests/security/nomad/files/hard_secr/hard_secr_user.nomad new file mode 100644 index 00000000..677e422c --- /dev/null +++ b/glitch/tests/security/nomad/files/hard_secr/hard_secr_user.nomad @@ -0,0 +1,46 @@ + +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + env { + DEV_USER = "clark_kent" + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/https_tls.nomad b/glitch/tests/security/nomad/files/https_tls.nomad new file mode 100644 index 00000000..89f314d0 --- /dev/null +++ b/glitch/tests/security/nomad/files/https_tls.nomad @@ -0,0 +1,45 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + env { + SOME_ENDPOINT = "http://1.1.1.1" + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/invalid_bind.nomad b/glitch/tests/security/nomad/files/invalid_bind.nomad new file mode 100644 index 00000000..d38a6264 --- /dev/null +++ b/glitch/tests/security/nomad/files/invalid_bind.nomad @@ -0,0 +1,45 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + env { + SOME_ENVVAR = "0.0.0.0" + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/missing_healthchecks.nomad b/glitch/tests/security/nomad/files/missing_healthchecks.nomad new file mode 100644 index 00000000..de5ad052 --- /dev/null +++ b/glitch/tests/security/nomad/files/missing_healthchecks.nomad @@ -0,0 +1,34 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/multiple_services_per_deployment_unit.nomad b/glitch/tests/security/nomad/files/multiple_services_per_deployment_unit.nomad new file mode 100644 index 00000000..0937d987 --- /dev/null +++ b/glitch/tests/security/nomad/files/multiple_services_per_deployment_unit.nomad @@ -0,0 +1,81 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + port "check" { } + } + + service { + name = "server-proxy" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + service { + name = "service-api" + port = "check" + tags = ["check"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + + task "example-service" { + driver = "docker" + + config { + image = "someone/exampleapp:v234.12@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + ports = [ "check" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + + task "example-service2" { + driver = "docker" + + config { + image = "someone/exampleapp2:v234.12@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + ports = [ "check" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/no_api_gateway.nomad b/glitch/tests/security/nomad/files/no_api_gateway.nomad new file mode 100644 index 00000000..af9280a7 --- /dev/null +++ b/glitch/tests/security/nomad/files/no_api_gateway.nomad @@ -0,0 +1,41 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "host" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "ubuntu:24.04@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/no_log_aggregation.nomad b/glitch/tests/security/nomad/files/no_log_aggregation.nomad new file mode 100644 index 00000000..d8087081 --- /dev/null +++ b/glitch/tests/security/nomad/files/no_log_aggregation.nomad @@ -0,0 +1,37 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + } + } + } +} diff --git a/glitch/tests/security/nomad/files/non_official_image.nomad b/glitch/tests/security/nomad/files/non_official_image.nomad new file mode 100644 index 00000000..42d23af8 --- /dev/null +++ b/glitch/tests/security/nomad/files/non_official_image.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "some_random_namespace/nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/priv_container.nomad b/glitch/tests/security/nomad/files/priv_container.nomad new file mode 100644 index 00000000..efb75930 --- /dev/null +++ b/glitch/tests/security/nomad/files/priv_container.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + privileged = true + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/susp_comment.nomad b/glitch/tests/security/nomad/files/susp_comment.nomad new file mode 100644 index 00000000..b6a908a0 --- /dev/null +++ b/glitch/tests/security/nomad/files/susp_comment.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + # FIXME TEST FILE DONT REMOVE + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/template_no_smells.nomad b/glitch/tests/security/nomad/files/template_no_smells.nomad new file mode 100644 index 00000000..d1753e8d --- /dev/null +++ b/glitch/tests/security/nomad/files/template_no_smells.nomad @@ -0,0 +1,45 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "grpc_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + + env { + SOME_ENVVAR = "value" + } + + } + } +} diff --git a/glitch/tests/security/nomad/files/weak_crypt.nomad b/glitch/tests/security/nomad/files/weak_crypt.nomad new file mode 100644 index 00000000..3000d72f --- /dev/null +++ b/glitch/tests/security/nomad/files/weak_crypt.nomad @@ -0,0 +1,40 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + connect { + sidecar_service {} + } + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + command = "/bin/sh -c 'wget -O - https://example.com/wow | md5sum > wow.txt'" + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/files/wobbly_service_interaction.nomad b/glitch/tests/security/nomad/files/wobbly_service_interaction.nomad new file mode 100644 index 00000000..377136ef --- /dev/null +++ b/glitch/tests/security/nomad/files/wobbly_service_interaction.nomad @@ -0,0 +1,37 @@ +job "example" { + + group "example" { + count = 2 + + network { + mode = "bridge" + port "http" { } + } + + service { + name = "server-api" + port = "http" + tags = ["http"] + + check { + name = "http_probe" + type = "http" + interval = "10s" + timeout = "1s" + } + } + + task "example-api" { + driver = "docker" + + config { + image = "nginx:1.29.1-alpine3.22-perl@sha256:9322c38c12e68706f47d42b53622e1c52a351bd963574f4a157b3048d21772e5" + + ports = [ "http" ] + logging { + driver = "elastic/elastic-logging-plugin" + } + } + } + } +} diff --git a/glitch/tests/security/nomad/test_security.py b/glitch/tests/security/nomad/test_security.py new file mode 100644 index 00000000..147441e5 --- /dev/null +++ b/glitch/tests/security/nomad/test_security.py @@ -0,0 +1,206 @@ +import unittest + +from glitch.analysis.security.visitor import SecurityVisitor +from glitch.parsers.nomad import NomadParser +from glitch.repr.inter import UnitBlockType +from glitch.tech import Tech +from typing import List + + +class TestSecurity(unittest.TestCase): + def __help_test( + self, path: str, n_errors: int, codes: List[str], lines: List[int] + ) -> None: + parser = NomadParser() + inter = parser.parse(path, UnitBlockType.script, False) + analysis = SecurityVisitor(Tech.nomad) + analysis.config("configs/default.ini") + errors = list( + filter( + lambda e: e.code.startswith("sec_") or e.code.startswith("arc_"), + set(analysis.check(inter)), + ) # type: ignore + ) + errors = sorted(errors, key=lambda e: (e.path, e.line, e.code)) + self.assertEqual(len(errors), n_errors) + for i in range(n_errors): + self.assertEqual(errors[i].code, codes[i]) + self.assertEqual(errors[i].line, lines[i]) + + def test_nomad_admin(self) -> None: + self.__help_test( + "tests/security/nomad/files/admin.nomad", + 3, + ["sec_def_admin", "sec_hard_secr", "sec_hard_user"], + [29, 29, 29], + ) + + def test_nomad_empty(self) -> None: + self.__help_test( + "tests/security/nomad/files/hard_secr/hard_secr_empty_password.nomad", + 1, + ["sec_empty_pass"], + [40], + ) + + def test_nomad_hard_secret(self) -> None: + self.__help_test( + "tests/security/nomad/files/hard_secr/hard_secr_password.nomad", + 2, + ["sec_hard_pass", "sec_hard_secr"], + [40, 40], + ) + self.__help_test( + "tests/security/nomad/files/hard_secr/hard_secr_secret.nomad", + 1, + ["sec_hard_secr"], + [40], + ) + self.__help_test( + "tests/security/nomad/files/hard_secr/hard_secr_user.nomad", + 2, + ["sec_hard_secr", "sec_hard_user"], + [40, 40], + ) + + def test_nomad_http(self) -> None: + self.__help_test( + "tests/security/nomad/files/https_tls.nomad", 1, ["sec_https"], [40] + ) + + def test_nomad_inv_bind(self) -> None: + self.__help_test( + "tests/security/nomad/files/invalid_bind.nomad", + 1, + ["sec_invalid_bind"], + [39], + ) + + def test_nomad_non_official_image(self) -> None: + self.__help_test( + "tests/security/nomad/files/non_official_image.nomad", + 1, + ["sec_non_official_image"], + [31], + ) + + def test_nomad_susp(self) -> None: + self.__help_test( + "tests/security/nomad/files/susp_comment.nomad", 1, ["sec_susp_comm"], [5] + ) + + def test_nomad_weak_crypt(self) -> None: + self.__help_test( + "tests/security/nomad/files/weak_crypt.nomad", + 1, + ["sec_weak_crypt"], + [32], + ) + + def test_nomad_container_image_tag_smells(self) -> None: + self.__help_test( + "tests/security/nomad/files/container_image_tag_smells/normal_tag_and_digest.nomad", + 0, + [], + [], + ) + self.__help_test( + "tests/security/nomad/files/container_image_tag_smells/normal_tag_and_no_digest.nomad", + 1, + ["sec_image_integrity"], + [31], + ) + self.__help_test( + "tests/security/nomad/files/container_image_tag_smells/no_tag_and_digest.nomad", + 0, + [], + [], + ) + self.__help_test( + "tests/security/nomad/files/container_image_tag_smells/no_tag_no_digest.nomad", + 1, + ["sec_no_image_tag"], + [31], + ) + + self.__help_test( + "tests/security/nomad/files/container_image_tag_smells/unstable_tag_and_digest.nomad", + 1, + ["sec_unstable_tag"], + [31], + ) + self.__help_test( + "tests/security/nomad/files/container_image_tag_smells/unstable_tag_no_digest.nomad", + 2, + ["sec_image_integrity", "sec_unstable_tag"], + [31, 31], + ) + + def test_nomad_deprecated_official_img(self) -> None: + self.__help_test( + "tests/security/nomad/files/deprecated_docker_official_image.nomad", + 1, + ["sec_depr_off_imgs"], + [31], + ) + + def test_nomad_missing_healthchecks(self) -> None: + self.__help_test( + "tests/security/nomad/files/missing_healthchecks.nomad", + 1, + ["arc_missing_healthchecks"], + [21], + ) + + def test_nomad_privileged_container(self) -> None: + self.__help_test( + "tests/security/nomad/files/priv_container.nomad", + 1, + ["sec_privileged_containers"], + [32], + ) + + def test_nomad_docker_socket_mounted(self) -> None: + self.__help_test( + "tests/security/nomad/files/docker_socket_mounted.nomad", + 1, + ["sec_mounted_docker_socket"], + [33], + ) + + def test_nomad_no_log_aggregation(self) -> None: + self.__help_test( + "tests/security/nomad/files/no_log_aggregation.nomad", + 1, + ["arc_no_logging"], + [27], + ) + + def test_nomad_multiple_services_per_deployment_unit(self) -> None: + self.__help_test( + "tests/security/nomad/files/multiple_services_per_deployment_unit.nomad", + 4, + [ + "arc_multiple_services", + "sec_non_official_image", + "arc_multiple_services", + "sec_non_official_image", + ], + [57, 61, 69, 73], + ) + + def test_nomad_no_api_gateway(self) -> None: + self.__help_test( + "tests/security/nomad/files/no_api_gateway.nomad", + 1, + ["arc_no_apig"], + [7], + ) + + def test_nomad_wobbly_service_interaction(self) -> None: + self.__help_test( + "tests/security/nomad/files/wobbly_service_interaction.nomad", + 1, + ["arc_wobbly_service_interaction"], + [24], + ) From 4ee287eee2648394f131f7d6b7367491d02cf711 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:50:06 +0100 Subject: [PATCH 62/65] fix: add hack to avoid using detection function tailored to nomad for swarm detections --- glitch/analysis/security/missing_healthchecks.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/glitch/analysis/security/missing_healthchecks.py b/glitch/analysis/security/missing_healthchecks.py index 7be36477..272d9454 100644 --- a/glitch/analysis/security/missing_healthchecks.py +++ b/glitch/analysis/security/missing_healthchecks.py @@ -20,6 +20,11 @@ def check(self, element: CodeElement, file: str) -> List[Error]: # nomad group tasks if isinstance(element, UnitBlock) and element.type == UnitBlockType.block: + # HACK: avoid wrong for swarm + for au in element.atomic_units: + if not au.type.startswith("task."): + return [] + services_info: List[Dict[str, bool | str | None]] = [] # getting the services and consul sidecars at group level @@ -108,7 +113,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) # swarm - if isinstance(element, AtomicUnit): + if isinstance(element, AtomicUnit) and element.type == "service": found_healthcheck = False for att in element.attributes: @@ -147,7 +152,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: break break - if element.type == "service" and not found_healthcheck: + if not found_healthcheck: errors.append( Error("arc_missing_healthchecks", element, file, repr(element)) ) From 9835c68076d7d75df1961b1a78c8629dfc8affe3 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Thu, 4 Sep 2025 19:46:38 +0100 Subject: [PATCH 63/65] fix name of nomad task config logging driver attribute --- glitch/analysis/security/log_aggregation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glitch/analysis/security/log_aggregation.py b/glitch/analysis/security/log_aggregation.py index c310161b..b2e140c9 100644 --- a/glitch/analysis/security/log_aggregation.py +++ b/glitch/analysis/security/log_aggregation.py @@ -67,7 +67,7 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ): for _k, _v in v.value.items(): if ( - _k.value == "driver" + _k.value == "type" and isinstance(_v, String) and _v.value in log_drivers ): From 7caa60e3d70b1a2af3c2c601d77d2560852deb47 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Tue, 9 Sep 2025 04:32:33 +0100 Subject: [PATCH 64/65] hack: support multiple artifacts missing integrity checks in Nomad --- glitch/analysis/security/visitor.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/glitch/analysis/security/visitor.py b/glitch/analysis/security/visitor.py index bea98230..bbd922d0 100644 --- a/glitch/analysis/security/visitor.py +++ b/glitch/analysis/security/visitor.py @@ -433,7 +433,11 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: for au in u.atomic_units: result = self.check_integrity_check(au, file) if result is not None and result[0] is None: - errors.append(result[1]) + if isinstance(result[1], Error): + errors.append(result[1]) + else: + for err in result[1]: + errors.append(err) continue if result is not None: missing_integrity_checks[result[0]] = result[1] @@ -455,7 +459,7 @@ def check_unitblock(self, u: UnitBlock, file: str) -> List[Error]: @staticmethod def check_integrity_check( au: AtomicUnit, path: str - ) -> Optional[Tuple[str | None, Error]]: + ) -> Optional[Tuple[str | None, Error | List[Error]]]: for item in SecurityVisitor.DOWNLOAD: if not isinstance(au.name, str): continue @@ -469,7 +473,7 @@ def check_integrity_check( return os.path.basename(au.name), Error( "sec_no_int_check", au, path, repr(au) ) - + errors: List[Error] = [] for a in au.attributes: value = ( a.value.strip().lower() @@ -502,8 +506,10 @@ def check_integrity_check( if checksum: found_checksum = True if not found_checksum: - return (None, Error("sec_no_int_check", a, path, repr(a))) # type: ignore + errors.append(Error("sec_no_int_check", a, path, repr(a))) # type: ignore + if len(errors) > 0: + continue for item in SecurityVisitor.DOWNLOAD: if not re.search( r"(http|https|www)[^ ,]*\.{text}".format(text=item), value @@ -515,6 +521,8 @@ def check_integrity_check( "sec_no_int_check", au, path, repr(a) ) # type: ignore + if len(errors) > 0: + return (None, errors) return None @staticmethod From 6d9cad3fbf59e65e4fceec72327d678dc8b8e520 Mon Sep 17 00:00:00 2001 From: xCoolHat <74071118+sfondev@users.noreply.github.com> Date: Tue, 9 Sep 2025 04:33:58 +0100 Subject: [PATCH 65/65] fix: pinned container image without digest smell --- .../security/container_image_tags_smells.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/glitch/analysis/security/container_image_tags_smells.py b/glitch/analysis/security/container_image_tags_smells.py index f7852953..91f35eaf 100644 --- a/glitch/analysis/security/container_image_tags_smells.py +++ b/glitch/analysis/security/container_image_tags_smells.py @@ -49,17 +49,17 @@ def check(self, element: CodeElement, file: str) -> List[Error]: ) ) + if image != "" and not has_digest: + errors.append( + Error( + "sec_image_integrity", + bad_element, + file, + repr(bad_element), + ) + ) if image != "" and has_tag: tag = tag.lower() - if not has_digest: - errors.append( - Error( - "sec_image_integrity", - bad_element, - file, - repr(bad_element), - ) - ) dangerous_tags: List[str] = SecurityVisitor.DANGEROUS_IMAGE_TAGS