From 2872a0ac85f163fce5c3f751b11ab22c42088710 Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Wed, 19 Jul 2023 09:48:24 +0200 Subject: [PATCH] patching: integrate `b4`-based lore link processing - give the yaml lore link and prefix in `000.patching.config.yaml` - if prefix not found in any file in main patch dir, will - grab full patch using `b4` - write it out with the prefix to main patch dir - force a rewrite - if file with prefix found, it will skip - just delete the file to re-do - prefixes should not ever change, and the full b4-provided slug will be appended; respect the "standard" of the patching dir by using the correct prefix --- lib/tools/common/b4_caller.py | 75 +++++++++++++++++++++++++++++ lib/tools/common/patching_config.py | 16 ++++++ lib/tools/patching.py | 37 +++++++++++++- requirements.txt | 1 + 4 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 lib/tools/common/b4_caller.py diff --git a/lib/tools/common/b4_caller.py b/lib/tools/common/b4_caller.py new file mode 100644 index 000000000000..3f439502c73c --- /dev/null +++ b/lib/tools/common/b4_caller.py @@ -0,0 +1,75 @@ +# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2023 Ricardo Pardini +# This file is a part of the Armbian Build Framework https://github.com/armbian/build/ +# ‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹‹ +import argparse +import logging + +import b4 +import b4.mbox + +log: logging.Logger = logging.getLogger("b4_caller") + +# +# Automatic grabbing of patches from mailing lists, using 'b4' tool 'am' command. +# Patches will be grabbed and written to disk in the order they are listed here, before any other processing is done. +#b4-am: +# - { prefix: "0666", lore: "https://lore.kernel.org/r/20230706-topic-amlogic-upstream-dt-fixes-take3-v1-0-63ed070eeab2@linaro.org" } + +def get_patch_via_b4(lore_link): + # Fool get_msgid with a fake argparse.Namespace + msgid_args = argparse.Namespace() + msgid_args.msgid = lore_link + msgid = b4.get_msgid(msgid_args) + log.debug(f"msgid: {msgid}") + + msgs = b4.get_pi_thread_by_msgid(msgid) + + count = len(msgs) + log.debug('Analyzing %s messages in the thread', count) + + lmbx = b4.LoreMailbox() + for msg in msgs: + lmbx.add_message(msg) + + lser = lmbx.get_series() + + # hack at the "main config" to avoid attestation etc; b4's config is a global MAIN_CONFIG + config = b4.get_main_config() + config['attestation-policy'] = "off" + + # hack at the "user config", since there is not really an user here; its a global USER_CONFIG + uconfig = b4.get_user_config() + uconfig['name'] = "Armbian Autopatcher" + uconfig['email'] = "auto.patcher@next.armbian.com" + log.debug(f"uconfig: {uconfig}") + + # prepare for git am + am_msgs = lser.get_am_ready(addlink=True, linkmask='https://lore.kernel.org/r/%s') + log.debug('Total patches: %s', len(am_msgs)) + + top_msgid = None + for lmsg in lser.patches: + if lmsg is not None: + top_msgid = lmsg.msgid + break + + if top_msgid is None: + raise Exception(f"Could not find any patches in the series '{lore_link}'.") + + # slug for possibly naming the file + slug = lser.get_slug(extended=True) + log.debug('slug: %s', slug) + + # final contents of patch file for our purposes + body = b'' + + for msg in am_msgs: + body += b'From git@z Thu Jan 1 00:00:00 1970\n' # OH! the b4-marker! + body += b4.LoreMessage.get_msg_as_bytes(msg, headers='decode') + + log.info("Done") + + return {"body": body, "slug": slug} + diff --git a/lib/tools/common/patching_config.py b/lib/tools/common/patching_config.py index 5f24d140778f..b4d1fdc13054 100644 --- a/lib/tools/common/patching_config.py +++ b/lib/tools/common/patching_config.py @@ -40,6 +40,15 @@ def __str__(self): return f"PatchingOverlayDirectoryConfig(source={self.source}, target={self.target})" +class PatchingB4AMConfig: + def __init__(self, data: dict): + self.prefix: str = data.get("prefix", None) + self.lore: str = data.get("lore", None) + + def __str__(self): + return f"PatchingB4AMConfig(prefix={self.prefix}, lore={self.lore})" + + class PatchingToGitConfig: def __init__(self, data: dict): self.do_not_commit_files: list[str] = data.get("do-not-commit-files", []) @@ -79,6 +88,13 @@ def __init__(self, yaml_config_file_paths: list[str]): ] self.has_overlay_directories: bool = len(self.overlay_directories) > 0 + # 'b4' auto 'am' patching config + # DTS directories to copy config + self.b4_am_configs: list[PatchingB4AMConfig] = [ + PatchingB4AMConfig(data) for data in self.yaml_config.get("b4-am", []) + ] + self.has_b4_am_configs: bool = len(self.b4_am_configs) > 0 + def read_yaml_config(self, yaml_config_file_path): with open(yaml_config_file_path) as f: yaml_config = yaml.load(f, Loader=yaml.FullLoader) diff --git a/lib/tools/patching.py b/lib/tools/patching.py index d14c53f6987c..c2bd2ddedbe8 100755 --- a/lib/tools/patching.py +++ b/lib/tools/patching.py @@ -7,6 +7,7 @@ # This file is a part of the Armbian Build Framework # https://github.com/armbian/build/ # +import glob import logging import os @@ -18,6 +19,7 @@ from git import Repo import common.armbian_utils as armbian_utils +import common.b4_caller as b4_caller import common.dt_makefile_patcher as dt_makefile_patcher import common.patching_utils as patching_utils from common.md_asset_log import SummarizedMarkdownWriter @@ -71,7 +73,7 @@ exit_with_exception: "Exception | None" = None # Some path possibilities -CONST_PATCH_ROOT_DIRS = [] +CONST_PATCH_ROOT_DIRS: list[patching_utils.PatchRootDir] = [] for patch_dir_to_apply in PATCH_DIRS_TO_APPLY: if USERPATCHES_PATH is not None: @@ -121,6 +123,37 @@ # load the configs and merge them. pconfig: PatchingConfig = PatchingConfig(all_yaml_config_files) +# First of all, process the b4 configuration, which might produce new files on disk. +if pconfig.has_b4_am_configs: + log.info("Processing b4 configuration...") + for b4_config in pconfig.b4_am_configs: + # Check in all dirs in CONST_PATCH_ROOT_DIRS if a file with the prefix exists. If so, skip it. + skip_config = False + for one_root_dir in CONST_PATCH_ROOT_DIRS: + glob_pattern = f"{one_root_dir.abs_dir}/{b4_config.prefix}*.patch" + log.debug(f"Checking for existing b4 patch files with glob: '{glob_pattern}'") + if glob.glob(glob_pattern): + log.info(f"Skipping b4 configuration '{b4_config.prefix}' as it already exists in '{one_root_dir.abs_dir}'.") + skip_config = True + break + if skip_config: + continue + # Grab data using b4, and write it to the second directory in CONST_PATCH_ROOT_DIRS (the first would be user patches). + log.info(f"Processing b4 configuration: '{b4_config.prefix}' {b4_config.lore}") + b4_patch_body_slug = b4_caller.get_patch_via_b4(b4_config.lore) + target_dir = CONST_PATCH_ROOT_DIRS[1].abs_dir # user patches dir + target_file = os.path.join(target_dir, f"{b4_config.prefix}-{b4_patch_body_slug['slug']}.patch") + log.debug(f"Writing b4 patch to file: '{target_file}'") + with open(target_file, 'wb') as f: + f.write(b4_patch_body_slug["body"]) + log.info(f"Written b4 patch to file: '{target_file}'") + # if we wrote something, force the patching operation to be a rewrite against the git repo, sans archeology. + git_archeology = False + apply_patches_to_git = True + rewrite_patches_in_place = True + + log.info("Processing b4 configuration... Done.") + PATCH_FILES_FIRST: list[patching_utils.PatchFileInDir] = [] EXTRA_PATCH_FILES_FIRST: list[str] = armbian_utils.parse_env_for_tokens("EXTRA_PATCH_FILES_FIRST") EXTRA_PATCH_HASHES_FIRST: list[str] = armbian_utils.parse_env_for_tokens("EXTRA_PATCH_HASHES_FIRST") @@ -359,7 +392,7 @@ if one_patch.rewritten_patch is None: log.warning(f"Skipping patch {one_patch} from rewrite because it was not rewritten.") continue - + # Skip the patch if it doesn't need rebasing if rewrite_only_patches_needing_rebase: if "needs_rebase" not in one_patch.problems: diff --git a/requirements.txt b/requirements.txt index 25b100e4063e..446a3a51e464 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ Jinja2 == 3.1.6 # for templating rich == 14.0.0 # for rich text formatting dtschema == 2025.2 # for checking dts files and dt bindings yamllint == 1.37.1 # for checking dts files and dt bindings +b4==0.12.3 # b4: patch/mailing list workhorse from kernel.org