Merge pull request #3556 from samsrabin/merge-master-20251016

Merge ctsm5.3.080 into b4b-dev

Author: samsrabin

.gitmodules

@@ -36,7 +36,7 @@ fxDONOTUSEurl = https://github.com/NGEET/fates
 [submodule "cism"]
 path = components/cism
 url = https://github.com/ESCOMP/CISM-wrapper
-fxtag = cismwrap_2_2_006
+fxtag = cismwrap_2_2_010
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CISM-wrapper
@@ -68,31 +68,31 @@ fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute
 [submodule "ccs_config"]
 path = ccs_config
 url = https://github.com/ESMCI/ccs_config_cesm.git
-fxtag = ccs_config_cesm1.0.56
+fxtag = ccs_config_cesm1.0.61
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 
 [submodule "cime"]
 path = cime
 url = https://github.com/ESMCI/cime
-fxtag = cime6.1.113
+fxtag = cime6.1.128
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/cime
 
 [submodule "cmeps"]
 path = components/cmeps
 url = https://github.com/ESCOMP/CMEPS.git
-fxtag = cmeps1.1.5
+fxtag = cmeps1.1.20
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
 
 [submodule "cdeps"]
 path = components/cdeps
 url = https://github.com/ESCOMP/CDEPS.git
-fxtag = cdeps1.0.79
+fxtag = cdeps1.0.81
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git

.lib/git-fleximod/git_fleximod/cli.py

@@ -1,38 +1,50 @@
 from pathlib import Path
-import argparse
+import argparse, os, sys
 from git_fleximod import utils
 
-__version__ = "0.9.3"
+__version__ = "1.0.2"
+
+class CustomArgumentParser(argparse.ArgumentParser):
+    def print_help(self, file=None):
+        # First print the default help message
+        super().print_help(file)
+
+        # Then append the contents of README.md
+        candidate_paths = [
+            Path(sys.prefix) / "share" / "git_fleximod" / "README.md",
+            Path(__file__).resolve().parent.parent / "README.md",  # fallback for dev
+        ]
+        for path in candidate_paths:
+            if os.path.exists(path):
+                with open(path) as f:
+                    print( f.read(), file=file)
+                    return
+        print( "README.md not found.", file=file)
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree
     which contains a file called filename """
-    try:
-        root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"],
-                                        output_to_caller=True ).rstrip()
-    except:
-        d = Path.cwd()
-        root = Path(d.root)
-        dirlist = []
-        dl = d
-        while dl != root:
-            dirlist.append(dl)
-            dl = dl.parent
-        dirlist.append(root)
-        dirlist.reverse()
-
-        for dl in dirlist:
-            attempt = dl / filename
-            if attempt.is_file():
-                return str(dl)
-        return None
-    return Path(root)
+    d = Path.cwd()
+    root = Path(d.root)
+    dirlist = []
+    dl = d
+    while dl != root:
+        dirlist.append(dl)
+        dl = dl.parent
+    dirlist.append(root)
+    dirlist.reverse()
+
+    for dl in dirlist:
+        attempt = dl / filename
+        if attempt.is_file():
+            return str(dl)
+    return None
 
 def get_parser():
     description = """
     %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models
     """
-    parser = argparse.ArgumentParser(
+    parser = CustomArgumentParser(
         description=description, formatter_class=argparse.RawDescriptionHelpFormatter
     )
 

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -9,6 +9,7 @@
 import shutil
 import logging
 import textwrap
+import asyncio
 from git_fleximod import utils
 from git_fleximod import cli
 from git_fleximod.gitinterface import GitInterface
@@ -181,6 +182,8 @@ def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger):
     url = gitmodules.get(name, "url")
     assert path and url, f"Malformed .gitmodules file {path} {url}"
     tag = gitmodules.get(name, "fxtag")
+    if not tag:
+        tag = gitmodules.get(name, "hash")
     fxurl = gitmodules.get(name, "fxDONOTUSEurl")
     fxsparse = gitmodules.get(name, "fxsparse")
     fxrequired = gitmodules.get(name, "fxrequired")
@@ -216,10 +219,10 @@ def git_toplevelroot(root_dir, logger):
     _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
     return superroot
 
-def submodules_update(gitmodules, root_dir, requiredlist, force):
-    for name in gitmodules.sections():
+async def submodules_update(gitmodules, root_dir, requiredlist, force):
+    async def update_submodule(name, requiredlist, force):
         submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger)
-    
+
         _, needsupdate, localmods, testfails = submod.status()
         if not submod.fxrequired:
             submod.fxrequired = "AlwaysRequired"
@@ -237,11 +240,11 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
             if "Optional" in fxrequired and "Optional" not in requiredlist:
                 if fxrequired.startswith("Always"):
                     print(f"Skipping optional component {name:>20}")
-                continue
+                return # continue to next submodule
         optional = "AlwaysOptional" in requiredlist
 
         if fxrequired in requiredlist:
-            submod.update()
+            await submod.update()
             repodir = os.path.join(root_dir, submod.path)
             if os.path.exists(os.path.join(repodir, ".gitmodules")):
                 # recursively handle this checkout
@@ -250,8 +253,10 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 newrequiredlist = ["AlwaysRequired"]
                 if optional:
                     newrequiredlist.append("AlwaysOptional")
+                await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
 
-                submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
+    tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()]
+    await asyncio.gather(*tasks)
 
 def local_mods_output():
     text = """\
@@ -345,7 +350,7 @@ def main():
         sys.exit(f"No submodule components found, root_dir={root_dir}")
     retval = 0
     if action == "update":
-        submodules_update(gitmodules, root_dir, fxrequired, force)
+        asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force))
     elif action == "status":
         tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True)
         if tfails + lmods + updates > 0:

.lib/git-fleximod/git_fleximod/gitinterface.py

@@ -2,6 +2,7 @@
 import sys
 from . import utils
 from pathlib import Path
+import asyncio
 
 class GitInterface:
     def __init__(self, repo_path, logger):
@@ -47,6 +48,16 @@ def _init_git_repo(self):
             command = ("git", "-C", str(self.repo_path), "init")
             utils.execute_subprocess(command)
 
+    def _git_operation_command(self, operation, args):
+        newargs = []
+        for a in args:
+            # Do not use ssh interface
+            if isinstance(a, str):
+                a = a.replace("[email protected]:", "https://github.com/")
+            newargs.append(a)
+
+        return self._git_command(operation, *newargs)
+
     # pylint: disable=unused-argument
     def git_operation(self, operation, *args, **kwargs):
         newargs = []
@@ -66,6 +77,25 @@ def git_operation(self, operation, *args, **kwargs):
         else:
             return 0, command
 
+    # pylint: disable=unused-argument
+    async def git_operation_async(self, operation, *args, **kwargs):
+        command = self._git_operation_command(operation, args)
+        if isinstance(command, list):
+            try:
+                process = await asyncio.create_subprocess_exec(
+                    *command,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE
+                )
+                stdout, stderr = await process.communicate()
+                status = process.returncode
+                output = stdout.decode().strip() if stdout else stderr.decode().strip()
+                return status, output
+            except Exception as e:
+                sys.exit(e)
+        else:
+            return 0, command
+
     def config_get_value(self, section, name):
         if self._use_module:
             config = self.repo.config_reader()

.lib/git-fleximod/git_fleximod/submodule.py

@@ -119,27 +119,25 @@ def status(self):
                                 atag = atag[:-1]
                             if atag == self.fxtag:
                                 break
-
-                
-                #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}")
-                #                atag = git.git_operation("describe", "--tags", "--always")
-                # ahash =  git.git_operation("rev-list", "HEAD").partition("\n")[0]
-                    
                 recurse = False
                 if rurl != self.url:
                     remote = self._add_remote(git)
                     git.git_operation("fetch", remote)
+                # Asked for a tag and found that tag
                 if self.fxtag and atag == self.fxtag:
                     result = f"  {self.name:>20} at tag {self.fxtag}"
                     recurse = True
                     testfails = False
+                # Asked for and found a hash
                 elif self.fxtag and (ahash[: len(self.fxtag)] == self.fxtag or (self.fxtag.find(ahash)==0)):
                     result = f"  {self.name:>20} at hash {ahash}"
                     recurse = True
                     testfails = False
+                # Asked for and found a hash
                 elif atag == ahash:
                     result = f"  {self.name:>20} at hash {ahash}"
                     recurse = True
+                # Did not find requested tag or hash
                 elif self.fxtag:
                     result = f"s {self.name:>20} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}"
                     testfails = True
@@ -284,17 +282,18 @@ def sparse_checkout(self):
                 if not os.path.isdir(infodir):
                     os.makedirs(infodir)
                 gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout"))
-            if os.path.isfile(gitsparse):
-                self.logger.warning(
-                    "submodule {} is already initialized {}".format(self.name, rootdotgit)
-                )
-                return
-
-            with utils.pushd(sprep_repo):
+                if os.path.isfile(gitsparse):
+                    self.logger.warning(
+                        "submodule {} is already initialized {}".format(self.name, rootdotgit)
+                    )
+                    os.remove(gitsparse)
+                
                 if os.path.isfile(self.fxsparse):
-                    
                     shutil.copy(self.fxsparse, gitsparse)
-                
+                else:
+                    self.logger.warning(
+                        "submodule {} could not find {}".format(self.name, self.fxsparse)
+                    )
 
         # Finally checkout the repo
         sprepo_git.git_operation("fetch", "origin", "--tags")
@@ -303,11 +302,18 @@ def sparse_checkout(self):
             print(f"Error checking out {self.name:>20} at {self.fxtag}")
         else:
             print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
+        status,f = sprepo_git.git_operation("status")
+        # Restore any files deleted from sandbox
+        for line in f.splitlines():
+            if "deleted:" in line:
+                deleted_file = line.split("deleted:")[1].strip()
+                sprepo_git.git_operation("checkout", deleted_file)
+
         rgit.config_set_value('submodule.' + self.name, "active", "true")
         rgit.config_set_value('submodule.' + self.name, "url", self.url)
         rgit.config_set_value('submodule.' + self.name, "path", self.path)
 
-    def update(self):
+    async def update(self):
         """
         Updates the submodule to the latest or specified version.
 
@@ -341,6 +347,9 @@ def update(self):
         # Look for a .gitmodules file in the newly checkedout repo
         if self.fxsparse:
             print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}")
+            if not os.path.isfile(self.fxsparse):
+                self.logger.info("Submodule {} fxsparse file not found".format(self.name))
+
             self.sparse_checkout()
         else:
             if not repo_exists and self.url:
@@ -378,19 +387,26 @@ def update(self):
                 git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) 
 
             if not repo_exists:
-                git.git_operation("submodule", "update", "--init", "--", self.path)
+                git.git_operation("submodule", "init", "--", self.path)
+                await git.git_operation_async("submodule", "update", "--", self.path)
 
             if self.fxtag:        
                 smgit = GitInterface(repodir, self.logger)
                 newremote = self._add_remote(smgit)
                 # Trying to distingush a tag from a hash
-                allowed = set(string.digits + 'abcdef') 
+                allowed = set(string.digits + 'abcdef')
+                status = 0
                 if not set(self.fxtag) <= allowed:
                     # This is a tag
                     tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}"
-                    smgit.git_operation("fetch", newremote, tag)
-                smgit.git_operation("checkout", self.fxtag)
-
+                    status,_ = smgit.git_operation("fetch", newremote, tag)
+                if status == 0:
+                    status,_ = smgit.git_operation("checkout", self.fxtag)
+                if status:
+                    utils.fatal_error(
+                        f"Failed to checkout {self.name} at tag or hash {self.fxtag} from {repodir}"
+                    )
+                    
             if not os.path.exists(os.path.join(repodir, ".git")):
                 utils.fatal_error(
                     f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}"
@@ -408,6 +424,18 @@ def update(self):
                 if fxtag and fxtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
                 status, atag = git.git_operation("describe", "--tags", "--always")
+                status, files = git.git_operation("diff", "--name-only", "-z")
+                modfiles = []
+                moddirs = []
+                if files:
+                    for f in files.split('\0'):
+                        if f:
+                            if os.path.exists(f):
+                                git.git_operation("checkout",f)
+                            elif os.path.isdir(f):
+                                moddirs.append(f)
+                            else:
+                                modfiles.append(f)
                 if fxtag and fxtag != atag:
                     try:
                         status, _ = git.git_operation("checkout", fxtag)
@@ -419,6 +447,10 @@ def update(self):
 
                 elif not fxtag:
                     print(f"No fxtag found for submodule {self.name:>20}")
+                elif modfiles:
+                    print(f"{self.name:>20} has modified files: {modfiles}")
+                elif moddirs:
+                    print(f"{self.name:>20} has modified directories: {moddirs}")
                 else:
                     print(f"{self.name:>20} up to date.")