Merge branch 'feat-timeseries-split' into experimental

Author: sanjayankur31

pyneuroml/archive/__init__.py

@@ -8,7 +8,6 @@
 
 import argparse
 import logging
-import os
 import pathlib
 import shutil
 import typing
@@ -18,6 +17,7 @@
 import pyneuroml.sedml as pynmls
 import pyneuroml.utils as pynmlu
 import pyneuroml.utils.cli as pynmluc
+from pyneuroml.utils.misc import chdir
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -170,22 +170,18 @@ def create_combine_archive(
         zipfile_name = rootfile
 
     # change to directory of rootfile
-    thispath = os.getcwd()
-    os.chdir(rootdir)
-
-    lems_def_dir = None
-    if len(filelist) == 0:
-        lems_def_dir = pynmlu.get_model_file_list(
-            rootfile, filelist, rootdir, lems_def_dir
-        )
-
-    create_combine_archive_manifest(rootfile, filelist + extra_files, rootdir)
-    filelist.append("manifest.xml")
+    with chdir(rootdir):
+        lems_def_dir = None
+        if len(filelist) == 0:
+            lems_def_dir = pynmlu.get_model_file_list(
+                rootfile, filelist, rootdir, lems_def_dir
+            )
+        create_combine_archive_manifest(rootfile, filelist + extra_files, rootdir)
+        filelist.append("manifest.xml")
 
-    with ZipFile(zipfile_name + zipfile_extension, "w") as archive:
-        for f in filelist + extra_files:
-            archive.write(f)
-    os.chdir(thispath)
+        with ZipFile(zipfile_name + zipfile_extension, "w") as archive:
+            for f in filelist + extra_files:
+                archive.write(f)
 
     if lems_def_dir is not None:
         logger.info(f"Removing LEMS definitions directory {lems_def_dir}")

pyneuroml/nsgr.py

@@ -14,6 +14,7 @@
 from zipfile import ZipFile
 
 from pyneuroml.runners import generate_sim_scripts_in_folder
+from pyneuroml.utils.misc import chdir
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -126,8 +127,6 @@ def run_on_nsg(
     # NSG requires that the top level directory exist
     nsg_dir = pathlib.Path(zipfile_name.replace(".zip", ""))
 
-    cwd = pathlib.Path.cwd()
-
     tdir = generate_sim_scripts_in_folder(
         engine=engine,
         lems_file_name=lems_file_name,
@@ -139,44 +138,43 @@ def run_on_nsg(
     logger.info("Generating zip file")
     runner_file = ""
 
-    os.chdir(str(tdir))
-    generated_files = os.listdir(nsg_dir)
-
-    print(f"Generated files are {generated_files}")
-
-    with ZipFile(zipfile_name, "w") as archive:
-        for f in generated_files:
-            if engine == "jneuroml_neuron":
-                if f.endswith("_nrn.py"):
-                    runner_file = f
-            elif engine == "jneuroml_netpyne":
-                if f.endswith("_netpyne.py"):
-                    runner_file = f
-            fpath = pathlib.Path(f)
-            moved_path = nsg_dir / fpath
-            archive.write(str(moved_path))
-
-    logger.debug("Printing testParam.properties")
-    nsg_sim_config_dict["filename_"] = runner_file
-    logger.debug(f"NSG sim config is: {nsg_sim_config_dict}")
-
-    with open("testParam.properties", "w") as file:
-        for key, val in nsg_sim_config_dict.items():
-            print(f"{key}={val}", file=file)
-
-    logger.debug("Printing testInput.properties")
-    with open("testInput.properties", "w") as file:
-        print(f"infile_=@./{zipfile_name}", file=file)
-
-    print(f"{zipfile_name} generated")
-    # uses argv, where the first argument is the script itself, so we must pass
-    # something as the 0th index of the list
-    if not dry_run:
-        if nsgr_submit(["", ".", "validate"]) == 0:
-            print("Attempting to submit to NSGR")
-            return nsgr_submit(["", ".", "run"])
-    else:
-        print("Dry run mode enabled. Not submitting to NSG.")
-
-    os.chdir(str(cwd))
+    with chdir(str(tdir)):
+        generated_files = os.listdir(nsg_dir)
+
+        print(f"Generated files are {generated_files}")
+
+        with ZipFile(zipfile_name, "w") as archive:
+            for f in generated_files:
+                if engine == "jneuroml_neuron":
+                    if f.endswith("_nrn.py"):
+                        runner_file = f
+                elif engine == "jneuroml_netpyne":
+                    if f.endswith("_netpyne.py"):
+                        runner_file = f
+                fpath = pathlib.Path(f)
+                moved_path = nsg_dir / fpath
+                archive.write(str(moved_path))
+
+        logger.debug("Printing testParam.properties")
+        nsg_sim_config_dict["filename_"] = runner_file
+        logger.debug(f"NSG sim config is: {nsg_sim_config_dict}")
+
+        with open("testParam.properties", "w") as file:
+            for key, val in nsg_sim_config_dict.items():
+                print(f"{key}={val}", file=file)
+
+        logger.debug("Printing testInput.properties")
+        with open("testInput.properties", "w") as file:
+            print(f"infile_=@./{zipfile_name}", file=file)
+
+        print(f"{zipfile_name} generated")
+        # uses argv, where the first argument is the script itself, so we must pass
+        # something as the 0th index of the list
+        if not dry_run:
+            if nsgr_submit(["", ".", "validate"]) == 0:
+                print("Attempting to submit to NSGR")
+                return nsgr_submit(["", ".", "run"])
+        else:
+            print("Dry run mode enabled. Not submitting to NSG.")
+
     return tdir

pyneuroml/runners.py

@@ -30,6 +30,7 @@
 import pyneuroml.utils.simdata as pynmls
 from pyneuroml import DEFAULTS, __version__
 from pyneuroml.errors import UNKNOWN_ERR
+from pyneuroml.utils.misc import chdir
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -1209,7 +1210,6 @@ def generate_sim_scripts_in_folder(
     if root_dir is None:
         root_dir = "."
 
-    cwd = Path.cwd()
     tdir = pyneuroml.utils.get_pyneuroml_tempdir(rootdir=run_dir, prefix="pyneuroml")
     os.mkdir(tdir)
 
@@ -1220,94 +1220,89 @@ def generate_sim_scripts_in_folder(
 
     # change to root_dir, so that we're in the directory where the lems file
     # is
-    os.chdir(root_dir)
-
-    logger.debug("Getting list of model files")
-    model_file_list = []  # type: list
-    lems_def_dir = None
-    lems_def_dir = pyneuroml.utils.get_model_file_list(
-        lems_file_name, model_file_list, root_dir, lems_def_dir
-    )
-
-    logger.debug(f"Model file list is {model_file_list}")
-
-    for model_file in model_file_list:
-        logger.debug(f"Copying: {model_file} -> {tdir}/{model_file}")
-        # if model file has directory structures in it, recreate the dirs in
-        # the temporary directory
-        if len(model_file.split("/")) > 1:
-            # throw error if files in parent directories are referred to
-            if "../" in model_file:
-                raise ValueError(
-                    """
-                    Cannot handle parent directories because we
-                    cannot create these directories correctly in
-                    the temporary location. Please re-organize
-                    your code such that all included files are in
-                    sub-directories of the root directory where the
-                    main file resides.
-                    """
-                )
-
-            model_file_path = pathlib.Path(tdir + "/" + model_file)
-            parent = model_file_path.parent
-            parent.mkdir(parents=True, exist_ok=True)
-        shutil.copy(model_file, tdir + "/" + model_file)
-
-    if lems_def_dir is not None:
-        logger.info(f"Removing LEMS definitions directory {lems_def_dir}")
-        shutil.rmtree(lems_def_dir)
-
-    os.chdir(tdir)
-    logger.info(f"Working in {tdir}")
-    start_time = time.time() - 1.0
-
-    if engine == "jneuroml_neuron":
-        run_lems_with(
-            engine,
-            lems_file_name=Path(lems_file_name).name,
-            compile_mods=False,
-            only_generate_scripts=True,
-            *engine_args,
-            **engine_kwargs,
-        )
-    elif engine == "jneuroml_netpyne":
-        run_lems_with(
-            engine,
-            lems_file_name=Path(lems_file_name).name,
-            only_generate_scripts=True,
-            *engine_args,
-            **engine_kwargs,
+    with chdir(root_dir):
+        logger.debug("Getting list of model files")
+        model_file_list = []  # type: list
+        lems_def_dir = None
+        lems_def_dir = pyneuroml.utils.get_model_file_list(
+            lems_file_name, model_file_list, root_dir, lems_def_dir
         )
 
-    generated_files = pyneuroml.utils.get_files_generated_after(
-        start_time, ignore_suffixes=["xml", "nml"]
-    )
+        logger.debug(f"Model file list is {model_file_list}")
+
+        for model_file in model_file_list:
+            logger.debug(f"Copying: {model_file} -> {tdir}/{model_file}")
+            # if model file has directory structures in it, recreate the dirs in
+            # the temporary directory
+            if len(model_file.split("/")) > 1:
+                # throw error if files in parent directories are referred to
+                if "../" in model_file:
+                    raise ValueError(
+                        """
+                        Cannot handle parent directories because we
+                        cannot create these directories correctly in
+                        the temporary location. Please re-organize
+                        your code such that all included files are in
+                        sub-directories of the root directory where the
+                        main file resides.
+                        """
+                    )
+
+                model_file_path = pathlib.Path(tdir + "/" + model_file)
+                parent = model_file_path.parent
+                parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy(model_file, tdir + "/" + model_file)
+
+        if lems_def_dir is not None:
+            logger.info(f"Removing LEMS definitions directory {lems_def_dir}")
+            shutil.rmtree(lems_def_dir)
+
+    with chdir(tdir):
+        logger.info(f"Working in {tdir}")
+        start_time = time.time() - 1.0
+
+        if engine == "jneuroml_neuron":
+            run_lems_with(
+                engine,
+                lems_file_name=Path(lems_file_name).name,
+                compile_mods=False,
+                only_generate_scripts=True,
+                *engine_args,
+                **engine_kwargs,
+            )
+        elif engine == "jneuroml_netpyne":
+            run_lems_with(
+                engine,
+                lems_file_name=Path(lems_file_name).name,
+                only_generate_scripts=True,
+                *engine_args,
+                **engine_kwargs,
+            )
 
-    # For NetPyNE, the channels are converted to NEURON mod files, but the
-    # network and cells are imported from the nml files.
-    # So we include all the model files too.
-    if engine == "jneuroml_netpyne":
-        generated_files.extend(model_file_list)
+        generated_files = pyneuroml.utils.get_files_generated_after(
+            start_time, ignore_suffixes=["xml", "nml"]
+        )
 
-    logger.debug(f"Generated files are: {generated_files}")
+        # For NetPyNE, the channels are converted to NEURON mod files, but the
+        # network and cells are imported from the nml files.
+        # So we include all the model files too.
+        if engine == "jneuroml_netpyne":
+            generated_files.extend(model_file_list)
 
-    if generated_files_dir_name is None:
-        generated_files_dir_name = Path(tdir).name + "_generated"
-    logger.debug(
-        f"Creating directory and moving generated files to it: {generated_files_dir_name}"
-    )
+        logger.debug(f"Generated files are: {generated_files}")
 
-    for f in generated_files:
-        fpath = pathlib.Path(f)
-        moved_path = generated_files_dir_name / fpath
-        # use os.renames because pathlib.Path.rename does not move
-        # recursively and so cannot move files within directories
-        os.renames(fpath, moved_path)
+        if generated_files_dir_name is None:
+            generated_files_dir_name = Path(tdir).name + "_generated"
+        logger.debug(
+            f"Creating directory and moving generated files to it: {generated_files_dir_name}"
+        )
 
-    # return to original directory
-    # doesn't affect scripts much, but does affect our tests
-    os.chdir(str(cwd))
+        for f in generated_files:
+            fpath = pathlib.Path(f)
+            moved_path = generated_files_dir_name / fpath
+            # use os.renames because pathlib.Path.rename does not move
+            # recursively and so cannot move files within directories
+            os.renames(fpath, moved_path)
 
     return tdir
 

pyneuroml/utils/__init__.py

@@ -713,17 +713,18 @@ def get_model_file_list(
             filelist.append(str(fullrootfile_rel))
 
     if str(rootfile_name).endswith(".nml"):
-        print(f"Processing NML file: {fullrootfile_rel}")
+        logger.info(f"Processing NML file: {fullrootfile_rel}")
         rootdoc = read_neuroml2_file(fullrootfile_rel)
         logger.debug(f"Has includes: {rootdoc.includes}")
 
         for inc in rootdoc.includes:
-            logger.debug(f"Processing includes: {inc.href} in {str(rootdirpath)}")
+            logger.debug(f"NML: Processing includes: {inc.href} in {str(rootdirpath)}")
             lems_def_dir = get_model_file_list(
                 inc.href, filelist, str(rootdirpath_rel), lems_def_dir
             )
 
     elif str(rootfile_name).endswith(".xml"):
+        logger.info(f"Processing LEMS file: {fullrootfile_rel}")
         # extract the standard NeuroML2 LEMS definitions into a directory
         # so that the LEMS parser can find them
         if lems_def_dir is None:
@@ -734,17 +735,25 @@ def get_model_file_list(
         model.import_from_file(fullrootfile_rel)
 
         for inc in model.included_files:
-            # `inc` includes the folder name, but we want to keep the file name
-            # and the directory in which it is located separtely as the
-            # directory may have to be passed on recursively to other included
-            # files. So, we separate the name out.
-            incfile = pathlib.Path(inc).name
-            logger.debug(f"Processing include file {incfile} ({inc})")
+            # `inc` includes the complete path relative to the current
+            # directory, which may repeat information such as the "rootdirpath"
+            # that we're tracking outselves. So, we need to do some massaging
+            # here to get the path to inc relative to the rootdirpath_rel
+            # again
+            incfile_path = pathlib.Path(inc)
+            incfile = incfile_path.name
+
+            if incfile_path.is_relative_to(rootdirpath_rel):
+                incfile_path_rel = incfile_path.relative_to(rootdirpath_rel)
+            else:
+                incfile_path_rel = incfile_path
+
+            logger.debug(f"LEMS: Processing include file {incfile} ({inc})")
             if incfile in STANDARD_LEMS_FILES:
                 logger.debug(f"Ignoring NeuroML2 standard LEMS file: {inc}")
                 continue
             lems_def_dir = get_model_file_list(
-                incfile, filelist, str(rootdirpath_rel), lems_def_dir
+                str(incfile_path_rel), filelist, str(rootdirpath_rel), lems_def_dir
             )
 
     elif str(rootfile_name).endswith(".sedml"):
@@ -754,6 +763,7 @@ def get_model_file_list(
             logger.error("Please install optional dependencies to use SED-ML features:")
             logger.error("pip install pyneuroml[combine]")
 
+        logger.info(f"Processing SED-ML file: {fullrootfile_rel}")
         rootdoc = libsedml.readSedMLFromFile(fullrootfile_rel)
 
         # there should only be one model