Add cli pathing application

biothings/cli/__init__.py

@@ -14,6 +14,7 @@
 from biothings.cli.commands.admin import build_admin_application
 from biothings.cli.commands.config import config_application, load_configuration
 from biothings.cli.commands.dataplugin import dataplugin_application
+from biothings.cli.commands.pathing import path_application
 
 
 def setup_logging_configuration(logging_level: Literal[10, 20, 30, 40, 50]) -> None:
@@ -63,4 +64,5 @@ def main():
 
     admin_application.add_typer(dataplugin_application, name="dataplugin")
     admin_application.add_typer(config_application, name="config")
+    admin_application.add_typer(path_application, name="path")
     return admin_application()

biothings/cli/commands/config.py

@@ -268,6 +268,13 @@ def default_biothings_configuration() -> dict:
         "HUB_MAX_WORKERS": os.cpu_count(),
         "MAX_QUEUED_JOBS": 1000
     }
+
+    # specific attributes to the biothings-cli application
+    cli_configuration = {
+        "BIOTHINGS_CLI_PATH": "biothings_hub/path",
+    }
+    configuration.update(cli_configuration)
+
     return configuration
 
 

biothings/cli/commands/decorators.py

@@ -0,0 +1,136 @@
+"""
+Collection of decorators for usage within the biothings-cli
+
+These are often method we want associated with many of the plugin methods we
+use, but don't directly impact the logic of the actual operation. Typically things
+related to paths and configurations that apply to large swaths of the cli
+would make sense as a decorator
+"""
+
+import functools
+import inspect
+import logging
+import pathlib
+import sys
+from typing import Callable
+
+from biothings.cli.exceptions import MissingPluginName
+
+
+logger = logging.getLogger(name="biothings-cli")
+
+
+def operation_mode(operation: Callable):
+    """
+    Based off the directory structure for where the biothings-cli
+    was invoked we set the "mode" to one of two states:
+
+    0) singular
+    The current working directory contains a singular data-plugin
+
+    In this case we don't require a plugin_name argument to be passed
+    at the command-line
+
+    1) hub
+    The current working directory contains N directories operating as a
+    "hub" or collection of data-plugins under one umbrella
+
+    In this case we do require a plugin_name argument to be passed
+    at the command-line. Otherwise we have no idea which data-plugin to
+    refer to
+
+    We attempt to load the plugin from this working directory. If we sucessfully load
+    either a manifest or advanced plugin, then we can safely say this is a singular
+    dataplugin
+
+    If we cannot load either a manifest or advanced plugin then we default assume that
+    the mode is hub
+    """
+
+    @functools.wraps(operation)
+    def determine_operation_mode(*args, **kwargs):
+
+        def determine_hub_mode():
+            working_directory = pathlib.Path.cwd()
+            working_directory_files = {file.name for file in working_directory.iterdir()}
+
+            mode = None
+            if "manifest.json" in working_directory_files or "manifest.yaml" in working_directory_files:
+                logger.debug("Inferring singular manifest plugin from directory structure")
+                mode = "SINGULAR"
+            elif "__init__.py" in working_directory_files:
+                logger.debug("Inferring singular advanced plugin from directory structure")
+                mode = "SINGULAR"
+            else:
+                logger.debug("Inferring multiple plugins from directory structure")
+                mode = "HUB"
+
+            if mode == "SINGULAR":
+                if kwargs.get("plugin_name", None) is not None:
+                    kwargs["plugin_name"] = None
+            elif mode == "HUB":
+                if kwargs.get("plugin_name", None) is None:
+                    raise MissingPluginName(working_directory)
+
+        @functools.wraps(operation)
+        def handle_function(*args, **kwargs):
+            operation_result = operation(*args, **kwargs)
+            return operation_result
+
+        @functools.wraps(operation)
+        async def handle_corountine(*args, **kwargs):
+            operation_result = await operation(*args, **kwargs)
+            return operation_result
+
+        determine_hub_mode()
+
+        if inspect.iscoroutinefunction(operation):
+            return handle_corountine(*args, **kwargs)
+        else:
+            return handle_function(*args, **kwargs)
+
+    return determine_operation_mode
+
+
+def cli_system_path(operation: Callable):  # pylint: disable=unused-argument
+    """
+    Used for ensuring that if we've appended files to biothings-cli
+    path file (stored under config.BIOTHINGS_CLI_PATH), then we need to update
+    the system path so we can discover the modules at runtime
+    """
+
+    @functools.wraps(operation)
+    def update_system_path(*args, **kwargs):
+
+        def update_system_path_from_file():
+            from biothings import config
+
+            discovery_path = pathlib.Path(config.BIOTHINGS_CLI_PATH).resolve().absolute()
+            path_file = discovery_path.joinpath("biothings_cli.pth")
+
+            if path_file.exists():
+                with open(path_file, "r", encoding="utf-8") as handle:
+                    path_entries = handle.readlines()
+                    path_entries = [entry.strip("\n") for entry in path_entries]
+                    sys.path.extend(path_entries)
+                    for path in path_entries:
+                        logger.debug("Adding %s to system path", path)
+
+        @functools.wraps(operation)
+        def handle_function(*args, **kwargs):
+            operation_result = operation(*args, **kwargs)
+            return operation_result
+
+        @functools.wraps(operation)
+        async def handle_corountine(*args, **kwargs):
+            operation_result = await operation(*args, **kwargs)
+            return operation_result
+
+        update_system_path_from_file()
+
+        if inspect.iscoroutinefunction(operation):
+            return handle_corountine(*args, **kwargs)
+        else:
+            return handle_function(*args, **kwargs)
+
+    return update_system_path

biothings/cli/commands/operations.py

@@ -47,7 +47,6 @@
 """
 
 import asyncio
-import functools
 import logging
 import multiprocessing
 import os
@@ -57,7 +56,7 @@
 import shutil
 import sys
 import uuid
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
 import jsonschema
 import rich
@@ -67,8 +66,9 @@
 from rich.console import Console
 from rich.panel import Panel
 
+from biothings.cli.commands.decorators import cli_system_path, operation_mode
 from biothings.cli.structure import TEMPLATE_DIRECTORY
-from biothings.cli.exceptions import MissingPluginName, UnknownUploaderSource
+from biothings.cli.exceptions import UnknownUploaderSource
 from biothings.cli.utils import (
     clean_dumped_files,
     clean_uploaded_sources,
@@ -87,65 +87,8 @@
 logger = logging.getLogger(name="biothings-cli")
 
 
-def operation_mode(operation_method: Callable):
-    """
-    Based off the directory structure for where the biothings-cli
-    was invoked we set the "mode" to one of two states:
-
-    0) singular
-    The current working directory contains a singular data-plugin
-
-    In this case we don't require a plugin_name argument to be passed
-    at the command-line
-
-    1) hub
-    The current working directory contains N directories operating as a
-    "hub" or collection of data-plugins under one umbrella
-
-    In this case we do require a plugin_name argument to be passed
-    at the command-line. Otherwise we have no idea which data-plugin to
-    refer to
-
-    We attempt to load the plugin from this working directory. If we sucessfully load
-    either a manifest or advanced plugin, then we can safely say this is a singular
-    dataplugin
-
-    If we cannot load either a manifest or advanced plugin then we default assume that
-    the mode is hub
-    """
-
-    @functools.wraps(operation_method)
-    def determine_operation_mode(*args, **kwargs):
-        working_directory = pathlib.Path.cwd()
-        working_directory_files = {file.name for file in working_directory.iterdir()}
-
-        mode = None
-        if "manifest.json" in working_directory_files or "manifest.yaml" in working_directory_files:
-            logger.debug("Inferring singular manifest plugin from directory structure")
-            mode = "SINGULAR"
-        elif "__init__.py" in working_directory_files:
-            logger.debug("Inferring singular advanced plugin from directory structure")
-            mode = "SINGULAR"
-        else:
-            logger.debug("Inferring multiple plugins from directory structure")
-            mode = "HUB"
-
-        if mode == "SINGULAR":
-            if kwargs.get("plugin_name", None) is not None:
-                kwargs["plugin_name"] = None
-        elif mode == "HUB":
-            if kwargs.get("plugin_name", None) is None:
-                raise MissingPluginName(working_directory)
-
-        operation_result = operation_method(*args, **kwargs)
-        return operation_result
-
-    return determine_operation_mode
-
-
 # do not apply operation_mode decorator since this operation means to create a new plugin
 # regardless what the current working directory has
-# @operation_mode
 def do_create(plugin_name: str, multi_uploaders: bool = False, parallelizer: bool = False):
     """
     Create a new data plugin from the template
@@ -178,6 +121,7 @@ def do_create(plugin_name: str, multi_uploaders: bool = False, parallelizer: boo
     logger.info("Successfully created data plugin template at: %s\n", new_plugin_directory)
 
 
+@cli_system_path
 @operation_mode
 async def do_dump(plugin_name: Optional[str] = None, show_dumped: bool = True) -> None:
     """
@@ -223,6 +167,7 @@ async def do_dump(plugin_name: Optional[str] = None, show_dumped: bool = True) -
         show_dumped_files(data_folder, assistant_instance.plugin_name)
 
 
+@cli_system_path
 @operation_mode
 async def do_upload(plugin_name: Optional[str] = None, batch_limit: int = 10000, show_uploaded: bool = True) -> None:
     """
@@ -277,6 +222,7 @@ async def do_upload(plugin_name: Optional[str] = None, batch_limit: int = 10000,
         show_uploaded_sources(pathlib.Path(assistant_instance.plugin_directory), assistant_instance.plugin_name)
 
 
+@cli_system_path
 @operation_mode
 async def do_parallel_upload(
     plugin_name: Optional[str] = None, batch_limit: int = 10000, show_uploaded: bool = True
@@ -344,6 +290,7 @@ async def do_parallel_upload(
         show_uploaded_sources(pathlib.Path(assistant_instance.plugin_directory), assistant_instance.plugin_name)
 
 
+@cli_system_path
 @operation_mode
 async def do_dump_and_upload(plugin_name: str) -> None:
     """
@@ -354,6 +301,7 @@ async def do_dump_and_upload(plugin_name: str) -> None:
     logger.info("[green]Success![/green] :rocket:", extra={"markup": True})
 
 
+@cli_system_path
 @operation_mode
 async def do_index(plugin_name: Optional[str] = None, sub_source_name: Optional[str] = None) -> None:
     """
@@ -540,6 +488,7 @@ async def do_index(plugin_name: Optional[str] = None, sub_source_name: Optional[
     await show_source_index(index_name, assistant_instance.index_manager, elasticsearch_mapping)
 
 
+@cli_system_path
 @operation_mode
 async def do_list(
     plugin_name: Optional[str] = None, dump: bool = True, upload: bool = True, hubdb: bool = False
@@ -569,6 +518,7 @@ async def do_list(
         show_hubdb_content()
 
 
+@cli_system_path
 @operation_mode
 async def do_inspect(
     plugin_name: Optional[str] = None,
@@ -633,6 +583,7 @@ async def do_inspect(
                 write_mapping_to_file(sub_output, inspection_mapping)
 
 
+@cli_system_path
 @operation_mode
 async def do_serve(plugin_name: Optional[str] = None, host: str = "localhost", port: int = 9999):
     """
@@ -651,6 +602,7 @@ async def do_serve(plugin_name: Optional[str] = None, host: str = "localhost", p
     await main(host=host, port=port, db=src_db, table_space=table_space)
 
 
+@cli_system_path
 @operation_mode
 async def do_clean(
     plugin_name: Optional[str] = None, dump: bool = False, upload: bool = False, clean_all: bool = False
@@ -714,6 +666,7 @@ async def display_schema():
     console.print(panel)
 
 
+@cli_system_path
 @operation_mode
 async def validate_manifest(plugin_name: Optional[str] = None):
     """