feat: omit telemetry logs

this does a few things:

1. fixes `on_start` so that all span [START] and [END] is printed. not just [END]
2. change `log.py` to omit the telemetry category by default, but it can be enabled like any other logger `LLAMA_STACK_LOGGING=telemetry=INFO`

This allows us to keep the metric logging and the verbosity of seeing the span [START] and [END] but by default hides it from normal users.

This conforms to our logging system since a user just need to enable the category to see the logs

Signed-off-by: Charlie Doern <[email protected]>

Author: cdoern

llama_stack/log.py

@@ -38,6 +38,10 @@
 # Initialize category levels with default level
 _category_levels: dict[str, int] = dict.fromkeys(CATEGORIES, DEFAULT_LOG_LEVEL)
 
+# Telemetry category - only added when explicitly enabled
+_telemetry_enabled = False
+_telemetry_level = None
+
 
 def config_to_category_levels(category: str, level: str):
     """
@@ -99,6 +103,8 @@ def parse_environment_config(env_config: str) -> dict[str, int]:
     Returns:
         Dict[str, int]: A dictionary mapping categories to their log levels.
     """
+    global _telemetry_enabled, _telemetry_level, CATEGORIES
+
     category_levels = {}
     delimiter = ","
     for pair in env_config.split(delimiter):
@@ -109,7 +115,19 @@ def parse_environment_config(env_config: str) -> dict[str, int]:
             category, level = pair.split("=", 1)
             category = category.strip().lower()
             level = level.strip().upper()  # Convert to uppercase for logging._nameToLevel
-            category_levels.update(config_to_category_levels(category=category, level=level))
+
+            if category == "telemetry":
+                # Add telemetry to CATEGORIES and enable it
+                level_value = logging._nameToLevel.get(level)
+                if level_value is not None:
+                    _telemetry_enabled = True
+                    _telemetry_level = level_value
+                    logging.info(f"Telemetry logging enabled at level '{level}'.")
+                else:
+                    logging.warning(f"Unknown telemetry log level '{level}'. Telemetry logging will remain disabled.")
+            else:
+                # Handle regular categories
+                category_levels.update(config_to_category_levels(category=category, level=level))
 
         except ValueError:
             logging.warning(f"Invalid logging configuration: '{pair}'. Expected format: 'category=level'.")
@@ -171,6 +189,23 @@ def filter(self, record):
                 record.category = "uncategorized"  # Default to 'uncategorized' if no category found
             return True
 
+    class TelemetryBlockFilter(logging.Filter):
+        """Block all telemetry logs when telemetry is not enabled."""
+
+        def __init__(self, telemetry_enabled=False):
+            super().__init__()
+            self.telemetry_enabled = telemetry_enabled
+
+        def filter(self, record):
+            # If telemetry is not enabled, block all telemetry logs
+            if not self.telemetry_enabled:
+                # Check if this is a telemetry log by logger name or category
+                if (hasattr(record, "name") and record.name.startswith("telemetry")) or (
+                    hasattr(record, "category") and record.category == "telemetry"
+                ):
+                    return False  # Block this log
+            return True  # Allow all other logs
+
     # Determine the root logger's level (default to WARNING if not specified)
     root_level = category_levels.get("root", logging.WARNING)
 
@@ -182,7 +217,7 @@ def filter(self, record):
             "show_time": False,
             "show_path": False,
             "markup": True,
-            "filters": ["category_filter"],
+            "filters": ["category_filter", "telemetry_block_filter"],
         }
     }
 
@@ -193,8 +228,27 @@ def filter(self, record):
             "filename": log_file,
             "mode": "a",
             "encoding": "utf-8",
+            "filters": ["category_filter", "telemetry_block_filter"],
         }
 
+    # Build loggers configuration step by step
+    loggers_config = {}
+    for category in CATEGORIES:
+        if category == "telemetry" and not _telemetry_enabled:
+            # When telemetry is disabled, configure it to be silent
+            loggers_config[category] = {
+                "level": logging.CRITICAL,
+                "propagate": False,
+                "handlers": [],  # No handlers = no output
+            }
+        else:
+            # Regular categories get standard handlers
+            loggers_config[category] = {
+                "handlers": list(handlers.keys()),  # Apply all handlers
+                "level": category_levels.get(category, DEFAULT_LOG_LEVEL),
+                "propagate": False,  # Disable propagation to root logger
+            }
+
     logging_config = {
         "version": 1,
         "disable_existing_loggers": False,
@@ -208,35 +262,43 @@ def filter(self, record):
         "filters": {
             "category_filter": {
                 "()": CategoryFilter,
-            }
-        },
-        "loggers": {
-            category: {
-                "handlers": list(handlers.keys()),  # Apply all handlers
-                "level": category_levels.get(category, DEFAULT_LOG_LEVEL),
-                "propagate": False,  # Disable propagation to root logger
-            }
-            for category in CATEGORIES
+            },
+            "telemetry_block_filter": {
+                "()": TelemetryBlockFilter,
+                "telemetry_enabled": _telemetry_enabled,
+            },
         },
+        "loggers": loggers_config,
         "root": {
             "handlers": list(handlers.keys()),
             "level": root_level,  # Set root logger's level dynamically
         },
     }
+
     dictConfig(logging_config)
 
     # Ensure third-party libraries follow the root log level
     for _, logger in logging.root.manager.loggerDict.items():
         if isinstance(logger, logging.Logger):
             logger.setLevel(root_level)
 
+    # Explicitly silence telemetry loggers when telemetry is not enabled
+    if not _telemetry_enabled:
+        telemetry_logger = logging.getLogger("telemetry")
+        telemetry_logger.setLevel(logging.CRITICAL)  # Silence all telemetry logs
+        # Also silence any child telemetry loggers
+        for name, logger in logging.root.manager.loggerDict.items():
+            if isinstance(logger, logging.Logger) and name.startswith("telemetry"):
+                logger.setLevel(logging.CRITICAL)
+
 
 def get_logger(
     name: str, category: str = "uncategorized", config: LoggingConfig | None | None = None
 ) -> logging.LoggerAdapter:
     """
     Returns a logger with the specified name and category.
     If no category is provided, defaults to 'uncategorized'.
+    Note: telemetry category is only available when explicitly enabled via LLAMA_STACK_LOGGING=telemetry=LEVEL
 
     Parameters:
         name (str): The name of the logger (e.g., module or filename).

llama_stack/providers/inline/telemetry/meta_reference/console_span_processor.py

@@ -21,9 +21,6 @@ def __init__(self, print_attributes: bool = False):
         self.print_attributes = print_attributes
 
     def on_start(self, span: ReadableSpan, parent_context=None) -> None:
-        if span.attributes and span.attributes.get("__autotraced__"):
-            return
-
         timestamp = datetime.fromtimestamp(span.start_time / 1e9, tz=UTC).strftime("%H:%M:%S.%f")[:-3]
         logger.info(f"[dim]{timestamp}[/dim] [bold magenta][START][/bold magenta] [dim]{span.name}[/dim]")