feat: omit telemetry logs

this does a few things:

1. fixes `on_start` so that all span [START] and [END] is printed. not just [END]
2. change `log.py` to omit the telemetry category by default, but it can be enabled like any other logger `LLAMA_STACK_LOGGING=telemetry=INFO`

This allows us to keep the metric logging and the verbosity of seeing the span [START] and [END] but by default hides it from normal users.

This conforms to our logging system since a user just need to enable the category to see the logs

Signed-off-by: Charlie Doern <[email protected]>

Author: cdoern

llama_stack/log.py

@@ -38,6 +38,9 @@
 # Initialize category levels with default level
 _category_levels: dict[str, int] = dict.fromkeys(CATEGORIES, DEFAULT_LOG_LEVEL)
 
+# telemetry is enabled only when explicitly added to the env list
+_telemetry_enabled = False
+
 
 def config_to_category_levels(category: str, level: str):
     """
@@ -99,6 +102,8 @@ def parse_environment_config(env_config: str) -> dict[str, int]:
     Returns:
         Dict[str, int]: A dictionary mapping categories to their log levels.
     """
+    global _telemetry_enabled
+
     category_levels = {}
     delimiter = ","
     for pair in env_config.split(delimiter):
@@ -109,6 +114,15 @@ def parse_environment_config(env_config: str) -> dict[str, int]:
             category, level = pair.split("=", 1)
             category = category.strip().lower()
             level = level.strip().upper()  # Convert to uppercase for logging._nameToLevel
+
+            if category == "telemetry":
+                # check if telemetry was passed in, if so we should disable the filter below
+                level_value = logging._nameToLevel.get(level)
+                if level_value is not None:
+                    _telemetry_enabled = True
+                    logging.info(f"Telemetry logging enabled at level '{level}'.")
+                else:
+                    logging.warning(f"Unknown telemetry log level '{level}'. Telemetry logging will remain disabled.")
             category_levels.update(config_to_category_levels(category=category, level=level))
 
         except ValueError:
@@ -171,6 +185,23 @@ def filter(self, record):
                 record.category = "uncategorized"  # Default to 'uncategorized' if no category found
             return True
 
+    class TelemetryBlockFilter(logging.Filter):
+        """Block all telemetry logs when telemetry is not enabled."""
+
+        def __init__(self, telemetry_enabled=False):
+            super().__init__()
+            self.telemetry_enabled = telemetry_enabled
+
+        def filter(self, record):
+            # If telemetry is not enabled, block all telemetry logs
+            if not self.telemetry_enabled:
+                # Check if this is a telemetry log by logger name or category
+                if (hasattr(record, "name") and record.name.startswith("telemetry")) or (
+                    hasattr(record, "category") and record.category == "telemetry"
+                ):
+                    return False  # Block this log
+            return True  # Allow all other logs
+
     # Determine the root logger's level (default to WARNING if not specified)
     root_level = category_levels.get("root", logging.WARNING)
 
@@ -182,7 +213,7 @@ def filter(self, record):
             "show_time": False,
             "show_path": False,
             "markup": True,
-            "filters": ["category_filter"],
+            "filters": ["category_filter", "telemetry_block_filter"],
         }
     }
 
@@ -193,6 +224,16 @@ def filter(self, record):
             "filename": log_file,
             "mode": "a",
             "encoding": "utf-8",
+            "filters": ["category_filter", "telemetry_block_filter"],
+        }
+
+    # Build loggers configuration step by step
+    loggers_config = {}
+    for category in CATEGORIES:
+        loggers_config[category] = {
+            "handlers": list(handlers.keys()),  # Apply all handlers
+            "level": category_levels.get(category, DEFAULT_LOG_LEVEL),
+            "propagate": False,  # Disable propagation to root logger
         }
 
     logging_config = {
@@ -208,21 +249,19 @@ def filter(self, record):
         "filters": {
             "category_filter": {
                 "()": CategoryFilter,
-            }
-        },
-        "loggers": {
-            category: {
-                "handlers": list(handlers.keys()),  # Apply all handlers
-                "level": category_levels.get(category, DEFAULT_LOG_LEVEL),
-                "propagate": False,  # Disable propagation to root logger
-            }
-            for category in CATEGORIES
+            },
+            "telemetry_block_filter": {
+                "()": TelemetryBlockFilter,
+                "telemetry_enabled": _telemetry_enabled,
+            },
         },
+        "loggers": loggers_config,
         "root": {
             "handlers": list(handlers.keys()),
             "level": root_level,  # Set root logger's level dynamically
         },
     }
+
     dictConfig(logging_config)
 
     # Ensure third-party libraries follow the root log level
@@ -237,6 +276,7 @@ def get_logger(
     """
     Returns a logger with the specified name and category.
     If no category is provided, defaults to 'uncategorized'.
+    Note: telemetry category is only available when explicitly enabled via LLAMA_STACK_LOGGING=telemetry=LEVEL
 
     Parameters:
         name (str): The name of the logger (e.g., module or filename).

llama_stack/providers/inline/telemetry/meta_reference/console_span_processor.py

@@ -21,9 +21,6 @@ def __init__(self, print_attributes: bool = False):
         self.print_attributes = print_attributes
 
     def on_start(self, span: ReadableSpan, parent_context=None) -> None:
-        if span.attributes and span.attributes.get("__autotraced__"):
-            return
-
         timestamp = datetime.fromtimestamp(span.start_time / 1e9, tz=UTC).strftime("%H:%M:%S.%f")[:-3]
         logger.info(f"[dim]{timestamp}[/dim] [bold magenta][START][/bold magenta] [dim]{span.name}[/dim]")