New metrics automation

README.adoc

@@ -966,6 +966,46 @@ asciidoc:
     - '@redpanda-data/docs-extensions-and-macros/macros/rp-connect-components'
 ```
 
+== CLI Tools
+
+This library provides automated documentation generation tools for writers working with Redpanda documentation.
+
+=== Metrics Documentation
+
+Generate metrics reference documentation for Redpanda:
+
+[,bash]
+----
+# Extract from a specific GitHub tag/branch (recommended)
+npx doc-tools generate source-metrics-docs --tag v25.2.1-rc4
+
+# Extract from a local repository
+npx doc-tools generate metrics-docs --redpanda-repo /path/to/redpanda
+
+# Legacy Docker-based extraction
+npx doc-tools generate metrics-docs-legacy --tag v25.2.1-rc4
+----
+
+All commands generate separate files for internal and external metrics:
+
+* `autogenerated/internal_metrics_reference.adoc` - Internal metrics for engineering teams
+* `autogenerated/public_metrics_reference.adoc` - Public metrics for documentation
+* `autogenerated/metrics.json` - Machine-readable metrics data
+
+=== Other Documentation Tools
+
+* `property-docs`: Generate configuration property documentation
+* `rpk-docs`: Generate RPK CLI documentation  
+* `helm-spec`: Generate Helm chart specifications
+* `crd-spec`: Generate CRD specifications
+
+For complete command options, run:
+
+[,bash]
+----
+npx doc-tools generate --help
+----
+
 == Development quickstart
 
 This section provides information on how to develop this project.

bin/doc-tools.js

@@ -264,6 +264,27 @@ function verifyMetricsDependencies() {
   requireDockerDaemon();
 }
 
+/**
+ * Ensures all dependencies required for generating metrics documentation from source code are installed.
+ *
+ * Checks for the presence of `make`, Python 3.10 or newer, Git, and at least one C++ compiler (`gcc` or `clang`). 
+ * Exits the process with an error message if any dependency is missing.
+ */
+function verifyMetricsExtractorDependencies() {
+  requireCmd('make', 'Your OS package manager');
+  requirePython();
+  requireCmd('git', 'Install Git: https://git-scm.com/downloads');
+  try {
+    execSync('gcc --version', { stdio: 'ignore' });
+  } catch {
+    try {
+      execSync('clang --version', { stdio: 'ignore' });
+    } catch {
+      fail('A C++ compiler (gcc or clang) is required for tree-sitter compilation.');
+    }
+  }
+}
+
 // --------------------------------------------------------------------
 // Main CLI Definition
 // --------------------------------------------------------------------
@@ -394,9 +415,19 @@ const commonOptions = {
 function runClusterDocs(mode, tag, options) {
   const script = path.join(__dirname, '../cli-utils/generate-cluster-docs.sh');
   const args = [mode, tag, options.dockerRepo, options.consoleTag, options.consoleDockerRepo];
-  console.log(`⏳ Running ${script} with arguments: ${args.join(' ')}`);
+
+  console.log(`🚀 Starting cluster (${mode}/${tag})...`);
+
+  const startTime = Date.now();
   const r = spawnSync('bash', [script, ...args], { stdio: 'inherit', shell: true });
-  if (r.status !== 0) process.exit(r.status);
+  const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+
+  if (r.status !== 0) {
+    console.error(`❌ Script failed with exit code ${r.status}`);
+    process.exit(r.status);
+  } else {
+    console.log(`✅ Completed ${mode} docs generation (${duration}s)`);
+  }
 }
 
 // helper to diff two autogenerated directories
@@ -429,7 +460,51 @@ function diffDirs(kind, oldTag, newTag) {
 
 automation
   .command('metrics-docs')
-  .description('Generate JSON and AsciiDoc documentation for Redpanda metrics')
+  .description('Generate JSON and AsciiDoc documentation for Redpanda metrics from source code')
+  .requiredOption('-r, --redpanda-repo <path>', 'Path to the Redpanda repository root directory')
+  .option('--json-output <path>', 'Custom path for JSON output file', 'autogenerated/metrics.json')
+  .option('--internal-asciidoc <path>', 'Custom path for internal metrics AsciiDoc file', 'autogenerated/internal_metrics_reference.adoc')
+  .option('--external-asciidoc <path>', 'Custom path for external/public metrics AsciiDoc file', 'autogenerated/public_metrics_reference.adoc')
+  .action((options) => {
+    console.log(`🎯 Starting enhanced metrics extraction from source code`);
+
+    verifyMetricsExtractorDependencies();
+
+    // Verify Redpanda repository path exists
+    if (!fs.existsSync(options.redpandaRepo)) {
+      console.error(`❌ Redpanda repository path does not exist: ${options.redpandaRepo}`);
+      process.exit(1);
+    }
+
+    console.log(`⏳ Extracting metrics from ${options.redpandaRepo}...`);
+
+    const startTime = Date.now();
+    const result = spawnSync('python3', [
+      path.join(__dirname, '../tools/metrics-extractor/metrics_extractor.py'),
+      '--redpanda-repo', options.redpandaRepo,
+      '--json-output', options.jsonOutput,
+      '--internal-asciidoc', options.internalAsciidoc,
+      '--external-asciidoc', options.externalAsciidoc
+    ], { stdio: 'inherit' });
+
+    const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+
+    if (result.status !== 0) {
+      console.error(`❌ Metrics extraction failed with exit code ${result.status}`);
+      process.exit(result.status);
+    }
+
+    console.log(`✅ Enhanced metrics extraction completed! (${duration}s)`);
+    console.log(`📄 Generated files:`);
+    console.log(`   JSON: ${options.jsonOutput}`);
+    console.log(`   Internal metrics: ${options.internalAsciidoc}`);
+    console.log(`   Public metrics: ${options.externalAsciidoc}`);
+    process.exit(0);
+  });
+
+automation
+  .command('metrics-docs-legacy')
+  .description('Generate JSON and AsciiDoc documentation for Redpanda metrics using Docker cluster (legacy)')
   .requiredOption('-t, --tag <tag>', 'Redpanda version to use when starting Redpanda in Docker')
   .option(
     '--docker-repo <repo>',
@@ -448,6 +523,8 @@ automation
   )
   .option('--diff <oldTag>', 'Also diff autogenerated metrics from <oldTag> → <tag>')
   .action((options) => {
+    console.log(`🎯 Starting legacy metrics docs generation for ${options.tag}`);
+
     verifyMetricsDependencies();
 
     const newTag = options.tag;
@@ -456,18 +533,20 @@ automation
     if (oldTag) {
       const oldDir = path.join('autogenerated', oldTag, 'metrics');
       if (!fs.existsSync(oldDir)) {
-        console.log(`⏳ Generating metrics docs for old tag ${oldTag}…`);
+        console.log(`⏳ Generating metrics docs for old tag ${oldTag}...`);
         runClusterDocs('metrics', oldTag, options);
       }
     }
 
-    console.log(`⏳ Generating metrics docs for new tag ${newTag}…`);
+    console.log(`⏳ Generating metrics docs for new tag ${newTag}...`);
     runClusterDocs('metrics', newTag, options);
 
     if (oldTag) {
+      console.log(`🔄 Diffing ${oldTag} → ${newTag}...`);
       diffDirs('metrics', oldTag, newTag);
     }
 
+    console.log(`✅ Legacy metrics docs generation completed!`);
     process.exit(0);
   });
 
@@ -713,6 +792,41 @@ automation
     process.exit(0);
   });
 
+automation
+  .command('source-metrics-docs')
+  .description('Generate metrics documentation from Redpanda source code using tree-sitter')
+  .option('--tag <tag>', 'Git tag or branch to extract from', 'dev')
+  .option('--diff <oldTag>', 'Also diff autogenerated metrics from <oldTag> → <tag>')
+  .action((options) => {
+    verifyMetricsExtractorDependencies();
+
+    const newTag = options.tag;
+    const oldTag = options.diff;
+    const cwd = path.resolve(__dirname, '../tools/metrics-extractor');
+    const make = (tag) => {
+      console.log(`⏳ Building source-based metrics docs for ${tag}…`);
+      const r = spawnSync('make', ['build', `TAG=${tag}`], { cwd, stdio: 'inherit' });
+      if (r.error) {
+        console.error(`❌ ${r.error.message}`);
+        process.exit(1);
+      }
+      if (r.status !== 0) process.exit(r.status);
+    };
+
+    if (oldTag) {
+      const oldDir = path.join('autogenerated', oldTag, 'source-metrics');
+      if (!fs.existsSync(oldDir)) make(oldTag);
+    }
+
+    make(newTag);
+
+    if (oldTag) {
+      diffDirs('source-metrics', oldTag, newTag);
+    }
+
+    process.exit(0);
+  });
+
 automation
   .command('rpk-docs')
   .description('Generate AsciiDoc documentation for rpk CLI commands')

cli-utils/generate-cluster-docs.sh

@@ -2,6 +2,13 @@
 set -euo pipefail
 IFS=$'\n\t'
 
+# Function to log with timestamp (only for key operations)
+log_step() {
+  echo "[$(date '+%H:%M:%S')] $1" 
+}
+
+log_step "🚀 Starting cluster setup..."
+
 ###############################################################################
 # Pre-flight: Ensure Docker is available and running
 ###############################################################################
@@ -15,6 +22,11 @@ if ! docker info &> /dev/null; then
   exit 1
 fi
 
+if ! command -v curl &> /dev/null; then
+  echo "❌ curl is not installed or not in PATH. Please install curl to continue."
+  exit 1
+fi
+
 ###############################################################################
 # Load overrides from an optional .env file in the current directory
 ###############################################################################
@@ -56,40 +68,63 @@ export REDPANDA_CONSOLE_DOCKER_REPO="$CONSOLE_REPO"
 ###############################################################################
 # Start Redpanda cluster
 ###############################################################################
+log_step "� Starting Redpanda cluster..."
 "$SCRIPT_DIR/start-cluster.sh" "$TAG"
 
 # Wait for the cluster to settle
 if [[ "$MODE" == "metrics" ]]; then
-  echo "⏳ Waiting 300 seconds for metrics to be available…"
-  sleep 300
+  log_step "⏳ Waiting for metrics endpoint..."
+
+  # Wait for metrics endpoint to be responsive
+  timeout=300
+  counter=0
+  metrics_url="http://localhost:19644/public_metrics/"
+
+  while ! curl -f -s "$metrics_url" > /dev/null 2>&1; do
+    if [ $counter -ge $timeout ]; then
+      echo "❌ Metrics endpoint did not become ready within ${timeout}s"
+      exit 1
+    fi
+    sleep 10
+    counter=$((counter + 10))
+  done
+
+  log_step "✅ Metrics endpoint ready"
 else
-  echo "⏳ Waiting 30 seconds for cluster to be ready…"
   sleep 30
 fi
 
 ###############################################################################
 # Python virtual environment setup
 ###############################################################################
+log_step "🐍 Setting up Python environment..."
 "$SCRIPT_DIR/python-venv.sh" \
   "$SCRIPT_DIR/venv" \
-  "$SCRIPT_DIR/../tools/metrics/requirements.txt"
+  "$SCRIPT_DIR/../tools/metrics-extractor/requirements.txt"
 
 ###############################################################################
 # Run documentation generator
 ###############################################################################
+log_step "📝 Generating $MODE documentation..."
+
 if [[ "$MODE" == "metrics" ]]; then
+  # Use enhanced metrics extractor with separate internal/external docs
   "$SCRIPT_DIR/venv/bin/python" \
-    "$SCRIPT_DIR/../tools/metrics/metrics.py" "$TAG"
+    "$SCRIPT_DIR/../tools/metrics-extractor/metrics_extractor.py" \
+    --json-output "autogenerated/${TAG}/metrics/metrics.json" \
+    --internal-asciidoc "autogenerated/${TAG}/metrics/internal_metrics_reference.adoc" \
+    --external-asciidoc "autogenerated/${TAG}/metrics/public_metrics_reference.adoc"
 else
   "$SCRIPT_DIR/venv/bin/python" \
     "$SCRIPT_DIR/../tools/gen-rpk-ascii.py" "$TAG"
 fi
 
-echo "✅ $MODE docs generated successfully!"
+log_step "✅ Documentation generated successfully"
 
 # Tear down the cluster
+log_step "🧹 Cleaning up cluster..."
 cd "$SCRIPT_DIR"/../docker-compose
-docker compose -p "$PROJECT_NAME" down --volumes
+docker compose -p "$PROJECT_NAME" down --volumes > /dev/null 2>&1
 
 # Return to the original directory
 cd "$ORIGINAL_PWD" || exit 1

cli-utils/install-test-dependencies.sh

@@ -13,7 +13,7 @@ install_node() {
     eval "$(fnm env)" || { echo "Failed to load fnm environment"; exit 1; }
     fnm install --lts || { echo "Failed to install Node.js"; exit 1; }
     fnm use --lts || { echo "Failed to use Node.js"; exit 1; }
-    echo "Node.js version: $(node -v)"
+    echo "Node.js version: $(node -v)" 
   fi
 }
 

cli-utils/python-venv.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 VENV_DIR="${1:-venv}"
 REQ_FILE="${2:-requirements.txt}"
 
-echo "Recreating Python venv at $VENV_DIR..."
+echo "Recreating Python venv at $VENV_DIR..." 
 rm -rf "$VENV_DIR"
 python3 -m venv "$VENV_DIR"
 "$VENV_DIR/bin/pip" install --upgrade pip --quiet