Skip to content

Commit 6b0ba87

Browse files
ascherjascherj
committed
feat: add comprehensive observability and metrics monitoring system
Add health and metrics commands for individual and collective services: - health: detailed health checks with dependency verification - metrics: p50/p95/p99 latency tracking across all services - metrics-summary: quick demo-ready system overview - watch-metrics: real-time monitoring dashboard - system-status: complete health and performance snapshot - demo: automated E2E system showcase script Add performance targets and SLO documentation: - Webhook response <100ms target, >500ms alert - E2E processing <5s target, >10s alert - Classification success >95% target, <90% alert - API response <200ms target, >1s alert - Consumer lag = 0 target, >100 for 5min alert Fix service port mappings and echo compatibility: - Correct ports: ingress:8000, classifier:8001, gateway:8002 - Replace echo -n with printf for cross-platform compatibility - Fix webhook payload with all required fields Production-ready features for interview demonstrations
1 parent 8727eb0 commit 6b0ba87

3 files changed

Lines changed: 691 additions & 14 deletions

File tree

‎Makefile‎

Lines changed: 114 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# DispatchAI Development Makefile
22
# Provides targets for development, testing, linting, and deployment
33

4-
.PHONY: help dev dev-up dev-down dev-logs dev-clean test test-docker test-ingress test-classifier test-gateway test-dashboard test-classifier-docker test-gateway-docker test-dashboard-docker lint lint-python lint-python-docker lint-js clean build setup-env check-env check-containers test-webhook-manual kafka-tail
4+
.PHONY: help dev dev-up dev-down dev-logs dev-clean test test-docker test-ingress test-classifier test-gateway test-dashboard test-classifier-docker test-gateway-docker test-dashboard-docker lint lint-python lint-python-docker lint-js clean build setup-env check-env check-containers test-webhook-manual demo kafka-tail health health-ingress health-classifier health-gateway metrics metrics-ingress metrics-classifier metrics-gateway metrics-summary watch-metrics system-status
55

66
# Default target
77
help: ## Show this help message
@@ -48,24 +48,116 @@ dev-clean: ## Stop services and remove volumes/networks
4848
@echo "Pruning unused Docker resources..."
4949
docker system prune -f
5050

51-
# Health Checks
51+
# Health Checks & Metrics
5252
check-services: ## Check if all services are healthy
5353
@echo "Checking service health..."
54-
@echo -n "PostgreSQL: "
55-
@docker exec dispatchai-postgres pg_isready -U postgres >/dev/null 2>&1 && echo "✅ Ready" || echo "❌ Not Ready"
56-
@echo -n "Redpanda: "
57-
@curl -s http://localhost:9644/v1/cluster/health >/dev/null 2>&1 && echo "✅ Ready" || echo "❌ Not Ready"
54+
@printf "PostgreSQL: " && docker exec dispatchai-postgres pg_isready -U postgres >/dev/null 2>&1 && echo "✅ Ready" || echo "❌ Not Ready"
55+
@printf "Redpanda: " && curl -s http://localhost:9644/v1/cluster/health >/dev/null 2>&1 && echo "✅ Ready" || echo "❌ Not Ready"
5856

5957
check-containers: ## Check if all containers are running
6058
@echo "Checking containers..."
61-
@echo -n "Ingress: "
62-
@docker ps --format "table {{.Names}}" | grep -q "dispatchai-ingress" && echo "✅ Running" || echo "❌ Not Running"
63-
@echo -n "Classifier: "
64-
@docker ps --format "table {{.Names}}" | grep -q "dispatchai-classifier" && echo "✅ Running" || echo "❌ Not Running"
65-
@echo -n "Gateway: "
66-
@docker ps --format "table {{.Names}}" | grep -q "dispatchai-gateway" && echo "✅ Running" || echo "❌ Not Running"
67-
@echo -n "Dashboard: "
68-
@docker ps --format "table {{.Names}}" | grep -q "dispatchai-dashboard" && echo "✅ Running" || echo "❌ Not Running"
59+
@printf "Ingress: " && docker ps --format "table {{.Names}}" | grep -q "dispatchai-ingress" && echo "✅ Running" || echo "❌ Not Running"
60+
@printf "Classifier: " && docker ps --format "table {{.Names}}" | grep -q "dispatchai-classifier" && echo "✅ Running" || echo "❌ Not Running"
61+
@printf "Gateway: " && docker ps --format "table {{.Names}}" | grep -q "dispatchai-gateway" && echo "✅ Running" || echo "❌ Not Running"
62+
@printf "Dashboard: " && docker ps --format "table {{.Names}}" | grep -q "dispatchai-dashboard" && echo "✅ Running" || echo "❌ Not Running"
63+
64+
health: ## Show detailed health status of all services
65+
@echo "======================================"
66+
@echo "DispatchAI System Health Report"
67+
@echo "======================================"
68+
@echo ""
69+
@echo "📊 Ingress Service (Port 8000):"
70+
@curl -s http://localhost:8000/health 2>/dev/null | python3 -m json.tool || echo "❌ Service unreachable"
71+
@echo ""
72+
@echo "🤖 Classifier Service (Port 8001):"
73+
@curl -s http://localhost:8001/health 2>/dev/null | python3 -m json.tool || echo "❌ Service unreachable"
74+
@echo ""
75+
@echo "🔗 Gateway Service (Port 8002):"
76+
@curl -s http://localhost:8002/health 2>/dev/null | python3 -m json.tool || echo "❌ Service unreachable"
77+
@echo ""
78+
79+
health-ingress: ## Show health status of ingress service
80+
@echo "Ingress Service Health:"
81+
@curl -s http://localhost:8000/health | python3 -m json.tool
82+
83+
health-classifier: ## Show health status of classifier service
84+
@echo "Classifier Service Health:"
85+
@curl -s http://localhost:8001/health | python3 -m json.tool
86+
87+
health-gateway: ## Show health status of gateway service
88+
@echo "Gateway Service Health:"
89+
@curl -s http://localhost:8002/health | python3 -m json.tool
90+
91+
metrics: ## Show comprehensive metrics from all services
92+
@echo "======================================"
93+
@echo "DispatchAI System Metrics Report"
94+
@echo "======================================"
95+
@echo ""
96+
@echo "📊 Ingress Service Metrics:"
97+
@curl -s http://localhost:8000/metrics 2>/dev/null | python3 -m json.tool || echo "❌ Metrics unavailable"
98+
@echo ""
99+
@echo "🤖 Classifier Service Metrics:"
100+
@curl -s http://localhost:8001/metrics 2>/dev/null | python3 -m json.tool || echo "❌ Metrics unavailable"
101+
@echo ""
102+
@echo "🔗 Gateway Service Metrics:"
103+
@curl -s http://localhost:8002/metrics 2>/dev/null | python3 -m json.tool || echo "❌ Metrics unavailable"
104+
@echo ""
105+
@echo "📡 Kafka Consumer Lag:"
106+
@docker exec dispatchai-redpanda rpk group describe dispatchai-classifier 2>/dev/null || echo "❌ Consumer group not found"
107+
@echo ""
108+
109+
metrics-ingress: ## Show metrics for ingress service
110+
@echo "Ingress Service Metrics:"
111+
@curl -s http://localhost:8000/metrics | python3 -m json.tool
112+
113+
metrics-classifier: ## Show metrics for classifier service
114+
@echo "Classifier Service Metrics:"
115+
@curl -s http://localhost:8001/metrics | python3 -m json.tool
116+
117+
metrics-gateway: ## Show metrics for gateway service
118+
@echo "Gateway Service Metrics:"
119+
@curl -s http://localhost:8002/metrics | python3 -m json.tool
120+
121+
metrics-summary: ## Show quick metrics summary (perfect for demos)
122+
@echo "======================================"
123+
@echo "📊 DispatchAI Metrics Summary"
124+
@echo "======================================"
125+
@echo ""
126+
@echo "Ingress (Webhook Processing):"
127+
@curl -s http://localhost:8000/metrics 2>/dev/null | python3 -c "import sys, json; d=json.load(sys.stdin); print(f\" Webhooks Received: {d.get('webhooks_received', 0)}\"); print(f\" Webhooks Accepted: {d.get('webhooks_accepted', 0)}\"); print(f\" Processing Time (p95): {d.get('processing_time_ms', {}).get('p95', 'N/A')}ms\"); print(f\" Uptime: {d.get('uptime_seconds', 0)}s\")" || echo " ❌ Unavailable"
128+
@echo ""
129+
@echo "Classifier (AI Processing):"
130+
@curl -s http://localhost:8001/metrics 2>/dev/null | python3 -c "import sys, json; d=json.load(sys.stdin); print(f\" Issues Processed: {d.get('issues_processed', 0)}\"); print(f\" Classification Time (p95): {d.get('classification_time_ms', {}).get('p95', 'N/A')}ms\"); print(f\" OpenAI Errors: {d.get('openai_api_errors', 0)}\"); print(f\" Consumer Lag: {d.get('consumer_lag', 0)}\"); print(f\" Uptime: {d.get('uptime_seconds', 0)}s\")" || echo " ❌ Unavailable"
131+
@echo ""
132+
@echo "Gateway (WebSocket Streaming):"
133+
@curl -s http://localhost:8002/metrics 2>/dev/null | python3 -c "import sys, json; d=json.load(sys.stdin); print(f\" Active WebSocket Connections: {d.get('websocket_connections_active', 0)}\"); print(f\" Messages Broadcast: {d.get('messages_broadcast', 0)}\"); print(f\" API Requests: {d.get('api_requests', 0)}\"); print(f\" Uptime: {d.get('uptime_seconds', 0)}s\")" || echo " ❌ Unavailable"
134+
@echo ""
135+
136+
watch-metrics: ## Watch metrics in real-time (refreshes every 3 seconds)
137+
@echo "Watching metrics (Ctrl+C to stop)..."
138+
@while true; do \
139+
clear; \
140+
make metrics-summary; \
141+
sleep 3; \
142+
done
143+
144+
system-status: ## Show complete system status (containers + health + metrics)
145+
@echo "======================================"
146+
@echo "🚀 DispatchAI System Status Report"
147+
@echo "======================================"
148+
@echo ""
149+
@echo "📦 Container Status:"
150+
@make check-containers
151+
@echo ""
152+
@echo "💚 Service Health:"
153+
@make check-services
154+
@echo ""
155+
@echo "📈 Performance Metrics:"
156+
@make metrics-summary
157+
@echo ""
158+
@echo "======================================"
159+
@echo "✅ System Status Report Complete"
160+
@echo "======================================"
69161

70162
# Testing
71163
test: ## Run all tests (requires running containers)
@@ -176,6 +268,14 @@ test-webhook-manual: ## Run manual webhook testing script
176268
echo "❌ Manual test script not found at scripts/test-webhook-manual.sh"; \
177269
fi
178270

271+
demo: ## Run comprehensive system showcase demo (perfect for interviews!)
272+
@echo "🎯 Running DispatchAI System Showcase Demo..."
273+
@if [ -f scripts/demo-system-showcase.sh ]; then \
274+
./scripts/demo-system-showcase.sh; \
275+
else \
276+
echo "❌ Demo script not found at scripts/demo-system-showcase.sh"; \
277+
fi
278+
179279
# Linting
180280
lint: ## Run all linters
181281
@echo "Running all linters..."

0 commit comments

Comments
 (0)