Skip to content

[Metrics] Add GPU launched metrics #7987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 17 commits into
base: master
Choose a base branch
from
+899 −11
Draft

[Metrics] Add GPU launched metrics #7987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
027bfb1
[Metrics] Add GPU launched per month in API server metrics
Michaelvll Nov 15, 2025
435d830
fix panels
Michaelvll Nov 15, 2025
9b63e8a
update panels
Michaelvll Nov 16, 2025
843e501
update the dashboard
Michaelvll Nov 16, 2025
f6cd5e1
update GPU logic
Michaelvll Nov 16, 2025
0b177e3
update metrics
Michaelvll Nov 16, 2025
dc0c928
update the grafana dashboard
Michaelvll Nov 16, 2025
445af10
Turn on metrics for jobs controller
Michaelvll Nov 16, 2025
a8a5233
update
Michaelvll Nov 16, 2025
b5dbb93
move variable name
Michaelvll Nov 16, 2025
57bcda3
fix metrics name
Michaelvll Nov 16, 2025
010899f
Fix GPU metrics tracking
Michaelvll Nov 16, 2025
b8bb799
move metrics to backend instead
Michaelvll Nov 16, 2025
ce7eb0d
Use DB for the daily GPU launches
Michaelvll Nov 16, 2025
dc06c1c
Get rid of other GPU panels
Michaelvll Nov 16, 2025
521b0d5
only keep the per-day user GPUs
Michaelvll Nov 16, 2025
5379221
Add `sky statistics` command
Michaelvll Nov 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
307 changes: 297 additions & 10 deletions charts/skypilot/manifests/api-server-overview.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1492,6 +1492,265 @@
"x": 0,
"y": 47
},
"id": 35,
"panels": [],
"title": "GPU Usage",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "GPUs"
},
"properties": [
{
"id": "custom.width",
"value": 100
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 48
},
"id": 37,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"enablePagination": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "GPUs"
}
]
},
"pluginVersion": "12.0.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(sky_apiserver_daily_gpu_launches{date=~\"$date_filter\"}) by (username)",
"format": "table",
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "A"
}
],
"title": "Daily GPU Launches per User",
"transformations": [
{
"id": "groupBy",
"options": {
"fields": {
"Value": {
"aggregations": [
"sum"
],
"operation": "aggregate"
},
"username": {
"aggregations": [],
"operation": "groupby"
}
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"includeByName": {},
"indexByName": {
"Value (sum)": 1,
"username": 0
},
"renameByName": {
"Value (sum)": "GPUs",
"username": "User"
}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "GPUs"
},
"properties": [
{
"id": "custom.width",
"value": 100
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 48
},
"id": 31,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"enablePagination": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "GPUs"
}
]
},
"pluginVersion": "12.0.1",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(sky_apiserver_daily_gpu_launches{date=~\"$date_filter\"}) by (username, accelerator_type)",
"format": "table",
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "A"
}
],
"title": "Daily GPU Launches per User per Accelerator Type",
"transformations": [
{
"id": "groupBy",
"options": {
"fields": {
"Value": {
"aggregations": [
"sum"
],
"operation": "aggregate"
},
"accelerator_type": {
"aggregations": [],
"operation": "groupby"
},
"username": {
"aggregations": [],
"operation": "groupby"
}
}
}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"includeByName": {},
"indexByName": {
"Value (sum)": 2,
"accelerator_type": 1,
"username": 0
},
"renameByName": {
"Value (sum)": "GPUs",
"accelerator_type": "Accelerator",
"username": "User"
}
}
}
],
"type": "table"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 56
},
"id": 17,
"panels": [],
"title": "Internals",
Expand Down Expand Up @@ -1561,7 +1820,7 @@
"h": 7,
"w": 8,
"x": 0,
"y": 48
"y": 57
},
"id": 7,
"options": {
Expand Down Expand Up @@ -1660,7 +1919,7 @@
"h": 7,
"w": 8,
"x": 8,
"y": 48
"y": 65
},
"id": 20,
"options": {
Expand Down Expand Up @@ -1759,7 +2018,7 @@
"h": 7,
"w": 8,
"x": 16,
"y": 48
"y": 65
},
"id": 21,
"options": {
Expand Down Expand Up @@ -1858,7 +2117,7 @@
"h": 7,
"w": 8,
"x": 0,
"y": 55
"y": 72
},
"id": 6,
"options": {
Expand Down Expand Up @@ -1957,7 +2216,7 @@
"h": 7,
"w": 8,
"x": 8,
"y": 55
"y": 72
},
"id": 5,
"options": {
Expand Down Expand Up @@ -2056,7 +2315,7 @@
"h": 7,
"w": 8,
"x": 16,
"y": 55
"y": 72
},
"id": 22,
"options": {
Expand Down Expand Up @@ -2155,7 +2414,7 @@
"h": 7,
"w": 8,
"x": 0,
"y": 62
"y": 79
},
"id": 10,
"options": {
Expand Down Expand Up @@ -2254,7 +2513,7 @@
"h": 7,
"w": 8,
"x": 8,
"y": 62
"y": 79
},
"id": 9,
"options": {
Expand Down Expand Up @@ -2353,7 +2612,7 @@
"h": 7,
"w": 8,
"x": 16,
"y": 62
"y": 79
},
"id": 23,
"options": {
Expand Down Expand Up @@ -2508,6 +2767,34 @@
"refresh": 1,
"regex": "",
"type": "query"
},
{
"allValue": ".+",
"current": {
"text": "All",
"value": [
"$__all"
]
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(sky_apiserver_daily_gpu_launches,date)",
"includeAll": true,
"label": "Date Range",
"multi": true,
"name": "date_filter",
"options": [],
"query": {
"qryType": 1,
"query": "label_values(sky_apiserver_daily_gpu_launches,date)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"sort": 1,
"type": "query"
}
]
},
Expand All @@ -2519,5 +2806,5 @@
"timezone": "browser",
"title": "SkyPilot API Server",
"uid": "skypilot-apiserver-overview",
"version": 1
"version": 11
}
Loading
Loading