Skip to content

Commit 4fcba62

Browse files
committed
feat(gpu): add NVIDIA GPU power monitoring via NVML
Add experimental GPU power monitoring support: - NVIDIA backend using go-nvml for device discovery and power readings - GPU sharing mode detection (time-slicing, exclusive, partitioned) - Per-process power attribution based on compute utilization - experimental.gpu.enabled config flag (disabled by default) - added tests Sharing mode detection: - Partitioned (MIG): GPU partitioned into isolated instances - Exclusive: Single process has exclusive GPU access (nvidia-smi -c EXCLUSIVE_PROCESS) - Time-slicing: Multiple processes share GPU (default mode) Detection logic: 1. Check if MIG enabled → Partitioned 2. Check NVML compute mode: - EXCLUSIVE_PROCESS/THREAD → Exclusive - DEFAULT → Time-slicing Includes: - nvidia/collector.go: GPUPowerMeter implementation - nvidia/detector.go: sharing mode detection - nvidia/nvml.go: NVML wrapper with MIG support - nvidia/types.go: NVIDIA-specific ComputeMode type - Dockerfile: enable CGO for NVML - Manifests: mount nvidia driver libs, set LD_LIBRARY_PATH Signed-off-by: Vimal Kumar <vimal78@gmail.com>
1 parent dfdd46a commit 4fcba62

23 files changed

Lines changed: 3730 additions & 1 deletion

File tree

Dockerfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ WORKDIR /workspace
1111
COPY . .
1212

1313
RUN make build \
14+
CGO_ENABLED=1 \
1415
PRODUCTION=1 \
1516
VERSION=${VERSION} \
1617
GIT_COMMIT=${GIT_COMMIT} \

cmd/kepler/main.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414

1515
"github.com/sustainable-computing-io/kepler/config"
1616
"github.com/sustainable-computing-io/kepler/internal/device"
17+
"github.com/sustainable-computing-io/kepler/internal/device/gpu"
1718
"github.com/sustainable-computing-io/kepler/internal/exporter/prometheus"
1819
"github.com/sustainable-computing-io/kepler/internal/exporter/stdout"
1920
"github.com/sustainable-computing-io/kepler/internal/k8s/pod"
@@ -24,6 +25,9 @@ import (
2425
"github.com/sustainable-computing-io/kepler/internal/server"
2526
"github.com/sustainable-computing-io/kepler/internal/service"
2627
"github.com/sustainable-computing-io/kepler/internal/version"
28+
29+
// Register GPU backends via init()
30+
_ "github.com/sustainable-computing-io/kepler/internal/device/gpu/nvidia"
2731
)
2832

2933
func main() {
@@ -187,6 +191,22 @@ func createServices(logger *slog.Logger, cfg *config.Config) ([]service.Service,
187191
}
188192
}
189193

194+
// Discover GPU devices if enabled (experimental feature)
195+
if cfg.IsFeatureEnabled(config.ExperimentalGPUFeature) {
196+
gpuMeters := gpu.DiscoverAll(logger)
197+
if len(gpuMeters) > 0 {
198+
logger.Info("GPU power monitoring enabled",
199+
"vendors", len(gpuMeters),
200+
"registered_vendors", gpu.RegisteredVendors())
201+
// GPU meters are initialized by DiscoverAll, store for future use
202+
// TODO: integrate with monitor for per-process power attribution
203+
_ = gpuMeters
204+
} else {
205+
logger.Warn("GPU power monitoring enabled but no GPUs discovered",
206+
"registered_vendors", gpu.RegisteredVendors())
207+
}
208+
}
209+
190210
// Add Prometheus exporter if enabled
191211
if cfg.IsFeatureEnabled(config.PrometheusFeature) {
192212
promExporter, err := createPrometheusExporter(logger, cfg, apiServer, pm, redfishService)

codecov.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# SPDX-FileCopyrightText: 2025 The Kepler Authors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
coverage:
5+
status:
6+
project:
7+
default:
8+
threshold: 1%
9+
patch:
10+
default:
11+
target: 70%
12+
13+
ignore:
14+
- internal/device/gpu/nvidia/nvml_lib.go

compose/default/kepler/etc/kepler/config.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,5 @@ experimental:
8383
hwmon:
8484
enabled: false # Enable experimental hwmon power monitoring
8585
zones: [] # List of zones to enable (default enable all)
86+
gpu:
87+
enabled: false # Enable experimental GPU power monitoring

compose/dev/kepler-dev/etc/kepler/config.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,5 @@ experimental:
8383
hwmon:
8484
enabled: false # Enable experimental hwmon power monitoring
8585
zones: [] # List of zones to enable (default enable all)
86+
gpu:
87+
enabled: false # Enable experimental GPU power monitoring

config/config.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,9 @@ const (
255255
ExperimentalHwmonEnabledFlag = "experimental.hwmon.enabled"
256256
ExperimentalHwmonZonesFlag = "experimental.hwmon.zones"
257257

258+
// Experimental GPU flags
259+
ExperimentalGPUEnabledFlag = "experimental.gpu.enabled"
260+
258261
// WARN: dev settings shouldn't be exposed as flags as flags are intended for end users
259262
)
260263

@@ -409,6 +412,9 @@ func RegisterFlags(app *kingpin.Application) ConfigUpdaterFn {
409412
hwmonEnabled := app.Flag(ExperimentalHwmonEnabledFlag, "Enable experimental hwmon power monitoring").Default("false").Bool()
410413
hwmonZones := app.Flag(ExperimentalHwmonZonesFlag, "Hwmon zone filter (power labels to monitor)").Strings()
411414

415+
// experimental GPU
416+
gpuEnabled := app.Flag(ExperimentalGPUEnabledFlag, "Enable experimental GPU power monitoring").Default("false").Bool()
417+
412418
return func(cfg *Config) error {
413419
// Logging settings
414420
if flagsSet[LogLevelFlag] {
@@ -481,6 +487,9 @@ func RegisterFlags(app *kingpin.Application) ConfigUpdaterFn {
481487
return err
482488
}
483489

490+
// Apply experimental GPU settings
491+
applyGPUConfig(cfg, flagsSet, gpuEnabled)
492+
484493
cfg.sanitize()
485494
return cfg.Validate()
486495
}
@@ -604,6 +613,28 @@ func applyHwmonFlags(hwmon *Hwmon, flagsSet map[string]bool, enabled *bool, zone
604613
}
605614
}
606615

616+
// applyGPUConfig applies GPU configuration from flags
617+
func applyGPUConfig(cfg *Config, flagsSet map[string]bool, enabled *bool) {
618+
// Early exit if no GPU flags are set and config file does not have experimental section
619+
if !hasGPUFlags(flagsSet) && cfg.Experimental == nil {
620+
return
621+
}
622+
623+
// Initialize experimental section if needed
624+
if cfg.Experimental == nil {
625+
cfg.Experimental = &Experimental{}
626+
}
627+
628+
if flagsSet[ExperimentalGPUEnabledFlag] {
629+
cfg.Experimental.GPU.Enabled = enabled
630+
}
631+
}
632+
633+
// hasGPUFlags returns true if any GPU experimental flags are set
634+
func hasGPUFlags(flagsSet map[string]bool) bool {
635+
return flagsSet[ExperimentalGPUEnabledFlag]
636+
}
637+
607638
// resolveNodeName resolves the node name using the following precedence:
608639
// 1. CLI flag / config.yaml (--experimental.platform.redfish.node-name)
609640
// 2. Kubernetes node name

docs/user/configuration.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ You can configure Kepler by passing flags when starting the service. The followi
3838
| `--experimental.platform.redfish.config-file` | Path to experimental Redfish BMC configuration file | `""` | Any valid file path |
3939
| `--experimental.hwmon.enabled` | Enable experimental hwmon power monitoring | `false` | `true`, `false` |
4040
| `--experimental.hwmon.zones` | hwmon zones to be enabled (can be specified multiple times) | All available zones | Any valid hwmon zone name |
41+
| `--experimental.gpu.enabled` | Enable experimental GPU power monitoring | `false` | `true`, `false` |
4142

4243
### 💡 Examples
4344

@@ -73,6 +74,9 @@ kepler --experimental.hwmon.enabled=true \
7374
--experimental.hwmon.zones=power1 \
7475
--experimental.hwmon.zones=power2
7576

77+
# Enable experimental GPU power monitoring
78+
kepler --experimental.gpu.enabled=true
79+
7680
# Export only node and container level metrics
7781
kepler --metrics=node --metrics=container
7882

@@ -153,6 +157,8 @@ experimental: # experimental features (no stability guarantees)
153157
hwmon: # hwmon power monitoring
154158
enabled: false # Enable hwmon power monitoring (default: false)
155159
zones: [] # hwmon zones to be enabled, empty enables all available zones
160+
gpu: # GPU power monitoring
161+
enabled: false # Enable GPU power monitoring (default: false)
156162

157163
# WARN: DO NOT ENABLE THIS IN PRODUCTION - for development/testing only
158164
dev:
@@ -328,6 +334,8 @@ experimental:
328334
hwmon:
329335
enabled: false
330336
zones: []
337+
gpu:
338+
enabled: false
331339
```
332340

333341
⚠️ **WARNING**: This section contains experimental features with no stability guarantees.
@@ -394,6 +402,21 @@ experimental:
394402
zones: ["power1", "power2"]
395403
```
396404

405+
#### GPU Power Monitoring
406+
407+
- **enabled**: Enable experimental GPU power monitoring (default: false)
408+
- When enabled, Kepler will collect power metrics from NVIDIA GPUs using NVML
409+
- Requires NVIDIA drivers and NVML library to be available
410+
- Supports per-process power attribution based on GPU compute utilization
411+
412+
**Example:**
413+
414+
```yaml
415+
experimental:
416+
gpu:
417+
enabled: true
418+
```
419+
397420
### 🧑‍🔬 Development Configuration
398421

399422
```yaml

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ toolchain go1.24.9
66

77
require (
88
dario.cat/mergo v1.0.2
9+
github.com/NVIDIA/go-nvml v0.13.0-1
910
github.com/alecthomas/kingpin/v2 v2.4.0
1011
github.com/go-logr/logr v1.4.2
1112
github.com/oklog/run v1.1.0

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
22
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
3+
github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw=
4+
github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4=
35
github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY=
46
github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
57
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=

hack/config.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,5 @@ experimental:
8383
hwmon:
8484
enabled: false # Enable experimental hwmon power monitoring
8585
zones: [] # List of zones to enable (default enable all)
86+
gpu:
87+
enabled: false # Enable experimental GPU power monitoring

0 commit comments

Comments
 (0)