Skip to content

Commit e1ab6fe

Browse files
committed
cpu: implement C-state enabling and disabling
1 parent a0a94ae commit e1ab6fe

File tree

3 files changed

+223
-0
lines changed

3 files changed

+223
-0
lines changed

pkg/resmgr/control/cpu/api.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ func Assign(c cache.Cache, class string, cpus ...int) error {
6161
if err := ctl.enforceCpufreq(class, cpus...); err != nil {
6262
log.Error("cpufreq enforcement failed: %v", err)
6363
}
64+
if err := ctl.enforceCstates(class, cpus...); err != nil {
65+
log.Error("cstate enforcement failed: %v", err)
66+
}
6467
if err := ctl.enforceUncore(assignments, cpus...); err != nil {
6568
log.Error("uncore frequency enforcement failed: %v", err)
6669
}

pkg/resmgr/control/cpu/cpu.go

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/containers/nri-plugins/pkg/resmgr/cache"
2626
"github.com/containers/nri-plugins/pkg/resmgr/control"
2727
"github.com/containers/nri-plugins/pkg/sysfs"
28+
"github.com/intel/goresctrl/pkg/cstates"
2829
"github.com/intel/goresctrl/pkg/utils"
2930
)
3031

@@ -41,6 +42,7 @@ type cpuctl struct {
4142
cache cache.Cache // resource manager cache
4243
system sysfs.System // system topology
4344
classes map[string]Class // configured CPU classes
45+
cstates *cstates.Cstates // C-states handler
4446
uncoreEnabled bool // whether we need to care about uncore
4547
started bool
4648
}
@@ -153,6 +155,43 @@ func (ctl *cpuctl) enforceCpufreq(class string, cpus ...int) error {
153155
return nil
154156
}
155157

158+
// enforceStates enforces a class-specific C-state configuration to a cpuset
159+
func (ctl *cpuctl) enforceCstates(class string, cpus ...int) error {
160+
c, ok := ctl.classes[class]
161+
if !ok {
162+
return fmt.Errorf("non-existent cpu class %q", class)
163+
}
164+
if ctl.cstates == nil || len(cpus) == 0 {
165+
return nil
166+
}
167+
enabledCstates := []string{}
168+
for _, name := range ctl.cstates.Names() {
169+
enabled := true
170+
for _, dname := range c.DisabledCstates {
171+
if name == dname {
172+
enabled = false
173+
break
174+
}
175+
}
176+
if enabled {
177+
enabledCstates = append(enabledCstates, name)
178+
}
179+
}
180+
cpuCstates := ctl.cstates.Copy(cstates.FilterCPUs(cpus...))
181+
enCpuCstates := cpuCstates.Copy(cstates.FilterNames(enabledCstates...))
182+
disCpuCstates := cpuCstates.Copy(cstates.FilterNames(c.DisabledCstates...))
183+
enCpuCstates.SetAttrs(cstates.DISABLE, "0")
184+
disCpuCstates.SetAttrs(cstates.DISABLE, "1")
185+
log.Debug("enforcing cstates: enable: %v disable: %v from class %q on cpus %v", enabledCstates, c.DisabledCstates, class, cpus)
186+
if err := enCpuCstates.Apply(); err != nil {
187+
return fmt.Errorf("cannot enable cstates %v on cpus %v: %w", enabledCstates, cpus, err)
188+
}
189+
if err := disCpuCstates.Apply(); err != nil {
190+
return fmt.Errorf("cannot disable cstates %v on cpus %v: %w", c.DisabledCstates, cpus, err)
191+
}
192+
return nil
193+
}
194+
156195
// enforceUncore enforces uncore frequency limits
157196
func (ctl *cpuctl) enforceUncore(assignments cpuClassAssignments, affectedCPUs ...int) error {
158197
if !ctl.uncoreEnabled {
@@ -251,6 +290,7 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error {
251290
log.Debug("applying cpu controller configuration:\n%s", utils.DumpJSON(ctl.classes))
252291

253292
// Sanity check
293+
cstatesNeeded := map[string]struct{}{}
254294
uncoreAvailable := utils.UncoreFreqAvailable()
255295
for name, conf := range ctl.classes {
256296
if conf.UncoreMinFreq != 0 || conf.UncoreMaxFreq != 0 {
@@ -260,6 +300,21 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error {
260300
ctl.uncoreEnabled = true
261301
break
262302
}
303+
for _, cstate := range conf.DisabledCstates {
304+
cstatesNeeded[cstate] = struct{}{}
305+
}
306+
}
307+
if len(cstatesNeeded) != 0 {
308+
var err error
309+
filter := cstates.FilterAttrs(cstates.DISABLE)
310+
if cstatesEnvOverridesJson != "" {
311+
ctl.cstates, err = NewCstatesFromOverride(filter)
312+
} else {
313+
ctl.cstates, err = cstates.NewCstatesFromSysfs(filter)
314+
}
315+
if err != nil {
316+
return fmt.Errorf("failed to read C-states: %w", err)
317+
}
263318
}
264319

265320
// Configure the system
@@ -269,6 +324,9 @@ func (ctl *cpuctl) configure(cfg *cfgapi.Config) error {
269324
if err := ctl.enforceCpufreq(class, cpus.SortedMembers()...); err != nil {
270325
log.Error("cpufreq enforcement on re-configure failed: %v", err)
271326
}
327+
if err := ctl.enforceCstates(class, cpus.SortedMembers()...); err != nil {
328+
log.Error("cpufreq enforcement on re-configure failed: %v", err)
329+
}
272330
} else {
273331
// TODO: what should we really do with classes that do not exist in
274332
// the configuration anymore? Now we remember the CPUs assigned to
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
// Copyright 2025 Intel Corporation. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package cpu
16+
17+
import (
18+
"encoding/json"
19+
"fmt"
20+
"maps"
21+
"os"
22+
"slices"
23+
"strconv"
24+
"strings"
25+
26+
"github.com/intel/goresctrl/pkg/cstates"
27+
"github.com/intel/goresctrl/pkg/utils"
28+
)
29+
30+
var (
31+
cstatesEnvOverridesVar = "OVERRIDE_SYS_CSTATES"
32+
cstatesEnvOverridesJson = os.Getenv(cstatesEnvOverridesVar)
33+
)
34+
35+
type cstatesOverrides []cstatesOverride
36+
type cstatesOverride struct {
37+
Cpus string `json:"cpus"` // CPU ids in list format
38+
Names []string `json:"names"` // list of C-state names, lexical order defines state number
39+
Files map[string]string `json:"files"` // map of attribute name to value for all above CPUs and C-states
40+
}
41+
42+
type overrideFs struct {
43+
overrides cstatesOverrides
44+
stateName map[int]string
45+
nameState map[string]int
46+
cpuStateFile map[utils.ID]map[int]map[string]string // cpu -> state -> attr -> value
47+
}
48+
49+
func NewCstatesFromOverride(filters ...cstates.CstatesFilter) (*cstates.Cstates, error) {
50+
cs := cstates.NewCstates()
51+
ofs, err := NewOverrideFs()
52+
if err != nil {
53+
return nil, fmt.Errorf("failed to create override fs from %s: %v", cstatesEnvOverridesVar, err)
54+
}
55+
cs.SetFs(ofs)
56+
err = cs.Read(filters...)
57+
if err != nil {
58+
return nil, fmt.Errorf("failed to refresh cstates from %s overrides: %v", cstatesEnvOverridesVar, err)
59+
}
60+
return cs, nil
61+
}
62+
63+
func NewOverrideFs() (*overrideFs, error) {
64+
ofs := &overrideFs{
65+
stateName: make(map[int]string),
66+
nameState: make(map[string]int),
67+
cpuStateFile: make(map[utils.ID]map[int]map[string]string),
68+
}
69+
if err := json.Unmarshal([]byte(cstatesEnvOverridesJson), &ofs.overrides); err != nil {
70+
return nil, err
71+
}
72+
if len(ofs.overrides) == 0 {
73+
return nil, fmt.Errorf("no overrides found in %s", cstatesEnvOverridesVar)
74+
}
75+
// Collect unique C-state names from all overrides and assign state numbers
76+
names := make(map[string]struct{})
77+
for _, o := range ofs.overrides {
78+
for _, name := range o.Names {
79+
names[name] = struct{}{}
80+
}
81+
}
82+
orderedNames := make([]string, 0, len(names))
83+
for name := range names {
84+
orderedNames = append(orderedNames, name)
85+
}
86+
slices.Sort(orderedNames)
87+
for state, name := range orderedNames {
88+
ofs.stateName[state] = name
89+
ofs.nameState[name] = state
90+
}
91+
92+
// Build cpuStateFile map for reading and writing attribute values by cpu and state
93+
for _, o := range ofs.overrides {
94+
cpus, err := utils.NewIDSetFromString(o.Cpus)
95+
if err != nil {
96+
return nil, fmt.Errorf("invalid CPU list %q in %s: %v", o.Cpus, cstatesEnvOverridesVar, err)
97+
}
98+
for cpu := range cpus {
99+
cpuid := utils.ID(cpu)
100+
if _, ok := ofs.cpuStateFile[cpuid]; !ok {
101+
ofs.cpuStateFile[cpuid] = make(map[int]map[string]string)
102+
}
103+
for _, name := range o.Names {
104+
state := ofs.nameState[name]
105+
if _, ok := ofs.cpuStateFile[cpuid][state]; !ok {
106+
ofs.cpuStateFile[cpuid][state] = make(map[string]string)
107+
}
108+
maps.Copy(ofs.cpuStateFile[cpuid][state], o.Files)
109+
ofs.cpuStateFile[cpuid][state]["name"] = name // always have name attribute
110+
}
111+
}
112+
}
113+
log.Debugf("cstates override fs: loaded overrides for %d CPUs C-states: %s", len(ofs.cpuStateFile), strings.Join(orderedNames, ", "))
114+
return ofs, nil
115+
}
116+
117+
func (fs *overrideFs) PossibleCpus() (string, error) {
118+
maxCpu := utils.ID(-1)
119+
for cpu := range fs.cpuStateFile {
120+
if cpu > maxCpu {
121+
maxCpu = cpu
122+
}
123+
}
124+
if maxCpu < 0 {
125+
return "", nil
126+
}
127+
return "0-" + strconv.Itoa(maxCpu), nil
128+
}
129+
130+
func (fs *overrideFs) CpuidleStates(cpuID utils.ID) ([]int, error) {
131+
states := []int{}
132+
for state := range fs.stateName {
133+
states = append(states, state)
134+
}
135+
slices.Sort(states)
136+
return states, nil
137+
}
138+
139+
func (fs *overrideFs) CpuidleStateAttrRead(cpu utils.ID, state int, attribute string) (string, error) {
140+
if stateFiles, ok := fs.cpuStateFile[cpu]; ok {
141+
if files, ok := stateFiles[state]; ok {
142+
if val, ok := files[attribute]; ok {
143+
log.Debugf("cstates override fs: read cpu%d cstate=%s %s=%q", cpu, fs.stateName[state], attribute, val)
144+
return val, nil
145+
}
146+
}
147+
}
148+
log.Errorf("cstates override fs: cannot read cpu%d cstate=%s attribute %q", cpu, fs.stateName[state], attribute)
149+
return "", os.ErrNotExist
150+
}
151+
152+
func (fs *overrideFs) CpuidleStateAttrWrite(cpu utils.ID, state int, attribute string, value string) error {
153+
if stateFiles, ok := fs.cpuStateFile[cpu]; ok {
154+
if files, ok := stateFiles[state]; ok {
155+
files[attribute] = value
156+
log.Debugf("cstates override fs: wrote cpu%d cstate=%s %s=%q", cpu, fs.stateName[state], attribute, value)
157+
return nil
158+
}
159+
}
160+
log.Errorf("cstates override fs: write to non-existing cpu%d cstate=%d %s=%q ignored", cpu, state, attribute, value)
161+
return nil
162+
}

0 commit comments

Comments
 (0)