Skip to content

Commit 81e87d8

Browse files
ningmingxiaoningmingxiao
committed
fix:cpu affinity
Signed-off-by: ningmingxiao <[email protected]>
1 parent a85b5fc commit 81e87d8

File tree

5 files changed

+131
-8
lines changed

5 files changed

+131
-8
lines changed

‎.github/workflows/test.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ jobs:
168168

169169
- name: integration test (systemd driver)
170170
run: |
171+
sudo taskset -pc 0-1 1
171172
# Delegate all cgroup v2 controllers to rootless user via --systemd-cgroup.
172173
# The default (since systemd v252) is "pids memory cpu".
173174
sudo mkdir -p /etc/systemd/system/[email protected]

‎libcontainer/process_linux.go‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -188,11 +188,7 @@ func (p *setnsProcess) startWithCPUAffinity() error {
188188
}
189189

190190
func (p *setnsProcess) setFinalCPUAffinity() error {
191-
aff := p.config.CPUAffinity
192-
if aff == nil || aff.Final == nil {
193-
return nil
194-
}
195-
if err := unix.SchedSetaffinity(p.pid(), aff.Final); err != nil {
191+
if err := unix.SchedSetaffinity(p.pid(), p.config.CPUAffinity.Final); err != nil {
196192
return fmt.Errorf("error setting final CPU affinity: %w", err)
197193
}
198194
return nil
@@ -254,9 +250,16 @@ func (p *setnsProcess) start() (retErr error) {
254250
}
255251
}
256252
}
257-
// Set final CPU affinity right after the process is moved into container's cgroup.
258-
if err := p.setFinalCPUAffinity(); err != nil {
259-
return err
253+
254+
if aff := p.config.CPUAffinity; aff == nil || aff.Final == nil {
255+
if err := setAffinityAll(p.pid()); err != nil {
256+
return err
257+
}
258+
} else {
259+
// Set final CPU affinity right after the process is moved into container's cgroup.
260+
if err := p.setFinalCPUAffinity(); err != nil {
261+
return err
262+
}
260263
}
261264
if p.intelRdtPath != "" {
262265
// if Intel RDT "resource control" filesystem path exists
@@ -615,6 +618,11 @@ func (p *initProcess) start() (retErr error) {
615618
return fmt.Errorf("unable to apply cgroup configuration: %w", err)
616619
}
617620
}
621+
622+
if err := setAffinityAll(p.pid()); err != nil {
623+
return err
624+
}
625+
618626
if p.intelRdtManager != nil {
619627
if err := p.intelRdtManager.Apply(p.pid()); err != nil {
620628
return fmt.Errorf("unable to apply Intel RDT configuration: %w", err)
@@ -981,3 +989,24 @@ func (p *Process) InitializeIO(rootuid, rootgid int) (i *IO, err error) {
981989
}
982990
return i, nil
983991
}
992+
993+
// Set all inherited cpu affinity. Old kernels do that automatically, but
994+
// new kernels remember the affinity that was set before the cgroup move.
995+
// This is undesirable, because it inherits the systemd affinity when the container
996+
// should really move to the container space cpus.
997+
// here we can't use runtime.NumCPU() to get cpu counts because it call sched_getaffinity to get cpu counts.
998+
// If systemd set CPUAffinity then use runtime.NumCPU() can't get real cpu counts.
999+
func setAffinityAll(pid int) error {
1000+
cpus, err := utils.SystemCPUCores()
1001+
if err != nil {
1002+
return err
1003+
}
1004+
cpuset := unix.CPUSet{}
1005+
for i := 0; i < int(cpus); i++ {
1006+
cpuset.Set(i)
1007+
}
1008+
if err := unix.SchedSetaffinity(pid, &cpuset); err != nil {
1009+
return fmt.Errorf("error resetting pid %d affinity: %w", pid, err)
1010+
}
1011+
return nil
1012+
}

‎libcontainer/utils/utils.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package utils
22

33
import (
4+
"bufio"
45
"encoding/json"
6+
"fmt"
57
"io"
68
"os"
79
"path/filepath"
@@ -113,3 +115,31 @@ func Annotations(labels []string) (bundle string, userAnnotations map[string]str
113115
}
114116
return
115117
}
118+
119+
// SystemCPUCores parses CPU usage information from a reader providing
120+
// /proc/stat format data. It returns the number of CPUs.
121+
func SystemCPUCores() (cpuNum uint32, _ error) {
122+
f, err := os.Open("/proc/stat")
123+
if err != nil {
124+
return 0, err
125+
}
126+
defer f.Close()
127+
return readSystemCPU(f)
128+
}
129+
130+
func readSystemCPU(r io.Reader) (cpuNum uint32, _ error) {
131+
reader := bufio.NewReader(r)
132+
for {
133+
line, err := reader.ReadString('\n')
134+
if err != nil {
135+
return 0, fmt.Errorf("error scanning /proc/stat file: %w", err)
136+
}
137+
if line[:3] != "cpu" {
138+
break
139+
}
140+
if '0' <= line[3] && line[3] <= '9' {
141+
cpuNum++
142+
}
143+
}
144+
return cpuNum, nil
145+
}

‎libcontainer/utils/utils_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package utils
22

33
import (
44
"bytes"
5+
"os"
56
"testing"
67

78
"golang.org/x/sys/unix"
@@ -137,3 +138,48 @@ func TestStripRoot(t *testing.T) {
137138
}
138139
}
139140
}
141+
142+
func TestSystemCPUCores(t *testing.T) {
143+
t.Run("MultiCore", func(t *testing.T) {
144+
content := `cpu 5263854 3354 5436110 61362568 22532 728994 208644 796742 0 0
145+
cpu0 720149 490 674391 7571042 4601 103938 42990 109735 0 0
146+
cpu1 595284 389 676327 7761080 2405 77856 25882 95566 0 0
147+
cpu2 727310 508 693322 7562543 3426 102842 28396 105651 0 0
148+
cpu3 601561 304 685817 7751082 2064 80219 17547 92322 0 0
149+
cpu4 713033 504 669261 7586506 2850 105624 39150 106688 0 0
150+
cpu5 595065 328 683341 7761812 2065 77750 17827 91675 0 0
151+
cpu6 720528 458 676161 7595093 3007 101744 21132 103530 0 0
152+
cpu7 590922 371 677486 7773406 2111 79018 15716 91570 0 0
153+
intr 1997458243 37 333 0 0 0 0 3 0 1 0 0 0 183 0 0 90125 0 0 0 0 0 0 0 0 0 458484 0 361539 0 0 0 256 0 1956792 15 0 918260 6 1450411 256422 0 49025 195 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
154+
ctxt 2640704037
155+
btime 1752714561
156+
processes 5253419
157+
procs_running 2
158+
procs_blocked 0
159+
softirq 580996229 23 230614056 282 2160733 45109 0 40037 116656548 0 231479441
160+
`
161+
tmpfile, err := os.CreateTemp("", "stat")
162+
if err != nil {
163+
t.Fatal(err)
164+
}
165+
defer os.Remove(tmpfile.Name())
166+
167+
if _, err := tmpfile.WriteString(content); err != nil {
168+
t.Fatal(err)
169+
}
170+
if err := tmpfile.Close(); err != nil {
171+
t.Fatal(err)
172+
}
173+
f, err := os.Open(tmpfile.Name())
174+
if err != nil {
175+
t.Fatal(err)
176+
}
177+
count, err := readSystemCPU(f)
178+
if err != nil {
179+
t.Errorf("unexpected error: %v", err)
180+
}
181+
if count != 8 {
182+
t.Errorf("expected 8 cores, got %d", count)
183+
}
184+
})
185+
}

‎tests/integration/cpu_affinity.bats‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,20 @@ function cpus_to_mask() {
9999
[[ "$output" == *"nsexec"*": affinity: $mask"* ]]
100100
[[ "$output" == *"Cpus_allowed_list: $final"* ]] # Mind the literal tab.
101101
}
102+
103+
@test "runc exec [CPU affinity set from config.json]" {
104+
update_config '.process.args = [ "/bin/grep", "-F", "Cpus_allowed_list", "/proc/self/status"]'
105+
cpus=$(grep -c "^processor" /proc/cpuinfo)
106+
cpus_minus_one=$((cpus - 1))
107+
runc run ct1
108+
[ "$status" -eq 0 ]
109+
last_col=$(echo "$output" | awk '{print $NF}')
110+
[[ "$last_col" == *"0-$cpus_minus_one"* ]] # Mind the literal tab.
111+
update_config '.process.args = ["/bin/sleep", "100"]'
112+
runc run -d --console-socket "$CONSOLE_SOCKET" ct2
113+
[ "$status" -eq 0 ]
114+
runc exec ct2 grep -F "Cpus_allowed_list:" /proc/self/status
115+
[ "$status" -eq 0 ]
116+
last_col=$(echo "$output" | awk '{print $NF}')
117+
[[ "$last_col" == *"0-$cpus_minus_one"* ]] # Mind the literal tab.
118+
}

0 commit comments

Comments
 (0)