fix:cpu affinity

Signed-off-by: ningmingxiao <[email protected]>

Author: ningmingxiao

libcontainer/process_linux.go

@@ -254,6 +254,9 @@ func (p *setnsProcess) start() (retErr error) {
 			}
 		}
 	}
+	if err := setAffinityAll(p.pid()); err != nil {
+		return err
+	}
 	// Set final CPU affinity right after the process is moved into container's cgroup.
 	if err := p.setFinalCPUAffinity(); err != nil {
 		return err
@@ -615,6 +618,11 @@ func (p *initProcess) start() (retErr error) {
 			return fmt.Errorf("unable to apply cgroup configuration: %w", err)
 		}
 	}
+
+	if err := setAffinityAll(p.pid()); err != nil {
+		return err
+	}
+
 	if p.intelRdtManager != nil {
 		if err := p.intelRdtManager.Apply(p.pid()); err != nil {
 			return fmt.Errorf("unable to apply Intel RDT configuration: %w", err)
@@ -981,3 +989,24 @@ func (p *Process) InitializeIO(rootuid, rootgid int) (i *IO, err error) {
 	}
 	return i, nil
 }
+
+// Set all inherited cpu affinity. Old kernels do that automatically, but
+// new kernels remember the affinity that was set before the cgroup move.
+// This is undesirable, because it inherits the systemd affinity when the container
+// should really move to the container space cpus.
+// here we can't use runtime.NumCPU() to get cpu counts because it call sched_getaffinity to get cpu counts.
+// If systemd set CPUAffinity then use runtime.NumCPU() can't get real cpu counts.
+func setAffinityAll(pid int) error {
+	cpus, err := utils.SystemCPUCores()
+	if err != nil {
+		return err
+	}
+	cpuset := unix.CPUSet{}
+	for i := 0; i < int(cpus); i++ {
+		cpuset.Set(i)
+	}
+	if err := unix.SchedSetaffinity(pid, &cpuset); err != nil {
+		return fmt.Errorf("error resetting pid %d affinity: %w", pid, err)
+	}
+	return nil
+}

libcontainer/utils/utils.go

@@ -1,7 +1,9 @@
 package utils
 
 import (
+	"bufio"
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -113,3 +115,37 @@ func Annotations(labels []string) (bundle string, userAnnotations map[string]str
 	}
 	return
 }
+
+// SystemCPUCores parses CPU usage information from a reader providing
+// /proc/stat format data. It returns the number of CPUs.
+func SystemCPUCores() (cpuNum uint32, _ error) {
+	f, err := os.Open("/proc/stat")
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	return readSystemCPU(f)
+}
+
+func readSystemCPU(r io.Reader) (cpuNum uint32, _ error) {
+	rdr := bufio.NewReaderSize(r, 1024)
+	for {
+		data, isPartial, err := rdr.ReadLine()
+		if err != nil {
+			return 0, fmt.Errorf("error scanning /proc/stat file: %w", err)
+		}
+		// Assume all cpu* records are at the start of the file, like glibc:
+		// https://github.com/bminor/glibc/blob/5d00c201b9a2da768a79ea8d5311f257871c0b43/sysdeps/unix/sysv/linux/getsysstats.c#L108-L135
+		if isPartial || len(data) < 4 {
+			break
+		}
+		line := string(data)
+		if line[:3] != "cpu" {
+			break
+		}
+		if '0' <= line[3] && line[3] <= '9' {
+			cpuNum++
+		}
+	}
+	return cpuNum, nil
+}

libcontainer/utils/utils_test.go

@@ -137,3 +137,13 @@ func TestStripRoot(t *testing.T) {
 		}
 	}
 }
+
+func TestSystemCPUCores(t *testing.T) {
+	numCpusNew, err := SystemCPUCores()
+	if err != nil {
+		t.Errorf("failed to run SystemCPUCores err:%v", err)
+	}
+	if numCpusNew == 0 {
+		t.Errorf("numCpusNew can't equal 0")
+	}
+}