Merge pull request #3979 from norio-nomura/ssh-over-vsock-on-vz

vz: add SSH over AF_VSOCK

Author: AkihiroSuda

hack/test-templates.sh

@@ -61,13 +61,15 @@ declare -A CHECKS=(
 	["set-user"]=""
 	["preserve-env"]="1"
 	["static-port-forwards"]=""
+	["ssh-over-vsock"]=""
 )
 
 case "$NAME" in
 "default")
 	# CI failure:
 	# "[hostagent] failed to confirm whether /c/Users/runneradmin [remote] is successfully mounted"
 	[ "${OS_HOST}" = "Msys" ] && CHECKS["mount-home"]=
+	[ "${OS_HOST}" = "Darwin" ] && CHECKS["ssh-over-vsock"]="1"
 	;;
 "alpine"*)
 	WARNING "Alpine does not support systemd"
@@ -341,14 +343,46 @@ if [[ -n ${CHECKS["preserve-env"]} ]]; then
 	"${scriptdir}"/test-preserve-env.sh "$NAME"
 fi
 
+if [[ -n ${CHECKS["ssh-over-vsock"]} ]]; then
+	if [[ "$(limactl ls "${NAME}" --yq .vmType)" == "vz" ]]; then
+		INFO "Testing SSH over vsock"
+		set -x
+		INFO "Testing LIMA_SSH_OVER_VSOCK=true environment"
+		limactl stop "${NAME}"
+		if ! LIMA_SSH_OVER_VSOCK=true limactl start "${NAME}" 2>&1 | grep -i "started vsock forwarder"; then
+			set +x
+			diagnose "${NAME}"
+			ERROR "LIMA_SSH_OVER_VSOCK=true did not enable vsock forwarder"
+			exit 1
+		fi
+		INFO 'Testing LIMA_SSH_OVER_VSOCK="" environment'
+		limactl stop "${NAME}"
+		if ! LIMA_SSH_OVER_VSOCK="" limactl start "${NAME}" 2>&1 | grep -i "started vsock forwarder"; then
+			set +x
+			diagnose "${NAME}"
+			ERROR "LIMA_SSH_OVER_VSOCK= did not enable vsock forwarder"
+			exit 1
+		fi
+		INFO "Testing LIMA_SSH_OVER_VSOCK=false environment"
+		limactl stop "${NAME}"
+		if ! LIMA_SSH_OVER_VSOCK=false limactl start "${NAME}" 2>&1 | grep -i "skipping detection of SSH server on vsock port"; then
+			set +x
+			diagnose "${NAME}"
+			ERROR "LIMA_SSH_OVER_VSOCK=false did not disable vsock forwarder"
+			exit 1
+		fi
+		set +x
+	fi
+fi
+
 # Use GHCR to avoid hitting Docker Hub rate limit
 nginx_image="ghcr.io/stargz-containers/nginx:1.19-alpine-org"
 alpine_image="ghcr.io/containerd/alpine:3.14.0"
 
 if [[ -n ${CHECKS["container-engine"]} ]]; then
 	sudo=""
 	# Currently WSL2 machines only support privileged engine. This requirement might be lifted in the future.
-	if [[ "$(limactl ls --json "${NAME}" | jq -r .vmType)" == "wsl2" ]]; then
+	if [[ "$(limactl ls "${NAME}" --yq .vmType)" == "wsl2" ]]; then
 		sudo="sudo"
 	fi
 	INFO "Run a nginx container with port forwarding 127.0.0.1:8080"
@@ -428,7 +462,7 @@ if [[ -n ${CHECKS["port-forwards"]} ]]; then
 				sudo="sudo"
 			fi
 			# Currently WSL2 machines only support privileged engine. This requirement might be lifted in the future.
-			if [[ "$(limactl ls --json "${NAME}" | jq -r .vmType)" == "wsl2" ]]; then
+			if [[ "$(limactl ls "${NAME}" --yq .vmType)" == "wsl2" ]]; then
 				sudo="sudo"
 			fi
 			limactl shell "$NAME" $sudo $CONTAINER_ENGINE info
@@ -613,7 +647,7 @@ if [[ -n ${CHECKS["clone"]} ]]; then
 	limactl start "$NAME"
 fi
 
-if [[ $NAME == "fedora" && "$(limactl ls --json "$NAME" | jq -r .vmType)" == "vz" ]]; then
+if [[ $NAME == "fedora" && "$(limactl ls "${NAME}" --yq .vmType)" == "vz" ]]; then
 	"${scriptdir}"/test-selinux.sh "$NAME"
 fi
 

pkg/driver/vz/vm_darwin.go

@@ -72,12 +72,8 @@ func startVM(ctx context.Context, inst *limatype.Instance, sshLocalPort int) (*v
 
 	errCh := make(chan error)
 
-	filesToRemove := make(map[string]struct{})
-	defer func() {
-		for f := range filesToRemove {
-			_ = os.RemoveAll(f)
-		}
-	}()
+	waitSSHLocalPortAccessible := make(chan struct{})
+	defer close(waitSSHLocalPortAccessible)
 	go func() {
 		// Handle errors via errCh and handle stop vm during context close
 		defer func() {
@@ -105,13 +101,36 @@ func startVM(ctx context.Context, inst *limatype.Instance, sshLocalPort int) (*v
 						logrus.Errorf("error writing to pid fil %q", pidFile)
 						errCh <- err
 					}
-					filesToRemove[pidFile] = struct{}{}
 					logrus.Info("[VZ] - vm state change: running")
 
-					err := usernetClient.ConfigureDriver(ctx, inst, sshLocalPort)
+					usernetSSHLocalPort := sshLocalPort
+					useSSHOverVsock := true
+					if envVar := os.Getenv("LIMA_SSH_OVER_VSOCK"); envVar != "" {
+						b, err := strconv.ParseBool(envVar)
+						if err != nil {
+							logrus.WithError(err).Warnf("invalid LIMA_SSH_OVER_VSOCK value %q", envVar)
+						} else {
+							useSSHOverVsock = b
+						}
+					}
+					if !useSSHOverVsock {
+						logrus.Info("LIMA_SSH_OVER_VSOCK is false, skipping detection of SSH server on vsock port")
+					} else if err := usernetClient.WaitOpeningSSHPort(ctx, inst); err == nil {
+						hostAddress := net.JoinHostPort(inst.SSHAddress, strconv.Itoa(usernetSSHLocalPort))
+						if err := wrapper.startVsockForwarder(ctx, 22, hostAddress); err == nil {
+							logrus.Infof("Detected SSH server is listening on the vsock port; changed %s to proxy for the vsock port", hostAddress)
+							usernetSSHLocalPort = 0 // disable gvisor ssh port forwarding
+						} else {
+							logrus.WithError(err).Warn("Failed to detect SSH server on vsock port, falling back to usernet forwarder")
+						}
+					} else {
+						logrus.WithError(err).Warn("Failed to wait for the guest SSH server to become available, falling back to usernet forwarder")
+					}
+					err := usernetClient.ConfigureDriver(ctx, inst, usernetSSHLocalPort)
 					if err != nil {
 						errCh <- err
 					}
+					waitSSHLocalPortAccessible <- struct{}{}
 				case vz.VirtualMachineStateStopped:
 					logrus.Info("[VZ] - vm state change: stopped")
 					wrapper.mu.Lock()
@@ -128,7 +147,7 @@ func startVM(ctx context.Context, inst *limatype.Instance, sshLocalPort int) (*v
 			}
 		}
 	}()
-
+	<-waitSSHLocalPortAccessible
 	return wrapper, errCh, err
 }
 

pkg/driver/vz/vsock_forwarder.go

@@ -0,0 +1,75 @@
+//go:build darwin && !no_vz
+
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package vz
+
+import (
+	"context"
+	"errors"
+	"net"
+
+	"github.com/containers/gvisor-tap-vsock/pkg/tcpproxy"
+	"github.com/sirupsen/logrus"
+)
+
+func (m *virtualMachineWrapper) startVsockForwarder(ctx context.Context, vsockPort uint32, hostAddress string) error {
+	// Test if the vsock port is open
+	conn, err := m.dialVsock(ctx, vsockPort)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	// Start listening on localhost:hostPort and forward to vsock:vsockPort
+	_, _, err = net.SplitHostPort(hostAddress)
+	if err != nil {
+		return err
+	}
+	var lc net.ListenConfig
+	l, err := lc.Listen(ctx, "tcp", hostAddress)
+	if err != nil {
+		return err
+	}
+	go func() {
+		<-ctx.Done()
+		l.Close()
+	}()
+	logrus.Infof("Started vsock forwarder: %s -> vsock:%d on VM", hostAddress, vsockPort)
+	go func() {
+		defer l.Close()
+		for {
+			conn, err := l.Accept()
+			if err != nil {
+				if errors.Is(err, net.ErrClosed) {
+					return
+				}
+				logrus.WithError(err).Errorf("vsock forwarder accept error: %v", err)
+			} else {
+				p := tcpproxy.DialProxy{
+					DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+						return m.dialVsock(ctx, vsockPort)
+					},
+				}
+				go p.HandleConn(conn)
+			}
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				continue
+			}
+		}
+	}()
+	return nil
+}
+
+func (m *virtualMachineWrapper) dialVsock(_ context.Context, port uint32) (conn net.Conn, err error) {
+	for _, socket := range m.SocketDevices() {
+		conn, err = socket.Connect(port)
+		if err == nil {
+			return conn, nil
+		}
+	}
+	return nil, err
+}

pkg/networks/usernet/client.go

@@ -38,9 +38,11 @@ func (c *Client) ConfigureDriver(ctx context.Context, inst *limatype.Instance, s
 	if err != nil {
 		return err
 	}
-	err = c.ResolveAndForwardSSH(ipAddress, sshLocalPort)
-	if err != nil {
-		return err
+	if sshLocalPort != 0 {
+		err = c.ResolveAndForwardSSH(ipAddress, sshLocalPort)
+		if err != nil {
+			return err
+		}
 	}
 	hosts := inst.Config.HostResolver.Hosts
 	if hosts == nil {
@@ -127,6 +129,30 @@ func (c *Client) Leases(ctx context.Context) (map[string]string, error) {
 	return leases, nil
 }
 
+// WaitOpeningSSHPort Wait until the guest ssh server is available.
+func (c *Client) WaitOpeningSSHPort(ctx context.Context, inst *limatype.Instance) error {
+	// This timeout is based on the maximum wait time for the first essential requirement.
+	timeoutSeconds := 600
+	ctx, cancel := context.WithTimeout(ctx, time.Duration(timeoutSeconds)*time.Second)
+	defer cancel()
+	macAddress := limayaml.MACAddress(inst.Dir)
+	ipAddr, err := c.ResolveIPAddress(ctx, macAddress)
+	if err != nil {
+		return err
+	}
+	// -1 avoids both sides timing out simultaneously.
+	u := fmt.Sprintf("%s/extension/wait_port?ip=%s&port=22&timeout=%d", c.base, ipAddr, timeoutSeconds-1)
+	res, err := httpclientutil.Get(ctx, c.client, u)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return errors.New("failed to wait for SSH port")
+	}
+	return nil
+}
+
 func NewClientByName(nwName string) *Client {
 	endpointSock, err := Sock(nwName, EndpointSock)
 	if err != nil {

pkg/networks/usernet/gvproxy.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -103,7 +104,8 @@ func run(ctx context.Context, g *errgroup.Group, configuration *types.Configurat
 	if err != nil {
 		return err
 	}
-	httpServe(ctx, g, ln, vn.Mux())
+
+	httpServe(ctx, g, ln, muxWithExtension(vn))
 
 	if opts.QemuSocket != "" {
 		err = listenQEMU(ctx, vn)
@@ -239,6 +241,58 @@ func httpServe(ctx context.Context, g *errgroup.Group, ln net.Listener, mux http
 	})
 }
 
+func muxWithExtension(n *virtualnetwork.VirtualNetwork) *http.ServeMux {
+	m := n.Mux()
+	m.HandleFunc("/extension/wait_port", func(w http.ResponseWriter, r *http.Request) {
+		ip := r.URL.Query().Get("ip")
+		if net.ParseIP(ip) == nil {
+			msg := fmt.Sprintf("invalid ip address: %s", ip)
+			http.Error(w, msg, http.StatusBadRequest)
+			return
+		}
+		port16, err := strconv.ParseUint(r.URL.Query().Get("port"), 10, 16)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		port := uint16(port16)
+		addr := fmt.Sprintf("%s:%d", ip, port)
+
+		timeoutSeconds := 10
+		if timeoutString := r.URL.Query().Get("timeout"); timeoutString != "" {
+			timeout16, err := strconv.ParseUint(timeoutString, 10, 16)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+			timeoutSeconds = int(timeout16)
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds)*time.Second)
+		defer cancel()
+		// Wait until the port is available.
+		for {
+			conn, err := n.DialContextTCP(ctx, addr)
+			if err == nil {
+				conn.Close()
+				logrus.Debugf("Port is available on %s", addr)
+				w.WriteHeader(http.StatusOK)
+				break
+			}
+			select {
+			case <-ctx.Done():
+				msg := fmt.Sprintf("timed out waiting for port to become available on %s", addr)
+				logrus.Warn(msg)
+				http.Error(w, msg, http.StatusRequestTimeout)
+				return
+			default:
+			}
+			logrus.Debugf("Waiting for port to become available on %s", addr)
+			time.Sleep(1 * time.Second)
+		}
+	})
+	return m
+}
+
 func searchDomains() []string {
 	if runtime.GOOS != "windows" {
 		return resolveSearchDomain("/etc/resolv.conf")

website/content/en/docs/config/environment-variables.md

@@ -106,6 +106,15 @@ This page documents the environment variables used in Lima.
   lima
   ```
 
+### `LIMA_SSH_OVER_VSOCK`
+- **Description**: Specifies to use vsock for SSH connection instead of port forwarding.
+- **Default**: `true` (since v2.0.0)
+- **Usage**: 
+  ```sh
+  export LIMA_SSH_OVER_VSOCK=true
+  ```
+- **Note**: This variable is effective only if the VM is VZ based and systemd is v256 or later (e.g. Ubuntu 24.10+).
+
 ### `LIMA_SSH_PORT_FORWARDER`
 
 - **Description**: Specifies to use the SSH port forwarder (slow) instead of gRPC (fast, previously unstable)

website/content/en/docs/config/port.md

@@ -36,6 +36,20 @@ LIMA_SSH_PORT_FORWARDER=true limactl start
 - Doesn't support UDP based port forwarding
 - Spawns child process on host for running SSH master.
 
+#### SSH over AF_VSOCK
+
+| ⚡ Requirement | Lima >= 2.0 |
+|---------------|-------------|
+
+If VM is VZ based and systemd is v256 or later (e.g. Ubuntu 24.10+), Lima uses AF_VSOCK for communication between host and guest.
+SSH based port forwarding is much faster when using AF_VSOCK compared to traditional virtual network based port forwarding.
+
+To disable this feature, set `LIMA_SSH_OVER_VSOCK` to `false`:
+
+```bash
+export LIMA_SSH_OVER_VSOCK=false
+```
+
 ### Using GRPC
 
 | ⚡ Requirement | Lima >= 1.0 |