diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 0028d5031047..7b30adc16e18 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -201,7 +201,7 @@ func initMinikubeFlags() { startCmd.Flags().String(network, "", "network to run minikube with. Used by docker/podman, qemu, kvm, and vfkit drivers. If left empty, minikube will create a new network.") startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]") startCmd.Flags().String(trace, "", "Send trace events. Options include: [gcp]") - startCmd.Flags().Int(extraDisks, 0, "Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit, kvm2, qemu2, and vfkit drivers)") + startCmd.Flags().Int(extraDisks, 0, "Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit, kvm2, qemu2, vfkit, and krunkit drivers)") startCmd.Flags().Duration(certExpiration, constants.DefaultCertExpiration, "Duration until minikube certificate expiration, defaults to three years (26280h).") startCmd.Flags().String(binaryMirror, "", "Location to fetch kubectl, kubelet, & kubeadm binaries from.") startCmd.Flags().Bool(disableOptimizations, false, "If set, disables optimizations that are set for local Kubernetes. Including decreasing CoreDNS replicas from 2 to 1. Defaults to false.") @@ -1011,7 +1011,7 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool { } func checkExtraDiskOptions(cmd *cobra.Command, driverName string) { - supportedDrivers := []string{driver.HyperKit, driver.KVM2, driver.QEMU2, driver.VFKit} + supportedDrivers := []string{driver.HyperKit, driver.KVM2, driver.QEMU2, driver.VFKit, driver.Krunkit} if cmd.Flags().Changed(extraDisks) { supported := false diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 45ff1a90cccc..c9349bf11049 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -145,6 +145,9 @@ case "${DRIVER}" in vfkit) echo "vfkit: $(vfkit --version)" ;; + krunkit) + echo "krunkit: $(krunkit --version)" + ;; esac echo "" diff --git a/hack/jenkins/osx_integration_tests_krunkit.sh b/hack/jenkins/osx_integration_tests_krunkit.sh new file mode 100755 index 000000000000..8e34568a9a17 --- /dev/null +++ b/hack/jenkins/osx_integration_tests_krunkit.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script runs the integration tests on an OSX machine for the Hyperkit Driver + +# The script expects the following env variables: +# MINIKUBE_LOCATION: GIT_COMMIT from upstream build. +# COMMIT: Actual commit ID from upstream build +# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests +# access_token: The GitHub API access token. Injected by the Jenkins credential provider. + + +set -ex + +ARCH="arm64" +OS="darwin" +DRIVER="krunkit" +JOB_NAME="Krunkit_macOS" +EXTRA_TEST_ARGS="" +EXTERNAL="yes" + +brew tap slp/krunkit +brew install krunkit + +source common.sh diff --git a/pkg/drivers/krunkit/krunkit.go b/pkg/drivers/krunkit/krunkit.go new file mode 100644 index 000000000000..a318c904b2a6 --- /dev/null +++ b/pkg/drivers/krunkit/krunkit.go @@ -0,0 +1,568 @@ +//go:build darwin + +/* +Copyright 2025 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package krunkit + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "strings" + "syscall" + "time" + + "github.com/docker/machine/libmachine/drivers" + "github.com/docker/machine/libmachine/log" + "github.com/docker/machine/libmachine/mcnutils" + "github.com/docker/machine/libmachine/ssh" + "github.com/docker/machine/libmachine/state" + "github.com/pkg/errors" + + "k8s.io/klog/v2" + pkgdrivers "k8s.io/minikube/pkg/drivers" + "k8s.io/minikube/pkg/drivers/vmnet" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/firewall" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/process" + "k8s.io/minikube/pkg/minikube/reason" + "k8s.io/minikube/pkg/minikube/style" +) + +const ( + driverName = "krunkit" + isoFileName = "boot2docker.iso" + pidFileName = "krunkit.pid" + sockFileName = "krunkit.sock" + logFileName = "krunkit.log" + serialFileName = "serial.log" + logLevelInfo = "3" + defaultSSHUser = "docker" +) + +// Driver is the machine driver for krunkit. +type Driver struct { + *drivers.BaseDriver + *pkgdrivers.CommonDriver + Boot2DockerURL string + DiskSize int + CPU int + Memory int + ExtraDisks int + MACAddress string + VmnetHelper vmnet.Helper +} + +// Ensure that Driver implements drivers.Driver interface +var _ drivers.Driver = &Driver{} + +// NewDriver returns a new krunkit.Driver. +func NewDriver(hostName, storePath string) drivers.Driver { + return &Driver{ + BaseDriver: &drivers.BaseDriver{ + SSHUser: defaultSSHUser, + MachineName: hostName, + StorePath: storePath, + }, + CommonDriver: &pkgdrivers.CommonDriver{}, + } +} + +func (d *Driver) PreCreateCheck() error { + return nil +} + +func (d *Driver) GetMachineName() string { + return d.MachineName +} + +func (d *Driver) DriverName() string { + return driverName +} + +func (d *Driver) GetSSHHostname() (string, error) { + return d.IPAddress, nil +} + +func (d *Driver) GetSSHKeyPath() string { + return d.ResolveStorePath("id_rsa") +} + +func (d *Driver) GetSSHPort() (int, error) { + if d.SSHPort == 0 { + d.SSHPort = 22 + } + return d.SSHPort, nil +} + +func (d *Driver) GetSSHUsername() string { + if d.SSHUser == "" { + d.SSHUser = defaultSSHUser + } + + return d.SSHUser +} + +func (d *Driver) GetURL() (string, error) { + if _, err := os.Stat(d.pidfilePath()); err != nil { + return "", nil + } + ip, err := d.GetIP() + if err != nil { + log.Warnf("Failed to get IP: %v", err) + return "", err + } + if ip == "" { + return "", nil + } + return fmt.Sprintf("tcp://%s:2376", ip), nil +} + +func (d *Driver) GetIP() (string, error) { + return d.IPAddress, nil +} + +// GetState returns driver state. Since krunkit driver uses 2 processes +// (vmnet-helper, krunkit), this returns combined state of both processes. +func (d *Driver) GetState() (state.State, error) { + if krunkitState, err := d.getKrunkitState(); err != nil { + return state.Error, err + } else if krunkitState == state.Running { + return state.Running, nil + } + return d.VmnetHelper.GetState() +} + +func (d *Driver) Create() error { + var err error + if d.SSHPort, err = d.GetSSHPort(); err != nil { + return err + } + + b2dutils := mcnutils.NewB2dUtils(d.StorePath) + if err := b2dutils.CopyIsoToMachineDir(d.Boot2DockerURL, d.MachineName); err != nil { + return err + } + + log.Info("Creating SSH key...") + if err := ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil { + return err + } + + log.Info("Creating disk image...") + if err := d.generateDiskImage(d.DiskSize); err != nil { + return err + } + + if d.ExtraDisks > 0 { + log.Info("Creating extra disk images...") + for i := 0; i < d.ExtraDisks; i++ { + path := pkgdrivers.ExtraDiskPath(d.BaseDriver, i) + if err := pkgdrivers.CreateRawDisk(path, d.DiskSize); err != nil { + return err + } + } + } + + log.Info("Starting krunkit VM...") + return d.Start() +} + +func (d *Driver) Start() error { + socketPath := d.VmnetHelper.SocketPath() + if err := d.VmnetHelper.Start(socketPath); err != nil { + return err + } + + d.MACAddress = d.VmnetHelper.GetMACAddress() + + if err := d.startKrunkit(socketPath); err != nil { + return err + } + + if err := d.setupIP(d.MACAddress); err != nil { + return err + } + + log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress) + + return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second) +} + +// startKrunkit starts the krunkit child process. +func (d *Driver) startKrunkit(socketPath string) error { + var args = []string{ + "--memory", fmt.Sprintf("%d", d.Memory), + "--cpus", fmt.Sprintf("%d", d.CPU), + "--restful-uri", d.restfulURI(), + "--device", fmt.Sprintf("virtio-net,unixSocketPath=%s,mac=%s", socketPath, d.MACAddress), + "--device", fmt.Sprintf("virtio-serial,logFilePath=%s", d.serialPath()), + "--krun-log-level", logLevelInfo, + + // The first device is the boot disk. + "--device", fmt.Sprintf("virtio-blk,path=%s", d.isoPath()), + "--device", fmt.Sprintf("virtio-blk,path=%s", d.diskPath()), + } + + for i := 0; i < d.ExtraDisks; i++ { + args = append(args, + "--device", fmt.Sprintf("virtio-blk,path=%s", pkgdrivers.ExtraDiskPath(d.BaseDriver, i))) + } + + log.Debugf("executing: krunkit %s", strings.Join(args, " ")) + cmd := exec.Command(driverName, args...) + + // Create krunkit in a new process group, so minikube caller can use killpg + // to terminate the entire process group without harming the krunkit process. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + logfile, err := d.openLogfile() + if err != nil { + return fmt.Errorf("failed to open krunkit logfile: %w", err) + } + defer logfile.Close() + cmd.Stderr = logfile + + if err := cmd.Start(); err != nil { + return err + } + return process.WritePidfile(d.pidfilePath(), cmd.Process.Pid) +} + +func (d *Driver) openLogfile() (*os.File, error) { + logfile := d.ResolveStorePath(logFileName) + return os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) +} + +// TODO: duplicate from vfkit and hyperkit: +// https://github.com/kubernetes/minikube/issues/21093 +func (d *Driver) setupIP(mac string) error { + var err error + getIP := func() error { + d.IPAddress, err = pkgdrivers.GetIPAddressByMACAddress(mac) + if err != nil { + return errors.Wrap(err, "failed to get IP address") + } + return nil + } + // Implement a retry loop because IP address isn't added to dhcp leases file immediately + for i := 0; i < 60; i++ { + log.Debugf("Attempt %d", i) + err = getIP() + if err == nil { + break + } + time.Sleep(2 * time.Second) + } + + if err == nil { + log.Debugf("IP: %s", d.IPAddress) + return nil + } + if !isBootpdError(err) { + return errors.Wrap(err, "IP address never found in dhcp leases file") + } + if unblockErr := firewall.UnblockBootpd(); unblockErr != nil { + klog.Errorf("failed unblocking bootpd from firewall: %v", unblockErr) + exit.Error(reason.IfBootpdFirewall, "ip not found", err) + } + out.Styled(style.Restarting, "Successfully unblocked bootpd process from firewall, retrying") + return fmt.Errorf("ip not found: %v", err) +} + +func isBootpdError(err error) bool { + return strings.Contains(err.Error(), "could not find an IP address") +} + +func (d *Driver) Stop() error { + if err := d.stopKrunkit(); err != nil { + return err + } + return d.VmnetHelper.Stop() +} + +func (d *Driver) Kill() error { + if err := d.killKrunkit(); err != nil { + return err + } + return d.VmnetHelper.Kill() +} + +func (d *Driver) Remove() error { + s, err := d.GetState() + if err != nil { + return errors.Wrap(err, "get state") + } + if s == state.Running { + if err := d.Kill(); err != nil { + return errors.Wrap(err, "kill") + } + } + return nil +} + +func (d *Driver) Restart() error { + s, err := d.GetState() + if err != nil { + return err + } + + if s == state.Running { + if err := d.Stop(); err != nil { + return err + } + } + return d.Start() +} + +func (d *Driver) StartDocker() error { + return fmt.Errorf("hosts without a driver cannot start docker") +} + +func (d *Driver) StopDocker() error { + return fmt.Errorf("hosts without a driver cannot stop docker") +} + +func (d *Driver) GetDockerConfigDir() string { + return "" +} + +func (d *Driver) getKrunkitState() (state.State, error) { + pidfile := d.pidfilePath() + pid, err := process.ReadPidfile(pidfile) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return state.Error, err + } + return state.Stopped, nil + } + exists, err := process.Exists(pid, driverName) + if err != nil { + return state.Error, err + } + if !exists { + // No process, stale pidfile. + if err := os.Remove(pidfile); err != nil { + log.Debugf("failed to remove %q: %s", pidfile, err) + } + return state.Stopped, nil + } + return state.Running, nil +} + +func (d *Driver) stopKrunkit() error { + // TODO: this stop request may be ignored by the guest: + // https://github.com/kubernetes/minikube/issues/21092 + if err := d.setKrunkitState("Stop"); err != nil { + // krunkit may be already stopped, shutting down, or not listening. It + // does not support HardStop so the only way to recover is to terminate + // the process. + log.Debugf("Failed to set krunkit state to 'Stop': %s", err) + pidfile := d.pidfilePath() + pid, err := process.ReadPidfile(pidfile) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + // No pidfile. + return nil + } + log.Debugf("Terminte krunkit (pid=%d)", pid) + if err := process.Terminate(pid, driverName); err != nil { + if err != os.ErrProcessDone { + return err + } + // No process, stale pidfile. + log.Debugf("Remove krunkit pidfile %q", pidfile) + if err := os.Remove(pidfile); err != nil { + log.Debugf("failed to remove %q: %s", pidfile, err) + } + } + } + return nil +} + +func (d *Driver) killKrunkit() error { + // krunkit does not support HardStop like vfkit. + pidfile := d.pidfilePath() + pid, err := process.ReadPidfile(pidfile) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + // No pidfile. + return nil + } + log.Debugf("Kill krunkit (pid=%d)", pid) + if err := process.Kill(pid, driverName); err != nil { + if err != os.ErrProcessDone { + return err + } + log.Debugf("Remove krunkit pidfile %q", pidfile) + // No process, stale pidfile. + if err := os.Remove(pidfile); err != nil { + log.Debugf("failed to remove %q: %s", pidfile, err) + } + } + return nil +} + +func (d *Driver) publicSSHKeyPath() string { + return d.GetSSHKeyPath() + ".pub" +} + +func (d *Driver) diskPath() string { + return d.ResolveStorePath("disk.img") +} + +func (d *Driver) pidfilePath() string { + return d.ResolveStorePath(pidFileName) +} + +func (d *Driver) logfilePath() string { + return d.ResolveStorePath(logFileName) +} + +func (d *Driver) serialPath() string { + return d.ResolveStorePath(serialFileName) +} + +func (d *Driver) isoPath() string { + return d.ResolveStorePath(isoFileName) +} + +func (d *Driver) sockfilePath() string { + return d.ResolveStorePath(sockFileName) +} + +func (d *Driver) restfulURI() string { + return fmt.Sprintf("unix://%s", d.sockfilePath()) +} + +func (d *Driver) vmStateURI() string { + return "http://_/vm/state" +} + +// generateDiskImage generates a boot2docker VM disk image. +// TODO: duplicate from vfkit and qemu: https://github.com/kubernetes/minikube/issues/21090 +func (d *Driver) generateDiskImage(size int) error { + log.Debugf("Creating %d MB hard disk image...", size) + + magicString := "boot2docker, please format-me" + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + // magicString first so the automount script knows to format the disk + file := &tar.Header{Name: magicString, Size: int64(len(magicString))} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(magicString)); err != nil { + return err + } + // .ssh/key.pub => authorized_keys + file = &tar.Header{Name: ".ssh", Typeflag: tar.TypeDir, Mode: 0700} + if err := tw.WriteHeader(file); err != nil { + return err + } + pubKey, err := os.ReadFile(d.publicSSHKeyPath()) + if err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write(pubKey); err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys2", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write(pubKey); err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + rawFile := d.diskPath() + if err := os.WriteFile(rawFile, buf.Bytes(), 0644); err != nil { + return nil + } + if err := os.Truncate(rawFile, int64(size)*int64(1024*1024)); err != nil { + return nil + } + log.Debugf("DONE writing to %s and %s", rawFile, d.diskPath()) + return nil +} + +func httpUnixClient(path string) http.Client { + return http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", path) + }, + }, + } +} + +type vmState struct { + State string `json:"state"` +} + +func (d *Driver) setKrunkitState(state string) error { + var vmstate vmState + vmstate.State = state + log.Infof("Set krunkit state: %+v", vmstate) + data, err := json.Marshal(&vmstate) + if err != nil { + return err + } + httpc := httpUnixClient(d.sockfilePath()) + _, err = httpc.Post(d.vmStateURI(), "application/json", bytes.NewReader(data)) + if err != nil { + return err + } + return nil +} + +// TODO: duplicate from vfkit and qemu: https://github.com/kubernetes/minikube/issues/21091 +func WaitForTCPWithDelay(addr string, duration time.Duration) error { + for { + conn, err := net.Dial("tcp", addr) + if err != nil { + continue + } + defer conn.Close() + if _, err := conn.Read(make([]byte, 1)); err != nil && err != io.EOF { + time.Sleep(duration) + continue + } + break + } + return nil +} diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go index 0e64bae99102..ca4d048a986d 100644 --- a/pkg/minikube/cluster/ip.go +++ b/pkg/minikube/cluster/ip.go @@ -139,7 +139,11 @@ func HostIP(hostInfo *host.Host, clusterName string) (net.IP, error) { return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address") } return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil - case driver.VFKit: + case driver.VFKit, driver.Krunkit: + // TODO: check why we need this and test with: + // - vfkkit+nat + // - vfkit+vmnet-shared + // - krunkit+vmnet-shared vmIPString, _ := hostInfo.Driver.GetIP() gatewayIPString := vmIPString[:strings.LastIndex(vmIPString, ".")+1] + "1" return net.ParseIP(gatewayIPString), nil diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index ca8221666116..108cd9509a17 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -63,6 +63,8 @@ const ( Parallels = "parallels" // VFKit driver VFKit = "vfkit" + // Krunkit driver + Krunkit = "krunkit" // AliasKVM is driver name alias for kvm2 AliasKVM = "kvm" @@ -178,6 +180,11 @@ func IsVFKit(name string) bool { return name == VFKit } +// IsKrunkit checks if the driver is krunkit +func IsKrunkit(name string) bool { + return name == Krunkit +} + // IsVM checks if the driver is a VM func IsVM(name string) bool { if IsKIC(name) || BareMetal(name) { diff --git a/pkg/minikube/driver/driver_darwin.go b/pkg/minikube/driver/driver_darwin.go index 6bf5f20d7c5b..a0faf66a4f8d 100644 --- a/pkg/minikube/driver/driver_darwin.go +++ b/pkg/minikube/driver/driver_darwin.go @@ -29,6 +29,7 @@ var supportedDrivers = func() []string { return []string{ QEMU2, VFKit, + Krunkit, Parallels, Docker, Podman, diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index ff5b4519389b..42d7691ed7c0 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -69,6 +69,7 @@ func TestMachineType(t *testing.T) { QEMU2: "VM", QEMU: "VM", VFKit: "VM", + Krunkit: "VM", VirtualBox: "VM", HyperKit: "VM", VMware: "VM", diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index afbf1a4cabe0..b47febb0bbea 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -552,15 +552,11 @@ var ( NotFoundVmnetHelper = Kind{ ID: "NOT_FOUND_VMNET_HELPER", ExitCode: ExProgramNotFound, - Advice: translate.T(`vmnet-helper was not found on the system, resolve by: + Advice: translate.T(`vmnet-helper was not found on the system. - Option 1) Installing vmnet-helper: + Please install vmnet-helper using these instructions: - https://github.com/nirs/vmnet-helper#installation - - Option 2) Using the nat network: - - minikube start{{.profile}} --driver vfkit --network nat`), + https://github.com/nirs/vmnet-helper#installation`), Style: style.SeeNoEvil, } NotConfiguredVmnetHelper = Kind{ diff --git a/pkg/minikube/registry/drvs/init.go b/pkg/minikube/registry/drvs/init.go index d4276d23afc3..c38ed7d1714c 100644 --- a/pkg/minikube/registry/drvs/init.go +++ b/pkg/minikube/registry/drvs/init.go @@ -21,6 +21,7 @@ import ( _ "k8s.io/minikube/pkg/minikube/registry/drvs/docker" _ "k8s.io/minikube/pkg/minikube/registry/drvs/hyperkit" _ "k8s.io/minikube/pkg/minikube/registry/drvs/hyperv" + _ "k8s.io/minikube/pkg/minikube/registry/drvs/krunkit" _ "k8s.io/minikube/pkg/minikube/registry/drvs/kvm2" _ "k8s.io/minikube/pkg/minikube/registry/drvs/none" _ "k8s.io/minikube/pkg/minikube/registry/drvs/parallels" diff --git a/pkg/minikube/registry/drvs/krunkit/doc.go b/pkg/minikube/registry/drvs/krunkit/doc.go new file mode 100644 index 000000000000..5023cbddd084 --- /dev/null +++ b/pkg/minikube/registry/drvs/krunkit/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package krunkit diff --git a/pkg/minikube/registry/drvs/krunkit/krunkit.go b/pkg/minikube/registry/drvs/krunkit/krunkit.go new file mode 100644 index 000000000000..d5388372d334 --- /dev/null +++ b/pkg/minikube/registry/drvs/krunkit/krunkit.go @@ -0,0 +1,101 @@ +//go:build darwin + +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package krunkit + +import ( + "errors" + "fmt" + "os/exec" + "path/filepath" + "runtime" + + "github.com/docker/machine/libmachine/drivers" + "github.com/google/uuid" + + "k8s.io/minikube/pkg/drivers/krunkit" + "k8s.io/minikube/pkg/drivers/vmnet" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/registry" +) + +const ( + docURL = "https://minikube.sigs.k8s.io/docs/reference/drivers/krunkit/" +) + +func init() { + if err := registry.Register(registry.DriverDef{ + Name: driver.Krunkit, + Init: func() drivers.Driver { return krunkit.NewDriver("", "") }, + Config: configure, + Status: status, + Default: true, + Priority: registry.Experimental, + }); err != nil { + panic(fmt.Sprintf("register failed: %v", err)) + } +} + +func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) { + machineName := config.MachineName(cfg, n) + storePath := localpath.MiniPath() + + // We generate a random UUID (or use a user provided one). vment-helper will + // obtain a mac address from the vmnet framework using the UUID. + u := cfg.UUID + if u == "" { + u = uuid.NewString() + } + + return &krunkit.Driver{ + BaseDriver: &drivers.BaseDriver{ + MachineName: machineName, + StorePath: storePath, + SSHUser: "docker", + }, + Boot2DockerURL: download.LocalISOResource(cfg.MinikubeISO), + DiskSize: cfg.DiskSize, + Memory: cfg.Memory, + CPU: cfg.CPUs, + ExtraDisks: cfg.ExtraDisks, + VmnetHelper: vmnet.Helper{ + MachineDir: filepath.Join(storePath, "machines", machineName), + InterfaceID: u, + // Required Until https://github.com/containers/libkrun/issues/264 is fixed. + Offloading: true, + }, + }, nil +} + +func status() registry.State { + if runtime.GOOS != "darwin" && runtime.GOARCH != "arm64" { + err := errors.New("The krunkit driver is only supported on macOS arm64 machines") + return registry.State{Error: err, Fix: "Use another driver", Doc: docURL} + } + if _, err := exec.LookPath("krunkit"); err != nil { + return registry.State{Error: err, Fix: "Run 'brew tap slp/krunkit && brew install krunkit'", Doc: docURL} + } + if err := vmnet.ValidateHelper(); err != nil { + vmnetErr := err.(*vmnet.Error) + return registry.State{Error: vmnetErr.Err, Fix: "Install and configure vment-helper", Doc: docURL} + } + return registry.State{Installed: true, Healthy: true, Running: true} +} diff --git a/site/content/en/docs/drivers/_index.md b/site/content/en/docs/drivers/_index.md index 59893f4f0ef7..65460b5219b3 100644 --- a/site/content/en/docs/drivers/_index.md +++ b/site/content/en/docs/drivers/_index.md @@ -33,6 +33,7 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [QEMU]({{}}) - VM * [Podman]({{}}) - VM + Container (experimental) * [VFkit]({{}}) - VM (preferred) +* [Krunkit]({{}}) - VM (experimental) * [SSH]({{}}) - remote ssh ## Windows diff --git a/site/content/en/docs/drivers/krunkit.md b/site/content/en/docs/drivers/krunkit.md new file mode 100644 index 000000000000..21364905d82c --- /dev/null +++ b/site/content/en/docs/drivers/krunkit.md @@ -0,0 +1,102 @@ +--- +title: "krunkit" +weight: 2 +aliases: + - /docs/reference/drivers/krunkit +--- + +## Overview + +[krunkit](https://github.com/containers/krunkit) is an open-source program for +macOS virtualization, optimized for GPU accelerated virtual machines and AI +workloads. + +## Requirements + +- Available only on Apple silicon. +- Requires macOS 13 or later. +- Requires minikube version 1.37.0 or later. +- Requires krunkit version 0.2.2 or later. +- Requires [vmnet-helper](https://github.com/nirs/vmnet-helper). + +## Installing krunkit + +To install krunkit run: + +```shell +brew tap slp/krunkit +brew install krunkit +``` + +## Networking + +To use the krunkit driver you must install +[vmnet-helper](https://github.com/nirs/vmnet-helper), see installation +instructions bellow. + +### Install vment-helper + +```shell +machine="$(uname -m)" +archive="vmnet-helper-$machine.tar.gz" +curl -LOf "https://github.com/nirs/vmnet-helper/releases/latest/download/$archive" +sudo tar xvf "$archive" -C / opt/vmnet-helper +rm "$archive" +``` + +The command downloads the latest release from github and installs it to +`/opt/vmnet-helper`. + +> [!IMPORTANT] +> The vmnet-helper executable and the directory where it is installed +> must be owned by root and may not be modifiable by unprivileged users. + +### Grant permission to run vmnet-helper + +The vment-helper process must run as root to create a vmnet interface. +To allow users in the `staff` group to run the vmnet helper without a +password, you can install the default sudoers rule: + +```shell +sudo install -m 0640 /opt/vmnet-helper/share/doc/vmnet-helper/sudoers.d/vmnet-helper /etc/sudoers.d/ +``` + +You can change the sudoers configuration to allow access to specific +users or other groups. + +### Usage + +```shell +minikube start --driver krunkit +``` + +## Issues + +### Other + +* [Full list of open 'krunkit' driver issues](https://github.com/kubernetes/minikube/labels/co%2Fkrunkit) + +## Troubleshooting + +### Run with logs + +Run `minikube start --driver krunkit --alsologtostderr -v=7` to debug crashes + +### Troubleshooting vmnet-helper + +Check for errors in vment-helper log: + +```shell +$MINIKUBE_HOME/.minikube/machines/MACHINE-NAME/vmnet-helper.log +``` + +Check that the `vmnet-helper` process is running: + +```shell +ps au | grep vmnet-helper | grep -v grep +``` + +If the helper is not running restart the minikube cluster. + +For help with vment-helper please use the +[discussions](https://github.com/nirs/vmnet-helper/discussions).