Skip to content

Commit 7f39af3

Browse files
committed
feat: add flag to configure platform while deploying
WIP: WIP: WIP: docs: update docu fix: udpate script with tee
1 parent 115c8ac commit 7f39af3

34 files changed

+1707
-23
lines changed

.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,3 @@ qodana.yaml
2727

2828
# Pipenv
2929
Pipfile*
30-

cmd/cluster/clusterCreate.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,9 @@ func NewCmdClusterCreate() *cobra.Command {
251251
cmd.Flags().StringArrayP("runtime-ulimit", "", nil, "Add ulimit to container runtime (Format: `NAME[=SOFT]:[HARD]`\n - Example: `k3d cluster create --agents 2 --runtime-ulimit \"nofile=1024:1024\" --runtime-ulimit \"noproc=1024:1024\"`")
252252
_ = ppViper.BindPFlag("cli.runtime-ulimits", cmd.Flags().Lookup("runtime-ulimit"))
253253

254+
cmd.Flags().StringArrayP("runtime-platform", "", nil, "Add platform to container runtime (Format: `<os>[(<OSVersion>)]|<arch>|<os>[(<OSVersion>)]/<arch>[/<variant>][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-platform \"linux/amd64@agent:0,1\" --runtime-platform \"linux/arm64/v8@server:0\"`")
255+
_ = ppViper.BindPFlag("cli.runtime-platform", cmd.Flags().Lookup("runtime-platform"))
256+
254257
cmd.Flags().String("registry-create", "", "Create a k3d-managed registry and connect it to the cluster (Format: `NAME[:HOST][:HOSTPORT]`\n - Example: `k3d cluster create --registry-create mycluster-registry:0.0.0.0:5432`")
255258
_ = ppViper.BindPFlag("cli.registries.create", cmd.Flags().Lookup("registry-create"))
256259

@@ -515,6 +518,33 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
515518
cfg.Options.Runtime.Ulimits = append(cfg.Options.Runtime.Ulimits, *cliutil.ParseRuntimeUlimit[conf.Ulimit](ulimit))
516519
}
517520

521+
// --runtime-platform
522+
// runtimePlatform will add container platform configuration to applied node filters
523+
runtimePlatformFilterMap := make(map[string][]string, 1)
524+
for _, platformFlag := range ppViper.GetStringSlice("cli.runtime-platform") {
525+
// split node filter from the specified platform
526+
platform, nodeFilters, err := cliutil.SplitFiltersFromFlag(platformFlag)
527+
if err != nil {
528+
l.Log().Fatalln(err)
529+
}
530+
531+
// create new entry or append filter to existing entry
532+
if _, exists := runtimePlatformFilterMap[platform]; exists {
533+
runtimePlatformFilterMap[platform] = append(runtimePlatformFilterMap[platform], nodeFilters...)
534+
} else {
535+
runtimePlatformFilterMap[platform] = nodeFilters
536+
}
537+
}
538+
539+
for platform, nodeFilters := range runtimePlatformFilterMap {
540+
cfg.Options.Runtime.Platforms = append(cfg.Options.Runtime.Platforms, conf.PlatformWithNodeFilters{
541+
Platform: platform,
542+
NodeFilters: nodeFilters,
543+
})
544+
}
545+
546+
l.Log().Tracef("RuntimePlatformFilterMap: %+v", runtimePlatformFilterMap)
547+
518548
// --env
519549
// envFilterMap will add container env vars to applied node filters
520550
envFilterMap := make(map[string][]string, 1)

cmd/node/nodeCreate.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ func NewCmdNodeCreate() *cobra.Command {
8282

8383
cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"")
8484
cmd.Flags().StringSliceP("runtime-ulimit", "", []string{}, "Specify container runtime ulimit in format \"ulimit=soft:hard\"")
85+
cmd.Flags().StringP("runtime-platform", "", "", "Specify container platform in format \"linux/amd64\"")
8586
cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"")
8687

8788
cmd.Flags().StringSliceP("network", "n", []string{}, "Add node to (another) runtime network")
@@ -166,6 +167,13 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, string)
166167
for index, ulimit := range runtimeUlimitsFlag {
167168
runtimeUlimits[index] = cliutil.ParseRuntimeUlimit[dockerunits.Ulimit](ulimit)
168169
}
170+
171+
// runtime-platform
172+
platform, err := cmd.Flags().GetString("runtime-platform")
173+
if err != nil {
174+
l.Log().Fatalf("No runtime-platform specified: %v", err)
175+
}
176+
169177
// --k3s-node-label
170178
k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label")
171179
if err != nil {
@@ -204,6 +212,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, string)
204212
K3sNodeLabels: k3sNodeLabels,
205213
RuntimeLabels: runtimeLabels,
206214
RuntimeUlimits: runtimeUlimits,
215+
Platform: platform,
207216
Restart: true,
208217
Memory: memory,
209218
Networks: networks,

docs/usage/configfile.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,11 @@ options:
148148
- name: nofile
149149
soft: 26677
150150
hard: 26677
151+
platforms: # overwrite the default platforms for the cluster to pass it as an runtime argument
152+
- platform: linux/arm64/v8
153+
nodeFilters:
154+
- server:*
155+
- agent:*
151156

152157
```
153158

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/k3d-io/k3d/v5
33
go 1.24.4
44

55
require (
6+
github.com/containerd/platforms v0.2.1
67
github.com/goodhosts/hostsfile v0.1.6
78
github.com/google/go-containerregistry v0.20.6
89
github.com/rancher/wharfie v0.6.2
@@ -76,7 +77,7 @@ require (
7677
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
7778
github.com/modern-go/reflect2 v1.0.2 // indirect
7879
github.com/opencontainers/go-digest v1.0.0 // indirect
79-
github.com/opencontainers/image-spec v1.1.1 // indirect
80+
github.com/opencontainers/image-spec v1.1.1
8081
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
8182
github.com/pkg/errors v0.9.1
8283
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X
3232
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
3333
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
3434
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
35+
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
36+
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
3537
github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8=
3638
github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU=
3739
github.com/corpix/uarand v0.0.0-20170723150923-031be390f409 h1:9A+mfQmwzZ6KwUXPc8nHxFtKgn9VIvO3gXAOspIcE3s=

pkg/client/node.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import (
4242
"golang.org/x/sync/errgroup"
4343
"sigs.k8s.io/yaml"
4444

45+
"github.com/containerd/platforms"
4546
"github.com/k3d-io/k3d/v5/pkg/actions"
4647
l "github.com/k3d-io/k3d/v5/pkg/logger"
4748
"github.com/k3d-io/k3d/v5/pkg/runtimes"
@@ -53,6 +54,7 @@ import (
5354
"github.com/k3d-io/k3d/v5/pkg/types/k3s"
5455
"github.com/k3d-io/k3d/v5/pkg/util"
5556
"github.com/k3d-io/k3d/v5/version"
57+
ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"
5658
)
5759

5860
// NodeAddToCluster adds a node to an existing cluster
@@ -650,7 +652,15 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c
650652
}
651653
node.Volumes = append(node.Volumes, fmt.Sprintf("%s:%s:ro", fakemempath, util.MemInfoPath))
652654
// mount empty edac folder, but only if it exists
653-
exists, err := docker.CheckIfDirectoryExists(ctx, node.Image, util.EdacFolderPath)
655+
var platform *ocispecv1.Platform
656+
if node.Platform != "" {
657+
p, err := platforms.Parse(node.Platform)
658+
if err != nil {
659+
return fmt.Errorf("failed to parse platform '%s': %w", node.Platform, err)
660+
}
661+
platform = &p
662+
}
663+
exists, err := docker.CheckIfDirectoryExists(ctx, node.Image, platform, util.EdacFolderPath)
654664
if err != nil {
655665
return fmt.Errorf("failed to check for the existence of edac folder: %w", err)
656666
}

pkg/config/transform.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,22 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
252252
}
253253
}
254254

255+
// -> RUNTIME PLATFORMS
256+
for _, runtimePlatformWithNodeFilters := range simpleConfig.Options.Runtime.Platforms {
257+
if len(runtimePlatformWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
258+
return nil, fmt.Errorf("RuntimePlatformmapping '%s' lacks a node filter, but there's more than one node", runtimePlatformWithNodeFilters.Platform)
259+
}
260+
261+
nodes, err := util.FilterNodes(nodeList, runtimePlatformWithNodeFilters.NodeFilters)
262+
if err != nil {
263+
return nil, fmt.Errorf("failed to filter nodes for runtime platform mapping '%s': %w", runtimePlatformWithNodeFilters.Platform, err)
264+
}
265+
266+
for _, node := range nodes {
267+
node.Platform = runtimePlatformWithNodeFilters.Platform
268+
}
269+
}
270+
255271
// -> ENV
256272
for _, envVarWithNodeFilters := range simpleConfig.Env {
257273
if len(envVarWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {

pkg/config/v1alpha5/types.go

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,11 @@ type LabelWithNodeFilters struct {
7373
NodeFilters []string `mapstructure:"nodeFilters" json:"nodeFilters,omitempty"`
7474
}
7575

76+
type PlatformWithNodeFilters struct {
77+
Platform string `mapstructure:"platform" json:"platform,omitempty"`
78+
NodeFilters []string `mapstructure:"nodeFilters" json:"nodeFilters,omitempty"`
79+
}
80+
7681
type EnvVarWithNodeFilters struct {
7782
EnvVar string `mapstructure:"envVar" json:"envVar,omitempty"`
7883
NodeFilters []string `mapstructure:"nodeFilters" json:"nodeFilters,omitempty"`
@@ -113,12 +118,13 @@ type SimpleConfigOptions struct {
113118
}
114119

115120
type SimpleConfigOptionsRuntime struct {
116-
GPURequest string `mapstructure:"gpuRequest" json:"gpuRequest,omitempty"`
117-
ServersMemory string `mapstructure:"serversMemory" json:"serversMemory,omitempty"`
118-
AgentsMemory string `mapstructure:"agentsMemory" json:"agentsMemory,omitempty"`
119-
HostPidMode bool `mapstructure:"hostPidMode" yjson:"hostPidMode,omitempty"`
120-
Labels []LabelWithNodeFilters `mapstructure:"labels" json:"labels,omitempty"`
121-
Ulimits []Ulimit `mapstructure:"ulimits" json:"ulimits,omitempty"`
121+
GPURequest string `mapstructure:"gpuRequest" json:"gpuRequest,omitempty"`
122+
ServersMemory string `mapstructure:"serversMemory" json:"serversMemory,omitempty"`
123+
AgentsMemory string `mapstructure:"agentsMemory" json:"agentsMemory,omitempty"`
124+
HostPidMode bool `mapstructure:"hostPidMode" yjson:"hostPidMode,omitempty"`
125+
Labels []LabelWithNodeFilters `mapstructure:"labels" json:"labels,omitempty"`
126+
Ulimits []Ulimit `mapstructure:"ulimits" json:"ulimits,omitempty"`
127+
Platforms []PlatformWithNodeFilters `mapstructure:"platforms" json:"platforms,omitempty"`
122128
}
123129

124130
type Ulimit struct {

pkg/runtimes/docker/container.go

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,21 @@ import (
2727
"fmt"
2828
"io"
2929

30+
"github.com/containerd/platforms"
3031
"github.com/docker/docker/api/types"
3132
"github.com/docker/docker/api/types/container"
3233
"github.com/docker/docker/api/types/filters"
3334
dockerimage "github.com/docker/docker/api/types/image"
3435
"github.com/docker/docker/client"
3536
l "github.com/k3d-io/k3d/v5/pkg/logger"
3637
k3d "github.com/k3d-io/k3d/v5/pkg/types"
38+
ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"
3739
"github.com/sirupsen/logrus"
3840
)
3941

4042
// createContainer creates a new docker container from translated specs
4143
func createContainer(ctx context.Context, dockerNode *NodeInDocker, name string) (string, error) {
42-
l.Log().Tracef("Creating docker container with translated config\n%+v\n", dockerNode)
44+
l.Log().Tracef("Creating docker container with translated config: %s\n%+v\n", name, dockerNode)
4345

4446
// initialize docker client
4547
docker, err := GetDockerClient()
@@ -51,10 +53,10 @@ func createContainer(ctx context.Context, dockerNode *NodeInDocker, name string)
5153
// create container
5254
var resp container.CreateResponse
5355
for {
54-
resp, err = docker.ContainerCreate(ctx, &dockerNode.ContainerConfig, &dockerNode.HostConfig, &dockerNode.NetworkingConfig, nil, name)
56+
resp, err = docker.ContainerCreate(ctx, &dockerNode.ContainerConfig, &dockerNode.HostConfig, &dockerNode.NetworkingConfig, dockerNode.PlatformConfig, name)
5557
if err != nil {
5658
if client.IsErrNotFound(err) {
57-
if err := pullImage(ctx, docker, dockerNode.ContainerConfig.Image); err != nil {
59+
if err := pullImage(ctx, docker, dockerNode.ContainerConfig.Image, dockerNode.PlatformConfig); err != nil {
5860
return "", fmt.Errorf("docker failed to pull image '%s': %w", dockerNode.ContainerConfig.Image, err)
5961
}
6062
continue
@@ -105,8 +107,15 @@ func removeContainer(ctx context.Context, ID string) error {
105107
}
106108

107109
// pullImage pulls a container image and outputs progress if --verbose flag is set
108-
func pullImage(ctx context.Context, docker client.APIClient, image string) error {
109-
resp, err := docker.ImagePull(ctx, image, dockerimage.PullOptions{})
110+
func pullImage(ctx context.Context, docker client.APIClient, image string, platform *ocispecv1.Platform) error {
111+
opts := dockerimage.PullOptions{}
112+
113+
if platform != nil {
114+
// if a platform is specified, use it to pull the image
115+
opts.Platform = platforms.Format(*platform)
116+
}
117+
118+
resp, err := docker.ImagePull(ctx, image, opts)
110119
if err != nil {
111120
return fmt.Errorf("docker failed to pull the image '%s': %w", image, err)
112121
}
@@ -168,7 +177,7 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er
168177

169178
// executes an arbitrary command in a container while returning its exit code.
170179
// useful to check something in docker env
171-
func executeCheckInContainer(ctx context.Context, image string, cmd []string) (int64, error) {
180+
func executeCheckInContainer(ctx context.Context, image string, platform *ocispecv1.Platform, cmd []string) (int64, error) {
172181
docker, err := GetDockerClient()
173182
if err != nil {
174183
return -1, fmt.Errorf("failed to create docker client: %w", err)
@@ -186,7 +195,7 @@ func executeCheckInContainer(ctx context.Context, image string, cmd []string) (i
186195
}, nil, nil, nil, "")
187196
if err != nil {
188197
if client.IsErrNotFound(err) {
189-
if err := pullImage(ctx, docker, image); err != nil {
198+
if err := pullImage(ctx, docker, image, platform); err != nil {
190199
return -1, fmt.Errorf("docker failed to pull image '%s': %w", image, err)
191200
}
192201
continue
@@ -219,11 +228,11 @@ func executeCheckInContainer(ctx context.Context, image string, cmd []string) (i
219228
}
220229

221230
// CheckIfDirectoryExists checks for the existence of a given path inside the docker environment
222-
func CheckIfDirectoryExists(ctx context.Context, image string, dir string) (bool, error) {
231+
func CheckIfDirectoryExists(ctx context.Context, image string, platform *ocispecv1.Platform, dir string) (bool, error) {
223232
l.Log().Tracef("checking if dir %s exists in docker environment...", dir)
224233
shellCmd := fmt.Sprintf("[ -d \"%s\" ] && exit 0 || exit 1", dir)
225234
cmd := []string{"sh", "-c", shellCmd}
226-
exitCode, err := executeCheckInContainer(ctx, image, cmd)
235+
exitCode, err := executeCheckInContainer(ctx, image, platform, cmd)
227236
l.Log().Tracef("check dir container returned %d exit code", exitCode)
228237
return exitCode == 0, err
229238
}

0 commit comments

Comments
 (0)