Skip to content

Commit 4f722be

Browse files
stevensbkangstevensbkang
authored
feat(config): add --full flag and strip redundant upstream defaults (… (#143)
1 parent 7e9dfb5 commit 4f722be

13 files changed

Lines changed: 193 additions & 234 deletions

File tree

‎cmd/kubesolo/main.go‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ type kubesolo struct {
4848
loadBalancer bool
4949
localStorage bool
5050
localStorageSharedPath string
51+
fullMode bool
5152
embedded types.Embedded
5253
}
5354

@@ -74,6 +75,7 @@ func service() (*kubesolo, error) {
7475
loadBalancer: *flags.LoadBalancer,
7576
localStorage: *flags.LocalStorage,
7677
localStorageSharedPath: *flags.LocalStorageSharedPath,
78+
fullMode: *flags.Full,
7779
}, nil
7880
}
7981

@@ -114,10 +116,16 @@ func (s *kubesolo) run() {
114116
cancel()
115117
}()
116118

119+
profile := "edge"
120+
if s.fullMode {
121+
profile = "full"
122+
}
123+
117124
log.Info().
118125
Str("version", Version).
119126
Str("build-date", BuildDate).
120127
Str("commit", Commit).
128+
Str("profile", profile).
121129
Msg("starting kubesolo...")
122130

123131
log.Info().Str("component", "kubesolo").Msg("ensuring all embedded dependencies are available...")
@@ -196,7 +204,7 @@ func (s *kubesolo) run() {
196204
{
197205
name: "kubeproxy",
198206
start: func() {
199-
kubeproxyService := kubeproxy.NewService(ctx, cancel, kubeproxyReadyCh, s.embedded.AdminKubeconfigFile)
207+
kubeproxyService := kubeproxy.NewService(ctx, cancel, kubeproxyReadyCh, s.embedded.AdminKubeconfigFile, s.embedded.FullMode)
200208
s.wg.Go(func() {
201209
kubeproxyService.Run(kubeletReadyCh)
202210
})
@@ -420,12 +428,12 @@ func (s *kubesolo) bootstrap() {
420428
},
421429

422430
// Containerd paths
423-
ContainerdDir: filepath.Join(basePath, types.DefaultContainerdDir),
424-
ContainerdSocketFile: filepath.Join(basePath, types.DefaultContainerdDir, types.DefaultContainerdSocket),
425-
ContainerdBinaryFile: filepath.Join(basePath, types.DefaultContainerdDir, "containerd"),
426-
ContainerdImagesDir: filepath.Join(basePath, types.DefaultContainerdDir, "images"),
427-
ContainerdShimBinaryFile: filepath.Join(basePath, types.DefaultContainerdDir, "containerd-shim-runc-v2"),
428-
ContainerdConfigFile: filepath.Join(basePath, types.DefaultContainerdDir, "config.toml"),
431+
ContainerdDir: filepath.Join(basePath, types.DefaultContainerdDir),
432+
ContainerdSocketFile: filepath.Join(basePath, types.DefaultContainerdDir, types.DefaultContainerdSocket),
433+
ContainerdBinaryFile: filepath.Join(basePath, types.DefaultContainerdDir, "containerd"),
434+
ContainerdImagesDir: filepath.Join(basePath, types.DefaultContainerdDir, "images"),
435+
ContainerdShimBinaryFile: filepath.Join(basePath, types.DefaultContainerdDir, "containerd-shim-runc-v2"),
436+
ContainerdConfigFile: filepath.Join(basePath, types.DefaultContainerdDir, "config.toml"),
429437
ContainerdRootDir: filepath.Join(basePath, types.DefaultContainerdDir, "root"),
430438
ContainerdStateDir: filepath.Join(basePath, types.DefaultContainerdDir, "state"),
431439
ContainerdRegistryConfigDir: filepath.Join(basePath, types.DefaultContainerdDir, "registry"),
@@ -476,5 +484,8 @@ func (s *kubesolo) bootstrap() {
476484

477485
// Portainer Edge
478486
IsPortainerEdge: s.portainerEdgeID != "" && s.portainerEdgeKey != "",
487+
488+
// Full mode
489+
FullMode: s.fullMode,
479490
}
480491
}

‎internal/config/flags/flags.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,5 @@ var (
2525
LocalStorageSharedPath = Application.Flag("local-storage-shared-path", "Path to the shared file system for the local storage. Defaults to empty string.").Envar("KUBESOLO_LOCAL_STORAGE_SHARED_PATH").Default("").String()
2626
Debug = Application.Flag("debug", "Enable debug logging. Defaults to false.").Envar("KUBESOLO_DEBUG").Default("false").Bool()
2727
PprofServer = Application.Flag("pprof-server", "Enable pprof server. Defaults to false.").Envar("KUBESOLO_PPROF_SERVER").Default("false").Bool()
28+
Full = Application.Flag("full", "Disable memory-saving overrides and use upstream Kubernetes defaults. Kubesolo still uses NodeSetter in favour of the scheduler. Recommended for CI and developer environments where memory is not constrained. Leave unset for edge deployments.").Envar("KUBESOLO_FULL").Default("false").Bool()
2829
)

‎pkg/kubernetes/apiserver/flags.go‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,11 @@ func (s *service) configureAPIServerFlags(command *cobra.Command) error {
1010

1111
// networking settings
1212
_ = flags.Set("insecure-port", "0")
13-
_ = flags.Set("secure-port", "6443")
14-
_ = flags.Set("bind-address", "0.0.0.0")
1513
_ = flags.Set("advertise-address", s.nodeIP)
1614
_ = flags.Set("service-cluster-ip-range", types.DefaultServiceClusterIPRange)
1715

1816
// etcd configuration
1917
_ = flags.Set("etcd-servers", types.DefaultKineEndpoint)
20-
_ = flags.Set("etcd-compaction-interval", "5m")
21-
_ = flags.Set("etcd-count-metric-poll-period", "0")
22-
_ = flags.Set("etcd-db-metric-poll-interval", "0")
2318

2419
// security and certificates
2520
_ = flags.Set("cert-dir", s.pkiAPIServerDir)
@@ -40,33 +35,40 @@ func (s *service) configureAPIServerFlags(command *cobra.Command) error {
4035
_ = flags.Set("proxy-client-cert-file", s.requestHeaderClientCert)
4136
_ = flags.Set("proxy-client-key-file", s.requestHeaderClientKey)
4237

43-
// authorization and admission
38+
// authorization
4439
_ = flags.Set("allow-privileged", "true")
4540
_ = flags.Set("authorization-mode", "Node,RBAC")
46-
_ = flags.Set("enable-admission-plugins", "NodeRestriction,ServiceAccount,ValidatingAdmissionWebhook,MutatingAdmissionWebhook,DefaultStorageClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionPolicy,MutatingAdmissionPolicy")
47-
_ = flags.Set("disable-admission-plugins", "RuntimeClass,PodSecurity,ClusterTrustBundleAttest,DefaultIngressClass,TaintNodesByCondition,DefaultTolerationSeconds,StorageObjectInUseProtection,PersistentVolumeClaimResize,ResourceQuota,LimitRanger,Priority")
48-
_ = flags.Set("enable-bootstrap-token-auth", "false")
49-
50-
// performance and resource limits
51-
_ = flags.Set("max-requests-inflight", "2000")
52-
_ = flags.Set("max-mutating-requests-inflight", "1000")
53-
_ = flags.Set("min-request-timeout", "180")
54-
_ = flags.Set("request-timeout", "900s")
55-
_ = flags.Set("kubelet-timeout", "30s")
56-
_ = flags.Set("watch-cache", "true")
57-
_ = flags.Set("event-ttl", "1h")
58-
59-
// features and garbage collection
60-
_ = flags.Set("enable-garbage-collector", "true")
61-
_ = flags.Set("profiling", "false")
6241

6342
// feature gates - disable SizeBasedListCostEstimate to suppress "Error getting keys" messages
6443
_ = flags.Set("feature-gates", "SizeBasedListCostEstimate=false")
6544

66-
// audit logging
67-
_ = flags.Set("audit-log-path", "-")
68-
_ = flags.Set("audit-log-maxage", "0")
69-
_ = flags.Set("audit-log-maxbackup", "0")
70-
_ = flags.Set("audit-log-maxsize", "0")
45+
// Edge-optimised overrides — only applied when not in full mode.
46+
// When full mode is enabled, upstream Kubernetes defaults are used instead.
47+
if !s.fullMode {
48+
// etcd metric collection
49+
_ = flags.Set("etcd-count-metric-poll-period", "0")
50+
_ = flags.Set("etcd-db-metric-poll-interval", "0")
51+
52+
// request throttling and timeouts
53+
_ = flags.Set("max-requests-inflight", "2000")
54+
_ = flags.Set("max-mutating-requests-inflight", "1000")
55+
_ = flags.Set("min-request-timeout", "180")
56+
_ = flags.Set("request-timeout", "900s")
57+
_ = flags.Set("kubelet-timeout", "30s")
58+
59+
// diagnostics
60+
_ = flags.Set("profiling", "false")
61+
62+
// admission control
63+
_ = flags.Set("enable-admission-plugins", "NodeRestriction,ServiceAccount,ValidatingAdmissionWebhook,MutatingAdmissionWebhook,DefaultStorageClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionPolicy,MutatingAdmissionPolicy")
64+
_ = flags.Set("disable-admission-plugins", "RuntimeClass,PodSecurity,ClusterTrustBundleAttest,DefaultIngressClass,TaintNodesByCondition,DefaultTolerationSeconds,StorageObjectInUseProtection,PersistentVolumeClaimResize,ResourceQuota,LimitRanger,Priority")
65+
66+
// audit logging
67+
_ = flags.Set("audit-log-path", "-")
68+
_ = flags.Set("audit-log-maxage", "0")
69+
_ = flags.Set("audit-log-maxbackup", "0")
70+
_ = flags.Set("audit-log-maxsize", "0")
71+
}
72+
7173
return nil
7274
}

‎pkg/kubernetes/apiserver/service.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ type service struct {
2828
requestHeaderCAFile string
2929
requestHeaderClientCert string
3030
requestHeaderClientKey string
31+
fullMode bool
3132
kubeSoloWebhook *webhook.Service
3233
}
3334

@@ -51,6 +52,7 @@ func NewService(ctx context.Context, cancel context.CancelFunc, apiServerReady c
5152
requestHeaderCAFile: embedded.RequestHeaderCerts.CACert,
5253
requestHeaderClientCert: embedded.RequestHeaderCerts.ClientCert,
5354
requestHeaderClientKey: embedded.RequestHeaderCerts.ClientKey,
55+
fullMode: embedded.FullMode,
5456
kubeSoloWebhook: webhook.NewService(nodeName, embedded.NodeIP, embedded.PKIDir, embedded.AdminKubeconfigFile, embedded.LoadBalancer),
5557
}
5658
}

‎pkg/kubernetes/controller/flags.go‎

Lines changed: 41 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,8 @@ func (s *service) configureControllerManagerFlags(command *cobra.Command) {
99
flags := command.Flags()
1010

1111
// controller manager settings
12-
_ = flags.Set("bind-address", "0.0.0.0")
13-
_ = flags.Set("secure-port", "10257")
1412
_ = flags.Set("allocate-node-cidrs", "true")
1513
_ = flags.Set("cluster-cidr", types.DefaultPodCIDR)
16-
_ = flags.Set("v", "0")
1714
_ = flags.Set("service-account-private-key-file", s.serviceAccountKeyFile)
1815
_ = flags.Set("kubeconfig", s.adminKubeconfigFile)
1916
_ = flags.Set("authentication-kubeconfig", s.adminKubeconfigFile)
@@ -23,50 +20,51 @@ func (s *service) configureControllerManagerFlags(command *cobra.Command) {
2320
_ = flags.Set("tls-cert-file", s.controllerManagerCertFile)
2421
_ = flags.Set("tls-private-key-file", s.controllerManagerKeyFile)
2522
_ = flags.Set("leader-elect", "false")
26-
_ = flags.Set("profiling", "false")
2723
_ = flags.Set("use-service-account-credentials", "true")
2824

29-
// controllers
30-
_ = flags.Set("controllers", "deployment,replicaset,service,serviceaccount,namespace,attachdetach,endpoint,daemonset,statefulset,root-ca-certificate-publisher-controller,serviceaccount-token-controller,node-ipam-controller,endpointslice-controller,persistentvolume-binder-controller,job-controller,cronjob-controller,garbage-collector-controller,disruption,csrsigning,clusterrole-aggregation")
25+
// Edge-optimised overrides — only applied when not in full mode.
26+
// When full mode is enabled, upstream Kubernetes defaults are used instead.
27+
if !s.fullMode {
28+
// controllers
29+
_ = flags.Set("controllers", "deployment,replicaset,service,serviceaccount,namespace,attachdetach,endpoint,daemonset,statefulset,root-ca-certificate-publisher-controller,serviceaccount-token-controller,node-ipam-controller,endpointslice-controller,persistentvolume-binder-controller,job-controller,cronjob-controller,garbage-collector-controller,disruption,csrsigning,clusterrole-aggregation")
3130

32-
// thresholds
33-
_ = flags.Set("terminated-pod-gc-threshold", "20")
34-
_ = flags.Set("large-cluster-size-threshold", "10")
35-
_ = flags.Set("unhealthy-zone-threshold", "0.7")
31+
_ = flags.Set("profiling", "false")
32+
_ = flags.Set("terminated-pod-gc-threshold", "20")
33+
_ = flags.Set("large-cluster-size-threshold", "10")
34+
_ = flags.Set("unhealthy-zone-threshold", "0.7")
3635

37-
// sync settings
38-
_ = flags.Set("concurrent-deployment-syncs", "2")
39-
_ = flags.Set("concurrent-replicaset-syncs", "2")
40-
_ = flags.Set("concurrent-daemonset-syncs", "2")
41-
_ = flags.Set("concurrent-job-syncs", "2")
42-
_ = flags.Set("concurrent-endpoint-syncs", "2")
43-
_ = flags.Set("concurrent-service-endpoint-syncs", "2")
44-
_ = flags.Set("concurrent-gc-syncs", "2")
45-
_ = flags.Set("concurrent-namespace-syncs", "2")
46-
_ = flags.Set("concurrent-cron-job-syncs", "2")
47-
_ = flags.Set("concurrent-horizontal-pod-autoscaler-syncs", "2")
48-
_ = flags.Set("concurrent-rc-syncs", "2")
49-
_ = flags.Set("concurrent-resource-quota-syncs", "2")
50-
_ = flags.Set("concurrent-service-syncs", "2")
51-
_ = flags.Set("concurrent-serviceaccount-token-syncs", "2")
52-
_ = flags.Set("concurrent-statefulset-syncs", "2")
53-
_ = flags.Set("concurrent-ttl-after-finished-syncs", "2")
54-
_ = flags.Set("concurrent-ephemeralvolume-syncs", "2")
55-
_ = flags.Set("concurrent-validating-admission-policy-status-syncs", "2")
56-
_ = flags.Set("mirroring-concurrent-service-endpoint-syncs", "2")
36+
// sync settings
37+
_ = flags.Set("concurrent-deployment-syncs", "2")
38+
_ = flags.Set("concurrent-replicaset-syncs", "2")
39+
_ = flags.Set("concurrent-job-syncs", "2")
40+
_ = flags.Set("concurrent-endpoint-syncs", "2")
41+
_ = flags.Set("concurrent-service-endpoint-syncs", "2")
42+
_ = flags.Set("concurrent-gc-syncs", "2")
43+
_ = flags.Set("concurrent-namespace-syncs", "2")
44+
_ = flags.Set("concurrent-cron-job-syncs", "2")
45+
_ = flags.Set("concurrent-horizontal-pod-autoscaler-syncs", "2")
46+
_ = flags.Set("concurrent-rc-syncs", "2")
47+
_ = flags.Set("concurrent-resource-quota-syncs", "2")
48+
_ = flags.Set("concurrent-service-syncs", "2")
49+
_ = flags.Set("concurrent-serviceaccount-token-syncs", "2")
50+
_ = flags.Set("concurrent-statefulset-syncs", "2")
51+
_ = flags.Set("concurrent-ttl-after-finished-syncs", "2")
52+
_ = flags.Set("concurrent-ephemeralvolume-syncs", "2")
53+
_ = flags.Set("concurrent-validating-admission-policy-status-syncs", "2")
54+
_ = flags.Set("mirroring-concurrent-service-endpoint-syncs", "2")
5755

58-
// sync period
59-
_ = flags.Set("horizontal-pod-autoscaler-sync-period", "60s")
60-
_ = flags.Set("node-monitor-period", "60s")
61-
_ = flags.Set("pvclaimbinder-sync-period", "120s")
62-
_ = flags.Set("resource-quota-sync-period", "15m")
63-
_ = flags.Set("namespace-sync-period", "15m")
64-
_ = flags.Set("route-reconciliation-period", "60s")
65-
_ = flags.Set("attach-detach-reconcile-sync-period", "10m")
66-
_ = flags.Set("node-monitor-grace-period", "300s")
67-
_ = flags.Set("min-resync-period", "12h")
56+
// sync period
57+
_ = flags.Set("horizontal-pod-autoscaler-sync-period", "60s")
58+
_ = flags.Set("node-monitor-period", "60s")
59+
_ = flags.Set("pvclaimbinder-sync-period", "120s")
60+
_ = flags.Set("resource-quota-sync-period", "15m")
61+
_ = flags.Set("namespace-sync-period", "15m")
62+
_ = flags.Set("route-reconciliation-period", "60s")
63+
_ = flags.Set("attach-detach-reconcile-sync-period", "10m")
64+
_ = flags.Set("node-monitor-grace-period", "300s")
6865

69-
// api server interactions
70-
_ = flags.Set("kube-api-qps", "50")
71-
_ = flags.Set("kube-api-burst", "100")
66+
// api server interactions
67+
_ = flags.Set("kube-api-qps", "50")
68+
_ = flags.Set("kube-api-burst", "100")
69+
}
7270
}

‎pkg/kubernetes/controller/service.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type service struct {
1919
caFile string
2020
adminKubeconfigFile string
2121
serviceAccountKeyFile string
22+
fullMode bool
2223
}
2324

2425
// NewService creates a new controller service
@@ -33,5 +34,6 @@ func NewService(ctx context.Context, cancel context.CancelFunc, controllerReady
3334
caFile: embedded.CACerts.Cert,
3435
adminKubeconfigFile: embedded.AdminKubeconfigFile,
3536
serviceAccountKeyFile: embedded.ServiceAccountKeyFile,
37+
fullMode: embedded.FullMode,
3638
}
3739
}

‎pkg/kubernetes/kubelet/config.go‎

Lines changed: 38 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,11 @@ func (s *service) writeKubeletConfigFile() error {
5151
}
5252

5353
func (s *service) generateKubeletConfig() map[string]any {
54-
return map[string]any{
55-
"kind": "KubeletConfiguration",
56-
"apiVersion": "kubelet.config.k8s.io/v1beta1",
57-
"enableServer": true,
54+
config := map[string]any{
55+
"kind": "KubeletConfiguration",
56+
"apiVersion": "kubelet.config.k8s.io/v1beta1",
5857

5958
"containerRuntimeEndpoint": "unix://" + s.containerdSockFile,
60-
"imageServiceEndpoint": "unix://" + s.containerdSockFile,
6159

6260
"authentication": map[string]any{
6361
"anonymous": map[string]any{
@@ -88,52 +86,42 @@ func (s *service) generateKubeletConfig() map[string]any {
8886

8987
"cgroupDriver": cgroupDriver(),
9088

91-
"registerNode": true,
92-
"readOnlyPort": 0,
93-
"port": 10250,
94-
"syncFrequency": "5m0s",
95-
"fileCheckFrequency": "2m0s",
96-
"httpCheckFrequency": "2m0s",
97-
"nodeStatusUpdateFrequency": "60s",
98-
"nodeStatusReportFrequency": "15m0s",
99-
"volumeStatsAggPeriod": "5m0s",
100-
"imageMinimumGCAge": "10m0s",
101-
"imageMaximumGCAge": "0s",
102-
"imageGCHighThresholdPercent": 95,
103-
"imageGCLowThresholdPercent": 80,
104-
"runtimeRequestTimeout": "60s",
105-
"cpuManagerReconcilePeriod": "60s",
106-
"streamingConnectionIdleTimeout": "1h0m0s",
107-
"rotateCertificates": true,
108-
109-
"registerWithTaints": []map[string]any{},
110-
111-
"evictionHard": map[string]string{
112-
"memory.available": "75Mi",
113-
"nodefs.available": "50Mi",
114-
},
115-
"systemReserved": map[string]string{"memory": "25Mi"},
116-
"kubeReserved": map[string]string{"memory": "25Mi"},
117-
"failSwapOn": false,
118-
119-
"kubeAPIQPS": 10,
120-
"kubeAPIBurst": 20,
121-
"serializeImagePulls": true,
122-
"imagePullProgressDeadline": "1m",
123-
124-
"registryPullQPS": 5,
125-
"registryBurst": 10,
126-
127-
"eventRecordQPS": 5,
128-
"eventBurst": 10,
89+
"readOnlyPort": 0,
90+
"rotateCertificates": true,
12991

130-
"containerLogMaxSize": "512Ki",
131-
"enableProfilingHandler": false,
132-
"enableDebugFlagsHandler": false,
133-
"maxPods": 20,
92+
"failSwapOn": false,
93+
}
13494

135-
"featureGates": map[string]bool{
136-
"RotateKubeletServerCertificate": true,
137-
},
95+
// Edge-optimised overrides — only applied when not in full mode.
96+
// When full mode is enabled, upstream Kubernetes defaults are used instead.
97+
if !s.fullMode {
98+
config["enableProfilingHandler"] = false
99+
config["enableDebugFlagsHandler"] = false
100+
config["streamingConnectionIdleTimeout"] = "1h0s"
101+
config["syncFrequency"] = "5m0s"
102+
config["fileCheckFrequency"] = "2m0s"
103+
config["httpCheckFrequency"] = "2m0s"
104+
config["nodeStatusUpdateFrequency"] = "60s"
105+
config["nodeStatusReportFrequency"] = "15m0s"
106+
config["volumeStatsAggPeriod"] = "5m0s"
107+
config["imageMinimumGCAge"] = "10m0s"
108+
config["imageMaximumGCAge"] = "0s"
109+
config["imageGCHighThresholdPercent"] = 95
110+
config["runtimeRequestTimeout"] = "60s"
111+
config["cpuManagerReconcilePeriod"] = "60s"
112+
config["kubeAPIQPS"] = 10
113+
config["kubeAPIBurst"] = 20
114+
config["eventRecordQPS"] = 5
115+
config["eventBurst"] = 10
116+
config["containerLogMaxSize"] = "512Ki"
117+
config["maxPods"] = 20
118+
config["evictionHard"] = map[string]string{
119+
"memory.available": "75Mi",
120+
"nodefs.available": "50Mi",
121+
}
122+
config["systemReserved"] = map[string]string{"memory": "25Mi"}
123+
config["kubeReserved"] = map[string]string{"memory": "25Mi"}
138124
}
125+
126+
return config
139127
}

0 commit comments

Comments
 (0)