Skip to content

Commit db0c573

Browse files
tkuchikiclaude
andauthored
fix: preserve percent scaledownStepSize in defaulting webhook (#252)
The SpannerAutoscaler mutating webhook defaulted ScaledownStepSize to 2000 whenever `IntValue() == 0`. For an intstr.IntOrString holding a percent string such as "10%", IntValue() returns 0, so any percentage was misclassified as "unset" and silently overwritten with 2000. As a result, `scaledownStepSize: 10%` scaled down by a fixed 2000 PU per step regardless of the current processing units. Detect "unset" by comparing against the zero value instead, so both percent and integer values are preserved. Omitted fields are still defaulted to 2000 (the CRD schema default already covers the omitted case, applied by the apiserver before the webhook runs). Add a webhook test asserting a percent value survives defaulting, and an emulator-based integration test that observes the scale-down step sequence for both integer and percent configurations. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 8522e9e commit db0c573

3 files changed

Lines changed: 266 additions & 2 deletions

File tree

internal/webhook/v1beta1/spannerautoscaler_webhook.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,12 @@ func (*SpannerAutoscalerCustomDefaulter) Default(_ context.Context, obj *spanner
8181
}
8282
}
8383

84-
// set default ScaledownStepSize
85-
if obj.Spec.ScaleConfig.ScaledownStepSize.IntValue() == 0 {
84+
// set default ScaledownStepSize (only when truly unset).
85+
// IntValue() returns 0 for percent strings such as "10%", so the previous
86+
// `IntValue() == 0` check misclassified a percentage as "unset" and clobbered
87+
// it with 2000. Compare against the zero value instead so percent and integer
88+
// values are both preserved.
89+
if obj.Spec.ScaleConfig.ScaledownStepSize == (intstr.IntOrString{}) {
8690
obj.Spec.ScaleConfig.ScaledownStepSize = intstr.FromInt(2000)
8791
}
8892

internal/webhook/v1beta1/spannerautoscaler_webhook_test.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,20 @@ var _ = Describe("SpannerAutoscaler validation", func() {
150150
Expect(result.Spec.ScaleConfig.ScaledownStepSize.IntVal).To(Equal(int32(2000)))
151151
})
152152
})
153+
154+
Context("scale down step size is set as a percentage", func() {
155+
BeforeEach(func() {
156+
testResource.Spec.ScaleConfig.ScaledownStepSize = intstr.FromString("10%")
157+
})
158+
159+
It("should preserve the percentage and not overwrite it with the default", func() {
160+
result, err := createResource(testResource)
161+
Expect(err).ToNot(HaveOccurred())
162+
Expect(result.Spec.ScaleConfig.ComputeType).To(Equal(spannerv1beta1.ComputeTypePU))
163+
Expect(result.Spec.ScaleConfig.ScaledownStepSize.Type).To(Equal(intstr.String))
164+
Expect(result.Spec.ScaleConfig.ScaledownStepSize.StrVal).To(Equal("10%"))
165+
})
166+
})
153167
})
154168

155169
Context("processing unit node is set", func() {
Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
//go:build integration
2+
3+
package integration_test
4+
5+
import (
6+
"context"
7+
"encoding/json"
8+
"fmt"
9+
"path/filepath"
10+
"testing"
11+
"time"
12+
13+
ctrl "sigs.k8s.io/controller-runtime"
14+
ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config"
15+
16+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17+
"k8s.io/apimachinery/pkg/types"
18+
"k8s.io/apimachinery/pkg/util/intstr"
19+
k8sscheme "k8s.io/client-go/kubernetes/scheme"
20+
"sigs.k8s.io/controller-runtime/pkg/envtest"
21+
logf "sigs.k8s.io/controller-runtime/pkg/log"
22+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
23+
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
24+
25+
spannerv1alpha1 "github.com/mercari/spanner-autoscaler/api/v1alpha1"
26+
spannerv1beta1 "github.com/mercari/spanner-autoscaler/api/v1beta1"
27+
"github.com/mercari/spanner-autoscaler/internal/controller"
28+
webhookv1beta1 "github.com/mercari/spanner-autoscaler/internal/webhook/v1beta1"
29+
)
30+
31+
// TestController_ScaledownStepSize_Percent observes, end-to-end via the
32+
// emulators, the scale-down step size when scaledownStepSize is set to "10%".
33+
//
34+
// It contrasts two paths:
35+
// - applyWebhookDefault=true: apply the mutating webhook Default() before
36+
// creation, exactly like production, to check whether "10%" is preserved
37+
// rather than being overwritten with 2000.
38+
// - applyWebhookDefault=false: pass "10%" straight to the controller without
39+
// going through Default(), to observe the step-size resolution logic alone.
40+
//
41+
// In every case the instance starts at initPU under a low CPU load so it scales
42+
// down, and the observed PU sequence (and each step delta) is recorded.
43+
func TestController_ScaledownStepSize_Percent(t *testing.T) {
44+
cases := []struct {
45+
name string
46+
projectID string
47+
instanceID string
48+
saName string
49+
initPU int
50+
maxPU int
51+
applyWebhookDefault bool
52+
}{
53+
{
54+
name: "with_webhook_default_production_path",
55+
projectID: "sd-def-project",
56+
instanceID: "sd-def-instance",
57+
saName: "sd-def-sa",
58+
initPU: 10000,
59+
maxPU: 10000,
60+
applyWebhookDefault: true,
61+
},
62+
{
63+
name: "without_webhook_default_direct_10pct",
64+
projectID: "sd-raw-project",
65+
instanceID: "sd-raw-instance",
66+
saName: "sd-raw-sa",
67+
initPU: 10000,
68+
maxPU: 10000,
69+
applyWebhookDefault: false,
70+
},
71+
{
72+
// Post-fix check: even on the production-equivalent path (after the
73+
// webhook Default() is applied), "10%" is preserved and the instance
74+
// scales down by roughly 10% of the current PU from 35000.
75+
name: "from_35000_with_webhook_default",
76+
projectID: "sd-35k-project",
77+
instanceID: "sd-35k-instance",
78+
saName: "sd-35k-sa",
79+
initPU: 35000,
80+
maxPU: 35000,
81+
applyWebhookDefault: true,
82+
},
83+
}
84+
85+
for _, tc := range cases {
86+
tc := tc
87+
t.Run(tc.name, func(t *testing.T) {
88+
seq, effective := runScaledownScenario(t, tc.projectID, tc.instanceID, tc.saName, tc.initPU, tc.maxPU, tc.applyWebhookDefault)
89+
t.Logf("[%s] effective scaledownStepSize on the object = %q", tc.name, effective)
90+
t.Logf("[%s] observed PU sequence = %v", tc.name, seq)
91+
for i := 1; i < len(seq); i++ {
92+
t.Logf("[%s] step %d: %d -> %d (delta=%d)",
93+
tc.name, i, seq[i-1], seq[i], seq[i-1]-seq[i])
94+
}
95+
})
96+
}
97+
}
98+
99+
func runScaledownScenario(t *testing.T, projectID, instanceID, saName string, initPU, maxPU int, applyWebhookDefault bool) (seq []int, effectiveStepSize string) {
100+
t.Helper()
101+
102+
const (
103+
referenceCPU = 0.10 // Workload = 0.10 * initPU; cpu = Workload / PU
104+
syncInterval = 1 * time.Second
105+
scaleUpInterval = 1 * time.Second
106+
scaleDownInterval = 1 * time.Second
107+
)
108+
targetCPUVal := 40
109+
targetCPU := &targetCPUVal
110+
111+
logf.SetLogger(zap.New(zap.UseDevMode(true)))
112+
113+
// Configure a low CPU load on the total metric to trigger scale-down.
114+
body, _ := json.Marshal(map[string]interface{}{
115+
"total": map[string]interface{}{
116+
"cpu_utilization": referenceCPU,
117+
"reference_processing_units": initPU,
118+
},
119+
})
120+
adminPUT(t, fmt.Sprintf("/workload/%s/%s", projectID, instanceID), body)
121+
t.Cleanup(func() { adminDELETE(t, fmt.Sprintf("/workload/%s/%s", projectID, instanceID)) })
122+
123+
createSpannerInstance(t, projectID, instanceID, initPU)
124+
125+
testEnv := &envtest.Environment{
126+
CRDDirectoryPaths: []string{filepath.Join(repoRoot(), "config", "crd", "bases")},
127+
ErrorIfCRDPathMissing: true,
128+
}
129+
cfg, err := testEnv.Start()
130+
if err != nil {
131+
t.Fatalf("failed to start envtest: %v", err)
132+
}
133+
t.Cleanup(func() { testEnv.Stop() }) //nolint:errcheck
134+
135+
if err := spannerv1alpha1.AddToScheme(k8sscheme.Scheme); err != nil {
136+
t.Fatalf("add v1alpha1 scheme: %v", err)
137+
}
138+
if err := spannerv1beta1.AddToScheme(k8sscheme.Scheme); err != nil {
139+
t.Fatalf("add v1beta1 scheme: %v", err)
140+
}
141+
142+
skipValidation := true
143+
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
144+
Scheme: k8sscheme.Scheme,
145+
Metrics: metricsserver.Options{BindAddress: "0"},
146+
Controller: ctrlconfig.Controller{SkipNameValidation: &skipValidation},
147+
})
148+
if err != nil {
149+
t.Fatalf("failed to create manager: %v", err)
150+
}
151+
152+
reconciler := controller.NewSpannerAutoscalerReconciler(
153+
mgr.GetClient(),
154+
mgr.GetAPIReader(),
155+
mgr.GetScheme(),
156+
mgr.GetEventRecorderFor("sd-controller"),
157+
logf.Log.WithName("sd"),
158+
controller.WithSpannerEndpoint(spannerEmulatorAddr()),
159+
controller.WithMetricsEndpoint(monitoringGRPCAddr()),
160+
controller.WithSyncInterval(syncInterval),
161+
controller.WithScaleUpInterval(scaleUpInterval),
162+
controller.WithScaleDownInterval(scaleDownInterval),
163+
)
164+
if err := reconciler.SetupWithManager(mgr); err != nil {
165+
t.Fatalf("failed to setup controller: %v", err)
166+
}
167+
168+
mgrCtx, mgrCancel := context.WithCancel(context.Background())
169+
t.Cleanup(mgrCancel)
170+
t.Cleanup(reconciler.StopAll)
171+
go func() {
172+
if err := mgr.Start(mgrCtx); err != nil {
173+
t.Logf("manager exited: %v", err)
174+
}
175+
}()
176+
177+
k8sClient := mgr.GetClient()
178+
ctx := context.Background()
179+
nn := types.NamespacedName{Namespace: "default", Name: saName}
180+
181+
sa := &spannerv1beta1.SpannerAutoscaler{
182+
ObjectMeta: metav1.ObjectMeta{
183+
Name: nn.Name,
184+
Namespace: nn.Namespace,
185+
},
186+
Spec: spannerv1beta1.SpannerAutoscalerSpec{
187+
TargetInstance: spannerv1beta1.TargetInstance{
188+
ProjectID: projectID,
189+
InstanceID: instanceID,
190+
},
191+
Authentication: spannerv1beta1.Authentication{
192+
Type: spannerv1beta1.AuthTypeADC,
193+
},
194+
ScaleConfig: spannerv1beta1.ScaleConfig{
195+
ComputeType: spannerv1beta1.ComputeTypePU,
196+
ProcessingUnits: spannerv1beta1.ScaleConfigPUs{
197+
Min: 100,
198+
Max: maxPU,
199+
},
200+
// User intent: scale down by 10% of the current PU per step.
201+
ScaledownStepSize: intstr.FromString("10%"),
202+
ScaleupStepSize: intstr.FromInt(1000),
203+
TargetCPUUtilization: spannerv1beta1.TargetCPUUtilization{
204+
Total: targetCPU,
205+
},
206+
},
207+
},
208+
}
209+
210+
// In production the mutating webhook Default() always runs before creation.
211+
// This envtest environment does not register the webhook, so call it
212+
// explicitly here to reproduce the same state as production.
213+
if applyWebhookDefault {
214+
d := &webhookv1beta1.SpannerAutoscalerCustomDefaulter{}
215+
if err := d.Default(ctx, sa); err != nil {
216+
t.Fatalf("webhook Default() failed: %v", err)
217+
}
218+
}
219+
effectiveStepSize = sa.Spec.ScaleConfig.ScaledownStepSize.String()
220+
221+
if err := k8sClient.Create(ctx, sa); err != nil {
222+
t.Fatalf("failed to create SpannerAutoscaler: %v", err)
223+
}
224+
225+
// Observe the PU sequence. status.CurrentProcessingUnits reflects the
226+
// instance's actual PU in the emulator (populated by the syncer); record it
227+
// whenever the value changes.
228+
seq = []int{}
229+
last := -1
230+
deadline := time.Now().Add(25 * time.Second)
231+
for time.Now().Before(deadline) {
232+
var updated spannerv1beta1.SpannerAutoscaler
233+
if err := k8sClient.Get(ctx, nn, &updated); err != nil {
234+
time.Sleep(200 * time.Millisecond)
235+
continue
236+
}
237+
cur := updated.Status.CurrentProcessingUnits
238+
if cur != 0 && cur != last {
239+
seq = append(seq, cur)
240+
last = cur
241+
t.Logf("[%s] observed PU=%d (totalCPU=%d%%)", saName, cur, updated.Status.CurrentTotalCPUUtilization)
242+
}
243+
time.Sleep(200 * time.Millisecond)
244+
}
245+
return seq, effectiveStepSize
246+
}

0 commit comments

Comments
 (0)