Skip to content

Commit 20491dd

Browse files
eberlepmajst01
eberlep
and
majst01
authored
Support for Dedicated IP Addresses (#513)
* Downgrade deps * Add new field * Initial implementation of dedicated IP * update status controller as well * cleanup + logging * cleanup * Use "" for deletion but ignore nil * Revert "Use "" for deletion but ignore nil" This reverts commit a0f21d8. * Add todo/comment * Check if the IP could actually be assigned before updating the status * Use correct errror variable * Add support for dedicatedLoadBalancerPort * Add support for dedicatedLoadBalancerPort * Replace the whole spec * Replace the whole spec * Refactor * Add support for additional sockets in status * Reworked the Status Socket handling to keep existing replications going * Apply suggestions from code review Co-authored-by: Stefan Majer <[email protected]> * Move consts for DedicatedLoadBalancerIP tags to postgreslet * capitalization * capitalization --------- Co-authored-by: Stefan Majer <[email protected]>
1 parent cf654c6 commit 20491dd

File tree

10 files changed

+601
-148
lines changed

10 files changed

+601
-148
lines changed

‎api/v1/postgres_types.go

Lines changed: 86 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,13 @@ const (
8383
defaultPostgresParamValueSSLCiphers = "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384"
8484
defaultPostgresParamValueWalKeepSegements = "64"
8585
defaultPostgresParamValueWalKeepSize = "1GB"
86+
87+
// PostgresAutoAssignedIPNamePrefix a prefix to add to the generated random name
88+
PostgresAutoAssignedIPNamePrefix = "pgaas-autoassign-"
89+
// PostgresAutoAssignedIPLabelKey tag to identify ips auto-assigned for a postgres
90+
PostgresAutoAssignedIPLabelKey = "postgres.database.fits.cloud/auto-assigned-ip"
91+
// PostgresAutoAssignedIPLabel tag to identify ips auto-assigned for a postgres
92+
PostgresAutoAssignedIPLabel = PostgresAutoAssignedIPLabelKey + "=true"
8693
)
8794

8895
var (
@@ -188,6 +195,12 @@ type PostgresSpec struct {
188195

189196
// PostgresParams additional parameters that are passed along to the postgres config
190197
PostgresParams map[string]string `json:"postgresParams,omitempty"`
198+
199+
// DedicatedLoadBalancerIP The ip to use for the load balancer
200+
DedicatedLoadBalancerIP *string `json:"dedicatedLoadBalancerIP,omitempty"`
201+
202+
// DedicatedLoadBalancerPort The port to use for the load balancer
203+
DedicatedLoadBalancerPort *int32 `json:"dedicatedLoadBalancerPort,omitempty"`
191204
}
192205

193206
// AccessList defines the type of restrictions to access the database
@@ -228,6 +241,8 @@ type PostgresStatus struct {
228241

229242
Socket Socket `json:"socket,omitempty"`
230243

244+
AdditionalSockets []Socket `json:"additionalSockets,omitempty"`
245+
231246
ChildName string `json:"childName,omitempty"`
232247
}
233248

@@ -324,7 +339,7 @@ func (p *Postgres) ToKey() *types.NamespacedName {
324339
}
325340
}
326341

327-
func (p *Postgres) ToSvcLB(lbIP string, lbPort int32, enableStandbyLeaderSelector bool, enableLegacyStandbySelector bool, standbyClustersSourceRanges []string) *corev1.Service {
342+
func (p *Postgres) ToSharedSvcLB(lbIP string, lbPort int32, enableStandbyLeaderSelector bool, enableLegacyStandbySelector bool, standbyClustersSourceRanges []string) *corev1.Service {
328343
lb := &corev1.Service{}
329344
lb.Spec.Type = "LoadBalancer"
330345

@@ -333,7 +348,7 @@ func (p *Postgres) ToSvcLB(lbIP string, lbPort int32, enableStandbyLeaderSelecto
333348
}
334349

335350
lb.Namespace = p.ToPeripheralResourceNamespace()
336-
lb.Name = p.ToSvcLBName()
351+
lb.Name = p.ToSharedSvcLBName()
337352
lb.SetLabels(SvcLoadBalancerLabel)
338353

339354
lbsr := []string{}
@@ -385,17 +400,82 @@ func (p *Postgres) ToSvcLB(lbIP string, lbPort int32, enableStandbyLeaderSelecto
385400
return lb
386401
}
387402

388-
// ToSvcLBName returns the name of the peripheral resource Service LoadBalancer.
403+
// ToSharedSvcLBName returns the name of the peripheral resource Service LoadBalancer.
389404
// It's different from all other peripheral resources because the operator
390405
// already generates one service with that name.
391-
func (p *Postgres) ToSvcLBName() string {
406+
func (p *Postgres) ToSharedSvcLBName() string {
392407
return p.ToPeripheralResourceName() + "-external"
393408
}
394409

395-
func (p *Postgres) ToSvcLBNamespacedName() *types.NamespacedName {
410+
func (p *Postgres) ToSharedSvcLBNamespacedName() *types.NamespacedName {
411+
return &types.NamespacedName{
412+
Namespace: p.ToPeripheralResourceNamespace(),
413+
Name: p.ToSharedSvcLBName(),
414+
}
415+
}
416+
417+
func (p *Postgres) ToDedicatedSvcLB(lbIP string, lbPort int32, standbyClustersSourceRanges []string) *corev1.Service {
418+
lb := &corev1.Service{}
419+
lb.Spec.Type = "LoadBalancer"
420+
421+
lb.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal
422+
423+
lb.Namespace = p.ToPeripheralResourceNamespace()
424+
lb.Name = p.ToDedicatedSvcLBName()
425+
lb.SetLabels(SvcLoadBalancerLabel)
426+
427+
lbsr := []string{}
428+
if p.HasSourceRanges() {
429+
for _, src := range p.Spec.AccessList.SourceRanges {
430+
lbsr = append(lbsr, src)
431+
}
432+
}
433+
for _, scsr := range standbyClustersSourceRanges {
434+
lbsr = append(lbsr, scsr)
435+
}
436+
if len(lbsr) == 0 {
437+
// block by default
438+
lbsr = append(lbsr, "255.255.255.255/32")
439+
}
440+
lb.Spec.LoadBalancerSourceRanges = lbsr
441+
442+
port := corev1.ServicePort{}
443+
port.Name = "postgresql"
444+
port.Port = lbPort
445+
port.Protocol = corev1.ProtocolTCP
446+
port.TargetPort = intstr.FromInt(5432)
447+
lb.Spec.Ports = []corev1.ServicePort{port}
448+
449+
lb.Spec.Selector = map[string]string{
450+
ApplicationLabelName: ApplicationLabelValue,
451+
"cluster-name": p.ToPeripheralResourceName(),
452+
"team": p.generateTeamID(),
453+
}
454+
if p.IsReplicationPrimary() {
455+
lb.Spec.Selector[SpiloRoleLabelName] = SpiloRoleLabelValueMaster
456+
} else {
457+
// select the first pod in the statefulset
458+
lb.Spec.Selector[StatefulsetPodNameLabelName] = p.ToPeripheralResourceName() + "-0"
459+
}
460+
461+
if len(lbIP) > 0 {
462+
lb.Spec.LoadBalancerIP = lbIP
463+
}
464+
465+
return lb
466+
}
467+
468+
// ToSharedSvcLBName returns the name of the peripheral resource Service LoadBalancer.
469+
// It's different from all other peripheral resources because the operator
470+
// already generates one service with that name.
471+
func (p *Postgres) ToDedicatedSvcLBName() string {
472+
return p.ToPeripheralResourceName() + "-dedicated"
473+
}
474+
475+
func (p *Postgres) ToDedicatedSvcLBNamespacedName() *types.NamespacedName {
396476
return &types.NamespacedName{
397477
Namespace: p.ToPeripheralResourceNamespace(),
398-
Name: p.ToSvcLBName(),
478+
Name: p.ToDedicatedSvcLBName(),
399479
}
400480
}
401481

‎api/v1/zz_generated.deepcopy.go

Lines changed: 16 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/database.fits.cloud_postgres.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,14 @@ spec:
9999
replication is used for the standby postgres
100100
type: boolean
101101
type: object
102+
dedicatedLoadBalancerIP:
103+
description: DedicatedLoadBalancerIP The ip to use for the load balancer
104+
type: string
105+
dedicatedLoadBalancerPort:
106+
description: DedicatedLoadBalancerPort The port to use for the load
107+
balancer
108+
format: int32
109+
type: integer
102110
description:
103111
description: Description
104112
type: string
@@ -169,6 +177,17 @@ spec:
169177
status:
170178
description: PostgresStatus defines the observed state of Postgres
171179
properties:
180+
additionalSockets:
181+
items:
182+
description: Socket represents load-balancer socket of Postgres
183+
properties:
184+
ip:
185+
type: string
186+
port:
187+
format: int32
188+
type: integer
189+
type: object
190+
type: array
172191
childName:
173192
type: string
174193
description:

‎controllers/postgres_controller.go

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -131,11 +131,16 @@ func (r *PostgresReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
131131
}
132132
log.Info("corresponding CRD ClusterwideNetworkPolicy deleted")
133133

134-
if err := r.LBManager.DeleteSvcLB(ctx, instance); err != nil {
135-
r.recorder.Eventf(instance, "Warning", "Error", "failed to delete Service: %v", err)
134+
if err := r.LBManager.DeleteSharedSvcLB(ctx, instance); err != nil {
135+
r.recorder.Eventf(instance, "Warning", "Error", "failed to delete Service with shared ip: %v", err)
136136
return ctrl.Result{}, err
137137
}
138-
log.Info("corresponding Service of type LoadBalancer deleted")
138+
139+
if err := r.LBManager.DeleteDedicatedSvcLB(ctx, instance); err != nil {
140+
r.recorder.Eventf(instance, "Warning", "Error", "failed to delete Service with dedicated ip: %v", err)
141+
return ctrl.Result{}, err
142+
}
143+
log.Info("corresponding Service(s) of type LoadBalancer deleted")
139144

140145
// delete the postgres-exporter service
141146
if err := r.deleteExporterSidecarService(ctx, namespace); client.IgnoreNotFound(err) != nil {
@@ -279,7 +284,7 @@ func (r *PostgresReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
279284
return ctrl.Result{}, fmt.Errorf("failed to create or update zalando postgresql: %w", err)
280285
}
281286

282-
if err := r.LBManager.CreateSvcLBIfNone(ctx, instance); err != nil {
287+
if err := r.LBManager.ReconcileSvcLBs(ctx, instance); err != nil {
283288
r.recorder.Eventf(instance, "Warning", "Error", "failed to create Service: %v", err)
284289
return ctrl.Result{}, err
285290
}

‎controllers/postgres_controller_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ var _ = Describe("postgres controller", func() {
5757
Eventually(func() bool {
5858
return svcClusterClient.Get(newCtx(), types.NamespacedName{
5959
Namespace: instance.ToPeripheralResourceNamespace(),
60-
Name: instance.ToSvcLBName(),
60+
Name: instance.ToSharedSvcLBName(),
6161
}, &core.Service{}) == nil
6262
}, timeout, interval).Should(BeTrue())
6363
})
@@ -96,7 +96,7 @@ var _ = Describe("postgres controller", func() {
9696
Eventually(func() bool {
9797
return svcClusterClient.Get(newCtx(), types.NamespacedName{
9898
Namespace: instance.ToPeripheralResourceNamespace(),
99-
Name: instance.ToSvcLBName(),
99+
Name: instance.ToSharedSvcLBName(),
100100
}, &core.Service{}) == nil
101101
}, timeout, interval).ShouldNot(BeTrue())
102102
})

‎controllers/status_controller.go

Lines changed: 75 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ type StatusReconciler struct {
3434
Scheme *runtime.Scheme
3535
PartitionID string
3636
ControlPlaneNamespace string
37+
EnableForceSharedIP bool
3738
}
3839

3940
// Reconcile updates the status of the remote Postgres object based on the status of the local zalando object.
@@ -69,6 +70,7 @@ func (r *StatusReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
6970
return ctrl.Result{}, fmt.Errorf("could not find the owner")
7071
}
7172

73+
log.Info("updating status")
7274
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
7375
// get a fresh copy of the owner object
7476
if err := r.CtrlClient.Get(ctx, types.NamespacedName{Name: owner.Name, Namespace: owner.Namespace}, owner); err != nil {
@@ -91,18 +93,81 @@ func (r *StatusReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
9193
return ctrl.Result{}, retryErr
9294
}
9395

94-
lb := &corev1.Service{}
95-
if err := r.SvcClient.Get(ctx, *owner.ToSvcLBNamespacedName(), lb); err == nil {
96-
owner.Status.Socket.IP = lb.Spec.LoadBalancerIP
97-
owner.Status.Socket.Port = lb.Spec.Ports[0].Port
96+
log.Info("updating socket")
97+
if owner.Spec.DedicatedLoadBalancerIP == nil {
98+
// no dedicated load balancer configured, use the shared one
99+
shared := &corev1.Service{}
100+
if err := r.SvcClient.Get(ctx, *owner.ToSharedSvcLBNamespacedName(), shared); err == nil {
101+
// update IP and port
102+
owner.Status.Socket.IP = shared.Spec.LoadBalancerIP
103+
owner.Status.Socket.Port = shared.Spec.Ports[0].Port
104+
105+
} else {
106+
// Todo: Handle errors other than `NotFound`
107+
log.Info("unable to fetch the shared LoadBalancer to update postgres status socket")
108+
owner.Status.Socket = pg.Socket{}
109+
}
98110

99111
if err := r.CtrlClient.Status().Update(ctx, owner); err != nil {
100-
return ctrl.Result{}, fmt.Errorf("failed to update lbSocket of Postgres: %w", err)
112+
return ctrl.Result{}, fmt.Errorf("failed to update simple postgres status socket: %w", err)
101113
}
102-
log.Info("postgres status socket updated")
114+
log.Info("simple postgres status socket updated")
115+
103116
} else {
104-
// Todo: Handle errors other than `NotFound`
105-
log.Info("unable to fetch the corresponding Service of type LoadBalancer")
117+
// dedicated load balancer configured, so we fetch it
118+
dedicated := &corev1.Service{}
119+
derr := r.SvcClient.Get(ctx, *owner.ToDedicatedSvcLBNamespacedName(), dedicated)
120+
121+
if r.EnableForceSharedIP {
122+
123+
// we still have a shared load balancer, so we keep using this one as primary socket
124+
shared := &corev1.Service{}
125+
serr := r.SvcClient.Get(ctx, *owner.ToSharedSvcLBNamespacedName(), shared)
126+
if serr == nil {
127+
// the shared load balancer is usable, use it for status
128+
owner.Status.Socket.IP = shared.Spec.LoadBalancerIP
129+
owner.Status.Socket.Port = shared.Spec.Ports[0].Port
130+
log.Info("using shared loadbalancer as primary status socket")
131+
} else {
132+
// we couldn't use the shared load balancer, use empty socket instead
133+
log.Info("failed to use shared loadbalancer as primary status socket")
134+
owner.Status.Socket = pg.Socket{}
135+
}
136+
137+
// now we use the dedicated load balancer as additional socket
138+
if derr == nil && dedicated.Status.LoadBalancer.Ingress != nil && len(dedicated.Status.LoadBalancer.Ingress) == 1 {
139+
// the dedicated load balancer is usable, use it for status
140+
additionalSocket := pg.Socket{
141+
IP: dedicated.Spec.LoadBalancerIP,
142+
Port: dedicated.Spec.Ports[0].Port,
143+
}
144+
owner.Status.AdditionalSockets = []pg.Socket{additionalSocket}
145+
log.Info("using dedicated loadbalancer as additional status socket")
146+
} else {
147+
log.Info("failed to use dedicated loadbalancer as additional status socket")
148+
}
149+
150+
} else {
151+
152+
// no more shared load balancer, only use the dedicated one
153+
if derr == nil && dedicated.Status.LoadBalancer.Ingress != nil && len(dedicated.Status.LoadBalancer.Ingress) == 1 {
154+
// the dedicated load balancer is usable, use it for status
155+
owner.Status.Socket.IP = dedicated.Spec.LoadBalancerIP
156+
owner.Status.Socket.Port = dedicated.Spec.Ports[0].Port
157+
log.Info("using dedicated loadbalancer as primary status socket")
158+
} else {
159+
// we couldn't use the dedicated load balancer, use empty socket instead
160+
log.Info("failed to use dedicated loadbalancer as primary status socket")
161+
owner.Status.Socket = pg.Socket{}
162+
}
163+
}
164+
165+
// actually perform the status update
166+
if err := r.CtrlClient.Status().Update(ctx, owner); err != nil {
167+
return ctrl.Result{}, fmt.Errorf("failed to update advanced postgres status socket: %w", err)
168+
}
169+
log.Info("advanced postgres status socket updated")
170+
106171
}
107172

108173
// TODO also update the port/ip of databases mentioned in owner.Spec.PostgresConnectionInfo so that e.g. CWNP are always up to date
@@ -123,6 +188,8 @@ func (r *StatusReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
123188
return ctrl.Result{}, err
124189
}
125190

191+
log.Info("status reconciled")
192+
126193
return ctrl.Result{}, nil
127194
}
128195

0 commit comments

Comments
 (0)