Skip to content

Commit 2f81fd0

Browse files
eberlepeberlep
authored
Add annotations for tenant and projectID to service (#350)
* Add annotations for tenant and projectID to servicemointor * Use correct namespace * Switch annotation names * Add annotations to service as well * Only fetch configMap once * Renaming * Renaming * Add comment * Make the name of the sidecars config map configurable (and thereby properly separate different postgres installs in the same cluster)
1 parent 106a237 commit 2f81fd0

File tree

3 files changed

+244
-203
lines changed

3 files changed

+244
-203
lines changed

‎controllers/postgres_controller.go

Lines changed: 211 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"fmt"
1515
"net/http"
1616
"net/url"
17+
"strconv"
1718
"strings"
1819

1920
"github.com/go-logr/logr"
@@ -23,10 +24,12 @@ import (
2324
"k8s.io/utils/pointer"
2425

2526
firewall "github.com/metal-stack/firewall-controller/api/v1"
27+
coreosv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
2628
apierrors "k8s.io/apimachinery/pkg/api/errors"
2729
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2830
"k8s.io/apimachinery/pkg/runtime"
2931
"k8s.io/apimachinery/pkg/types"
32+
"k8s.io/apimachinery/pkg/util/intstr"
3033
ctrl "sigs.k8s.io/controller-runtime"
3134
"sigs.k8s.io/controller-runtime/pkg/client"
3235
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -36,6 +39,13 @@ import (
3639
"github.com/fi-ts/postgreslet/pkg/operatormanager"
3740
)
3841

42+
const (
43+
postgresExporterServiceName string = "postgres-exporter"
44+
postgresExporterServicePortName string = "metrics"
45+
postgresExporterServiceTenantAnnotationName string = pg.TenantLabelName
46+
postgresExporterServiceProjectIDAnnotationName string = pg.ProjectIDLabelName
47+
)
48+
3949
// requeue defines in how many seconds a requeue should happen
4050
var requeue = ctrl.Result{
4151
Requeue: true,
@@ -53,6 +63,8 @@ type PostgresReconciler struct {
5363
recorder record.EventRecorder
5464
PgParamBlockList map[string]bool
5565
StandbyClustersSourceRanges []string
66+
PostgresletNamespace string
67+
SidecarsConfigMapName string
5668
}
5769

5870
// Reconcile is the entry point for postgres reconciliation.
@@ -105,6 +117,11 @@ func (r *PostgresReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
105117
}
106118
log.Info("corresponding Service of type LoadBalancer deleted")
107119

120+
// delete the postgres-exporter service
121+
if err := r.deleteExporterSidecarService(ctx, namespace); client.IgnoreNotFound(err) != nil {
122+
return ctrl.Result{}, fmt.Errorf("error while deleting the postgres-exporter service: %w", err)
123+
}
124+
108125
if err := r.deleteZPostgresqlByLabels(ctx, matchingLabels, namespace); err != nil {
109126
r.recorder.Eventf(instance, "Warning", "Error", "failed to delete Zalando resource: %v", err)
110127
return ctrl.Result{}, err
@@ -183,7 +200,29 @@ func (r *PostgresReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
183200
return ctrl.Result{}, fmt.Errorf("unable to create or update egress ClusterwideNetworkPolicy: %w", err)
184201
}
185202

186-
if err := r.createOrUpdateZalandoPostgresql(ctx, instance, log); err != nil {
203+
// try to fetch the global sidecars configmap
204+
cns := types.NamespacedName{
205+
Namespace: r.PostgresletNamespace,
206+
Name: r.SidecarsConfigMapName,
207+
}
208+
globalSidecarsCM := &corev1.ConfigMap{}
209+
if err := r.SvcClient.Get(ctx, cns, globalSidecarsCM); err != nil {
210+
// configmap with configuration does not exists, nothing we can do here...
211+
return ctrl.Result{}, fmt.Errorf("could not fetch config for sidecars")
212+
}
213+
// Add services for our sidecars
214+
namespace := instance.ToPeripheralResourceNamespace()
215+
if err := r.createOrUpdateExporterSidecarServices(ctx, namespace, globalSidecarsCM, instance); err != nil {
216+
return ctrl.Result{}, fmt.Errorf("error while creating sidecars services %v: %w", namespace, err)
217+
}
218+
219+
// Add service monitor for our exporter sidecar
220+
err := r.createOrUpdateExporterSidecarServiceMonitor(ctx, namespace, instance)
221+
if err != nil {
222+
return ctrl.Result{}, fmt.Errorf("error while creating sidecars servicemonitor %v: %w", namespace, err)
223+
}
224+
225+
if err := r.createOrUpdateZalandoPostgresql(ctx, instance, log, globalSidecarsCM); err != nil {
187226
r.recorder.Eventf(instance, "Warning", "Error", "failed to create Zalando resource: %v", err)
188227
return ctrl.Result{}, fmt.Errorf("failed to create or update zalando postgresql: %w", err)
189228
}
@@ -224,21 +263,7 @@ func (r *PostgresReconciler) SetupWithManager(mgr ctrl.Manager) error {
224263
Complete(r)
225264
}
226265

227-
func (r *PostgresReconciler) createOrUpdateZalandoPostgresql(ctx context.Context, instance *pg.Postgres, log logr.Logger) error {
228-
// Get the sidecar config
229-
// try to fetch the global sidecars configmap
230-
cns := types.NamespacedName{
231-
// TODO don't use string literals here! name is dependent of the release name of the helm chart!
232-
Namespace: "postgreslet-system",
233-
Name: "postgreslet-postgres-sidecars",
234-
}
235-
c := &corev1.ConfigMap{}
236-
if err := r.SvcClient.Get(ctx, cns, c); err != nil {
237-
// configmap with configuration does not exists, nothing we can do here...
238-
log.Info("could not fetch config for sidecars")
239-
c = nil
240-
}
241-
266+
func (r *PostgresReconciler) createOrUpdateZalandoPostgresql(ctx context.Context, instance *pg.Postgres, log logr.Logger, sidecarsCM *corev1.ConfigMap) error {
242267
var restoreBackupConfig *pg.BackupConfig
243268
var restoreSouceInstance *pg.Postgres
244269
if instance.Spec.PostgresRestore != nil {
@@ -273,7 +298,7 @@ func (r *PostgresReconciler) createOrUpdateZalandoPostgresql(ctx context.Context
273298
return fmt.Errorf("failed to fetch zalando postgresql: %w", err)
274299
}
275300

276-
u, err := instance.ToUnstructuredZalandoPostgresql(nil, c, r.StorageClass, r.PgParamBlockList, restoreBackupConfig, restoreSouceInstance)
301+
u, err := instance.ToUnstructuredZalandoPostgresql(nil, sidecarsCM, r.StorageClass, r.PgParamBlockList, restoreBackupConfig, restoreSouceInstance)
277302
if err != nil {
278303
return fmt.Errorf("failed to convert to unstructured zalando postgresql: %w", err)
279304
}
@@ -289,7 +314,7 @@ func (r *PostgresReconciler) createOrUpdateZalandoPostgresql(ctx context.Context
289314
// Update zalando postgresql
290315
mergeFrom := client.MergeFrom(rawZ.DeepCopy())
291316

292-
u, err := instance.ToUnstructuredZalandoPostgresql(rawZ, c, r.StorageClass, r.PgParamBlockList, restoreBackupConfig, restoreSouceInstance)
317+
u, err := instance.ToUnstructuredZalandoPostgresql(rawZ, sidecarsCM, r.StorageClass, r.PgParamBlockList, restoreBackupConfig, restoreSouceInstance)
293318
if err != nil {
294319
return fmt.Errorf("failed to convert to unstructured zalando postgresql: %w", err)
295320
}
@@ -748,3 +773,171 @@ func (r *PostgresReconciler) getBackupConfig(ctx context.Context, ns, name strin
748773
}
749774
return &backupConfig, nil
750775
}
776+
777+
// createOrUpdateExporterSidecarServices ensures the neccessary services to acces the sidecars exist
778+
func (r *PostgresReconciler) createOrUpdateExporterSidecarServices(ctx context.Context, namespace string, c *corev1.ConfigMap, in *pg.Postgres) error {
779+
log := r.Log.WithValues("namespace", namespace)
780+
781+
exporterServicePort, error := strconv.ParseInt(c.Data["postgres-exporter-service-port"], 10, 32)
782+
if error != nil {
783+
// todo log error
784+
exporterServicePort = 9187
785+
}
786+
787+
exporterServiceTargetPort, error := strconv.ParseInt(c.Data["postgres-exporter-service-target-port"], 10, 32)
788+
if error != nil {
789+
// todo log error
790+
exporterServiceTargetPort = exporterServicePort
791+
}
792+
793+
pes := &corev1.Service{}
794+
795+
if err := r.SetName(pes, postgresExporterServiceName); err != nil {
796+
return fmt.Errorf("error while setting the name of the postgres-exporter service to %v: %w", namespace, err)
797+
}
798+
if err := r.SetNamespace(pes, namespace); err != nil {
799+
return fmt.Errorf("error while setting the namespace of the postgres-exporter service to %v: %w", namespace, err)
800+
}
801+
labels := map[string]string{
802+
// "application": "spilo", // TODO check if we still need that label, IsOperatorDeletable won't work anymore if we set it.
803+
"app": "postgres-exporter",
804+
}
805+
if err := r.SetLabels(pes, labels); err != nil {
806+
return fmt.Errorf("error while setting the labels of the postgres-exporter service to %v: %w", labels, err)
807+
}
808+
annotations := map[string]string{
809+
postgresExporterServiceTenantAnnotationName: in.Spec.Tenant,
810+
postgresExporterServiceProjectIDAnnotationName: in.Spec.ProjectID,
811+
}
812+
if err := r.SetAnnotations(pes, annotations); err != nil {
813+
return fmt.Errorf("error while setting the annotations of the postgres-exporter service to %v: %w", annotations, err)
814+
}
815+
816+
pes.Spec.Ports = []corev1.ServicePort{
817+
{
818+
Name: postgresExporterServicePortName,
819+
Port: int32(exporterServicePort),
820+
Protocol: corev1.ProtocolTCP,
821+
TargetPort: intstr.FromInt(int(exporterServiceTargetPort)),
822+
},
823+
}
824+
selector := map[string]string{
825+
"application": "spilo",
826+
}
827+
pes.Spec.Selector = selector
828+
pes.Spec.Type = corev1.ServiceTypeClusterIP
829+
830+
// try to fetch any existing postgres-exporter service
831+
ns := types.NamespacedName{
832+
Namespace: namespace,
833+
Name: postgresExporterServiceName,
834+
}
835+
old := &corev1.Service{}
836+
if err := r.SvcClient.Get(ctx, ns, old); err == nil {
837+
// service exists, overwriting it (but using the same clusterip)
838+
pes.Spec.ClusterIP = old.Spec.ClusterIP
839+
pes.ObjectMeta.ResourceVersion = old.GetObjectMeta().GetResourceVersion()
840+
if err := r.SvcClient.Update(ctx, pes); err != nil {
841+
return fmt.Errorf("error while updating the postgres-exporter service: %w", err)
842+
}
843+
log.Info("postgres-exporter service updated")
844+
return nil
845+
}
846+
// todo: handle errors other than `NotFound`
847+
848+
// local servicemonitor does not exist, creating it
849+
if err := r.SvcClient.Create(ctx, pes); err != nil {
850+
return fmt.Errorf("error while creating the postgres-exporter service: %w", err)
851+
}
852+
log.Info("postgres-exporter service created")
853+
854+
return nil
855+
}
856+
857+
// createOrUpdateExporterSidecarServiceMonitor ensures the servicemonitors for the sidecars exist
858+
func (r *PostgresReconciler) createOrUpdateExporterSidecarServiceMonitor(ctx context.Context, namespace string, in *pg.Postgres) error {
859+
log := r.Log.WithValues("namespace", namespace)
860+
861+
pesm := &coreosv1.ServiceMonitor{}
862+
863+
// TODO what's the correct name?
864+
if err := r.SetName(pesm, postgresExporterServiceName); err != nil {
865+
return fmt.Errorf("error while setting the name of the postgres-exporter servicemonitor to %v: %w", namespace, err)
866+
}
867+
if err := r.SetNamespace(pesm, namespace); err != nil {
868+
return fmt.Errorf("error while setting the namespace of the postgres-exporter servicemonitor to %v: %w", namespace, err)
869+
}
870+
labels := map[string]string{
871+
"app": "postgres-exporter",
872+
"release": "prometheus",
873+
}
874+
if err := r.SetLabels(pesm, labels); err != nil {
875+
return fmt.Errorf("error while setting the namespace of the postgres-exporter servicemonitor to %v: %w", namespace, err)
876+
}
877+
annotations := map[string]string{
878+
postgresExporterServiceTenantAnnotationName: in.Spec.Tenant,
879+
postgresExporterServiceProjectIDAnnotationName: in.Spec.ProjectID,
880+
}
881+
if err := r.SetAnnotations(pesm, annotations); err != nil {
882+
return fmt.Errorf("error while setting the annotations of the postgres-exporter service monitor to %v: %w", annotations, err)
883+
}
884+
885+
pesm.Spec.Endpoints = []coreosv1.Endpoint{
886+
{
887+
Port: postgresExporterServicePortName,
888+
},
889+
}
890+
pesm.Spec.NamespaceSelector = coreosv1.NamespaceSelector{
891+
MatchNames: []string{namespace},
892+
}
893+
matchLabels := map[string]string{
894+
// TODO use extraced string
895+
"app": "postgres-exporter",
896+
}
897+
pesm.Spec.Selector = metav1.LabelSelector{
898+
MatchLabels: matchLabels,
899+
}
900+
901+
// try to fetch any existing postgres-exporter service
902+
ns := types.NamespacedName{
903+
Namespace: namespace,
904+
Name: postgresExporterServiceName,
905+
}
906+
old := &coreosv1.ServiceMonitor{}
907+
if err := r.SvcClient.Get(ctx, ns, old); err == nil {
908+
// Copy the resource version
909+
pesm.ObjectMeta.ResourceVersion = old.ObjectMeta.ResourceVersion
910+
if err := r.SvcClient.Update(ctx, pesm); err != nil {
911+
return fmt.Errorf("error while updating the postgres-exporter servicemonitor: %w", err)
912+
}
913+
log.Info("postgres-exporter servicemonitor updated")
914+
return nil
915+
}
916+
// todo: handle errors other than `NotFound`
917+
918+
// local servicemonitor does not exist, creating it
919+
if err := r.SvcClient.Create(ctx, pesm); err != nil {
920+
return fmt.Errorf("error while creating the postgres-exporter servicemonitor: %w", err)
921+
}
922+
log.Info("postgres-exporter servicemonitor created")
923+
924+
return nil
925+
}
926+
927+
func (r *PostgresReconciler) deleteExporterSidecarService(ctx context.Context, namespace string) error {
928+
log := r.Log.WithValues("namespace", namespace)
929+
930+
s := &corev1.Service{}
931+
if err := r.SetName(s, postgresExporterServiceName); err != nil {
932+
return fmt.Errorf("error while setting the name of the postgres-exporter service to delete to %v: %w", postgresExporterServiceName, err)
933+
}
934+
if err := r.SetNamespace(s, namespace); err != nil {
935+
return fmt.Errorf("error while setting the namespace of the postgres-exporter service to delete to %v: %w", namespace, err)
936+
}
937+
if err := r.SvcClient.Delete(ctx, s); err != nil {
938+
return fmt.Errorf("error while deleting the postgres-exporter service: %w", err)
939+
}
940+
log.Info("postgres-exporter service deleted")
941+
942+
return nil
943+
}

‎main.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ const (
5353
pgParamBlockListFlg = "postgres-param-blocklist"
5454
majorVersionUpgradeModeFlg = "major-version-upgrade-mode"
5555
standbyClustersSourceRangesFlg = "standby-clusters-source-ranges"
56+
postgresletNamespaceFlg = "postgreslet-namespace"
57+
sidecarsCMNameFlg = "sidecars-configmap-name"
5658
)
5759

5860
var (
@@ -71,7 +73,7 @@ func init() {
7173
}
7274

7375
func main() {
74-
var metricsAddrCtrlMgr, metricsAddrSvcMgr, partitionID, tenant, ctrlClusterKubeconfig, pspName, lbIP, storageClass, postgresImage, etcdHost, operatorImage, majorVersionUpgradeMode string
76+
var metricsAddrCtrlMgr, metricsAddrSvcMgr, partitionID, tenant, ctrlClusterKubeconfig, pspName, lbIP, storageClass, postgresImage, etcdHost, operatorImage, majorVersionUpgradeMode, postgresletNamespace, sidecarsCMName string
7577
var enableLeaderElection, enableCRDValidation bool
7678
var portRangeStart, portRangeSize int
7779
var pgParamBlockList map[string]bool
@@ -143,6 +145,12 @@ func main() {
143145
viper.SetDefault(standbyClustersSourceRangesFlg, "255.255.255.255/32")
144146
standbyClusterSourceRanges = viper.GetStringSlice(standbyClustersSourceRangesFlg)
145147

148+
viper.SetDefault(postgresletNamespaceFlg, "postgreslet-system")
149+
postgresletNamespace = viper.GetString(postgresletNamespaceFlg)
150+
151+
viper.SetDefault(sidecarsCMNameFlg, "postgreslet-postgres-sidecars")
152+
sidecarsCMName = viper.GetString(sidecarsCMNameFlg)
153+
146154
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
147155

148156
ctrl.Log.Info("flag",
@@ -164,6 +172,8 @@ func main() {
164172
pgParamBlockListFlg, pgParamBlockList,
165173
majorVersionUpgradeModeFlg, majorVersionUpgradeMode,
166174
standbyClustersSourceRangesFlg, standbyClusterSourceRanges,
175+
postgresletNamespaceFlg, postgresletNamespace,
176+
sidecarsCMNameFlg, sidecarsCMName,
167177
)
168178

169179
svcClusterConf := ctrl.GetConfigOrDie()
@@ -203,6 +213,8 @@ func main() {
203213
EtcdHost: etcdHost,
204214
CRDValidation: enableCRDValidation,
205215
MajorVersionUpgradeMode: majorVersionUpgradeMode,
216+
PostgresletNamespace: postgresletNamespace,
217+
SidecarsConfigMapName: sidecarsCMName,
206218
}
207219
opMgr, err := operatormanager.New(svcClusterConf, "external/svc-postgres-operator.yaml", scheme, ctrl.Log.WithName("OperatorManager"), opMgrOpts)
208220
if err != nil {
@@ -227,6 +239,8 @@ func main() {
227239
LBManager: lbmanager.New(svcClusterMgr.GetClient(), lbMgrOpts),
228240
PgParamBlockList: pgParamBlockList,
229241
StandbyClustersSourceRanges: standbyClusterSourceRanges,
242+
PostgresletNamespace: postgresletNamespace,
243+
SidecarsConfigMapName: sidecarsCMName,
230244
}).SetupWithManager(ctrlPlaneClusterMgr); err != nil {
231245
setupLog.Error(err, "unable to create controller", "controller", "Postgres")
232246
os.Exit(1)

0 commit comments

Comments
 (0)