diff --git a/controllers/postgres_controller.go b/controllers/postgres_controller.go index 15c92999..d2d32602 100644 --- a/controllers/postgres_controller.go +++ b/controllers/postgres_controller.go @@ -56,12 +56,19 @@ import ( const ( postgresExporterServiceName string = "postgres-exporter" postgresExporterServicePortName string = "metrics" + postgresExporterServicePortKeyName string = "postgres-exporter-service-port" + postgresExporterServiceTargetPortKeyName string = "postgres-exporter-service-target-port" + walGExporterServicePortName string = "backup-metrics" + walGExporterServicePortKeyName string = "wal-g-exporter-service-port" + walGExporterServiceTargetPortKeyName string = "wal-g-exporter-service-target-port" postgresExporterServiceTenantAnnotationName string = pg.TenantLabelName postgresExporterServiceProjectIDAnnotationName string = pg.ProjectIDLabelName storageEncryptionKeyName string = "storage-encryption-key" storageEncryptionKeyFinalizerName string = "postgres.database.fits.cloud/secret-finalizer" walGEncryptionSecretNamePostfix string = "-walg-encryption" walGEncryptionSecretKeyName string = "key" + podMonitorName string = "patroni" + podMonitorPort string = "8008" initDBName string = "postgres-initdb" initDBSQLDummy string = `SELECT 'NOOP';` debugLogLevel int = 1 @@ -102,6 +109,7 @@ type PostgresReconciler struct { EnableCustomTLSCert bool TLSClusterIssuer string TLSSubDomain string + EnableWalGExporter bool } type PatroniStandbyCluster struct { @@ -300,6 +308,12 @@ func (r *PostgresReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, fmt.Errorf("error while creating sidecars servicemonitor %v: %w", namespace, err) } + // Add pod monitor + if err := r.createOrUpdatePatroniPodMonitor(ctx, namespace, instance); err != nil { + r.recorder.Eventf(instance, "Warning", "Error", "failed to create podmonitor: %v", err) + return ctrl.Result{}, fmt.Errorf("error while creating podmonitor %v: %w", namespace, err) + } + // Make sure the storage secret exist, if necessary if err := r.ensureStorageEncryptionSecret(log, ctx, instance); err != nil { r.recorder.Eventf(instance, "Warning", "Error", "failed to create storage secret: %v", err) @@ -1473,16 +1487,28 @@ func (r *PostgresReconciler) createOrUpdateNetPol(ctx context.Context, instance // createOrUpdateExporterSidecarServices ensures the necessary services to access the sidecars exist func (r *PostgresReconciler) createOrUpdateExporterSidecarServices(log logr.Logger, ctx context.Context, namespace string, c *corev1.ConfigMap, in *pg.Postgres) error { - exporterServicePort, error := strconv.ParseInt(c.Data["postgres-exporter-service-port"], 10, 32) + pesPort, error := strconv.ParseInt(c.Data[postgresExporterServicePortKeyName], 10, 32) + if error != nil { + log.Error(error, "postgres-exporter-service-port could not be parsed to int32, falling back to default value") + pesPort = 9187 + } + + pesTargetPort, error := strconv.ParseInt(c.Data[postgresExporterServiceTargetPortKeyName], 10, 32) + if error != nil { + log.Error(error, "postgres-exporter-service-target-port could not be parsed to int32, falling back to default value") + pesTargetPort = pesPort + } + + wgesPort, error := strconv.ParseInt(c.Data[walGExporterServicePortKeyName], 10, 32) if error != nil { - // todo log error - exporterServicePort = 9187 + log.Error(error, "wal-g-exporter-service-port could not be parsed to int32, falling back to default value") + pesPort = 9351 } - exporterServiceTargetPort, error := strconv.ParseInt(c.Data["postgres-exporter-service-target-port"], 10, 32) + wgesTargetPort, error := strconv.ParseInt(c.Data[walGExporterServiceTargetPortKeyName], 10, 32) if error != nil { - // todo log error - exporterServiceTargetPort = exporterServicePort + log.Error(error, "wal-g-exporter-service-target-port could not be parsed to int32, falling back to default value") + pesTargetPort = pesPort } labels := map[string]string{ @@ -1506,11 +1532,20 @@ func (r *PostgresReconciler) createOrUpdateExporterSidecarServices(log logr.Logg pes.Spec.Ports = []corev1.ServicePort{ { Name: postgresExporterServicePortName, - Port: int32(exporterServicePort), //nolint + Port: int32(pesPort), //nolint Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt(int(exporterServiceTargetPort)), + TargetPort: intstr.FromInt(int(pesTargetPort)), }, } + if r.EnableWalGExporter { + wgesp := corev1.ServicePort{ + Name: walGExporterServicePortName, + Port: int32(wgesPort), //nolint + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(int(wgesTargetPort)), + } + pes.Spec.Ports = append(pes.Spec.Ports, wgesp) + } selector := map[string]string{ pg.ApplicationLabelName: pg.ApplicationLabelValue, } @@ -1586,6 +1621,12 @@ func (r *PostgresReconciler) createOrUpdateExporterSidecarServiceMonitor(log log Port: postgresExporterServicePortName, }, } + if r.EnableWalGExporter { + wgesme := coreosv1.Endpoint{ + Port: walGExporterServicePortName, + } + pesm.Spec.Endpoints = append(pesm.Spec.Endpoints, wgesme) + } pesm.Spec.NamespaceSelector = coreosv1.NamespaceSelector{ MatchNames: []string{namespace}, } @@ -1596,6 +1637,10 @@ func (r *PostgresReconciler) createOrUpdateExporterSidecarServiceMonitor(log log pesm.Spec.Selector = metav1.LabelSelector{ MatchLabels: matchLabels, } + pesm.Spec.TargetLabels = []string{ + "postgres_partition_id=" + in.Spec.PartitionID, + "is_primary=" + strconv.FormatBool(in.IsReplicationPrimaryOrStandalone()), + } // try to fetch any existing postgres-exporter service ns := types.NamespacedName{ @@ -1623,6 +1668,69 @@ func (r *PostgresReconciler) createOrUpdateExporterSidecarServiceMonitor(log log return nil } +// createOrUpdatePatroniPodMonitor ensures the servicemonitors for the sidecars exist +func (r *PostgresReconciler) createOrUpdatePatroniPodMonitor(ctx context.Context, namespace string, in *pg.Postgres) error { + log := r.Log.WithValues("namespace", namespace) + + labels := map[string]string{ + "app": "postgres-exporter", + "release": "prometheus", + } + + annotations := map[string]string{ + postgresExporterServiceTenantAnnotationName: in.Spec.Tenant, + postgresExporterServiceProjectIDAnnotationName: in.Spec.ProjectID, + } + + pm := &coreosv1.PodMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: podMonitorName, + Namespace: namespace, + Labels: labels, + Annotations: annotations, + }, + } + + pm.Spec.PodMetricsEndpoints = []coreosv1.PodMetricsEndpoint{ + { + Port: podMonitorPort, + }, + } + pm.Spec.NamespaceSelector = coreosv1.NamespaceSelector{ + MatchNames: []string{namespace}, + } + matchLabels := map[string]string{ + "application": "spilo", + } + pm.Spec.Selector = metav1.LabelSelector{ + MatchLabels: matchLabels, + } + + // try to fetch any existing podmonitor + ns := types.NamespacedName{ + Namespace: namespace, + Name: podMonitorName, + } + old := &coreosv1.PodMonitor{} + if err := r.SvcClient.Get(ctx, ns, old); err == nil { + // Copy the resource version + pm.ObjectMeta.ResourceVersion = old.ObjectMeta.ResourceVersion + if err := r.SvcClient.Update(ctx, pm); err != nil { + return fmt.Errorf("error while updating the podmonitor: %w", err) + } + log.Info("pod monitor updated") + return nil + } + + // local podmonitor does not exist, creating it + if err := r.SvcClient.Create(ctx, pm); err != nil { + return fmt.Errorf("error while creating the podmonitor: %w", err) + } + log.Info("podmonitor created") + + return nil +} + func (r *PostgresReconciler) deleteExporterSidecarService(log logr.Logger, ctx context.Context, namespace string) error { s := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/main.go b/main.go index e8fb606e..9e63a831 100644 --- a/main.go +++ b/main.go @@ -89,6 +89,7 @@ const ( tlsSubDomainFlg = "tls-sub-domain" enablePatroniFailsafeModeFlg = "enable-patroni-failsafe-mode" enableFsGroupChangePolicyWebhookFlg = "enable-fsgroup-change-policy-webhook" + enableWalGExporterFlg = "enable-wal-g-exporter" ) var ( @@ -150,6 +151,7 @@ func main() { enableSuperUserForDBO bool enablePatroniFailsafeMode bool enableFsGroupChangePolicyWebhook bool + enableWalGExporter bool portRangeStart int32 portRangeSize int32 @@ -315,6 +317,9 @@ func main() { viper.SetDefault(enableFsGroupChangePolicyWebhookFlg, true) enableFsGroupChangePolicyWebhook = viper.GetBool(enableFsGroupChangePolicyWebhookFlg) + viper.SetDefault(enableWalGExporterFlg, true) + enableWalGExporter = viper.GetBool(enableWalGExporterFlg) + ctrl.Log.Info("flag", metricsAddrSvcMgrFlg, metricsAddrSvcMgr, metricsAddrCtrlMgrFlg, metricsAddrCtrlMgr, @@ -360,6 +365,7 @@ func main() { tlsSubDomainFlg, tlsSubDomain, enablePatroniFailsafeModeFlg, enablePatroniFailsafeMode, enableFsGroupChangePolicyWebhookFlg, enableFsGroupChangePolicyWebhook, + enableWalGExporterFlg, enableWalGExporter, ) svcClusterConf := ctrl.GetConfigOrDie() @@ -478,6 +484,7 @@ func main() { EnableCustomTLSCert: enableCustomTLSCert, TLSClusterIssuer: tlsClusterIssuer, TLSSubDomain: tlsSubDomain, + EnableWalGExporter: enableWalGExporter, }).SetupWithManager(ctrlPlaneClusterMgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Postgres") os.Exit(1)