SECURESIGN-3167 CTLog recovery

internal/controller/ctlog/actions/server_config.go

@@ -6,6 +6,7 @@
 	"fmt"
 	"maps"
 	"slices"
+	"strings"
 
 	rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1"
 	"github.com/securesign/operator/internal/action"
@@ -17,6 +18,7 @@
 	"github.com/securesign/operator/internal/utils/kubernetes/ensure"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	labels2 "k8s.io/apimachinery/pkg/labels"
@@ -51,8 +53,11 @@
 		return true
 	case instance.Spec.ServerConfigRef != nil:
 		return !equality.Semantic.DeepEqual(instance.Spec.ServerConfigRef, instance.Status.ServerConfigRef)
+	case c.ObservedGeneration != instance.Generation:
+		return true
 	default:
-		return instance.Generation != c.ObservedGeneration
+		// Always run Handle() to validate the secret: exists and is valid
+		return true
 	}
 }
 
@@ -62,6 +67,29 @@
 	)
 
 	if instance.Spec.ServerConfigRef != nil {
+		// Validate that the custom secret is accessible
+		secret, err := kubernetes.GetSecret(i.Client, instance.Namespace, instance.Spec.ServerConfigRef.Name)
+		if err != nil {
+			return i.Error(ctx, fmt.Errorf("error accessing custom server config secret: %w", err), instance,
+				metav1.Condition{
+					Type:               ConfigCondition,
+					Status:             metav1.ConditionFalse,
+					Reason:             constants.Failure,
+					Message:            fmt.Sprintf("Error accessing custom server config secret: %s", instance.Spec.ServerConfigRef.Name),
+					ObservedGeneration: instance.Generation,
+				})
+		}
+		if secret.Data == nil || secret.Data[ctlogUtils.ConfigKey] == nil {
+			return i.Error(ctx, fmt.Errorf("custom server config secret is invalid"), instance,
+				metav1.Condition{
+					Type:               ConfigCondition,
+					Status:             metav1.ConditionFalse,
+					Reason:             constants.Failure,
+					Message:            fmt.Sprintf("Custom server config secret is missing '%s' key: %s", ctlogUtils.ConfigKey, instance.Spec.ServerConfigRef.Name),
+					ObservedGeneration: instance.Generation,
+				})
+		}
+
 		instance.Status.ServerConfigRef = instance.Spec.ServerConfigRef
 		i.Recorder.Event(instance, corev1.EventTypeNormal, "CTLogConfigUpdated", "CTLog config updated")
 		meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
@@ -74,6 +102,7 @@
 		return i.StatusUpdate(ctx, instance)
 	}
 
+	// Validate prerequisites and normalize Trillian address before validation
 	switch {
 	case instance.Status.TreeID == nil:
 		return i.Error(ctx, fmt.Errorf("%s: %v", i.Name(), ctlogUtils.ErrTreeNotSpecified), instance)
@@ -87,6 +116,56 @@
 
 	trillianUrl := fmt.Sprintf("%s:%d", instance.Spec.Trillian.Address, *instance.Spec.Trillian.Port)
 
+	// Validate existing secret before attempting recreation
+	if instance.Status.ServerConfigRef != nil && instance.Status.ServerConfigRef.Name != "" {
+		secret, err := kubernetes.GetSecret(i.Client, instance.Namespace, instance.Status.ServerConfigRef.Name)
+
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				i.Logger.Info("Server config secret is missing, will recreate",
+					"secret", instance.Status.ServerConfigRef.Name)
+				i.Recorder.Event(instance, corev1.EventTypeWarning, "CTLogConfigMissing",
+					"Config secret is missing, will recreate")
+			} else {
+				i.Logger.Error(err, "Error accessing server config secret, will attempt to recreate",
+					"secret", instance.Status.ServerConfigRef.Name)
+				i.Recorder.Event(instance, corev1.EventTypeWarning, "CTLogConfigError",
+					"Error accessing config secret, will recreate")
+			}
+		} else {
+			// Secret exists and is accessible - validate it
+			if !ctlogUtils.IsSecretDataValid(secret.Data, trillianUrl) {
+				// Secret has wrong Trillian configuration, will recreate
+				i.Logger.Info("Server config secret is invalid, will recreate",
+					"secret", secret.Name,
+					"reason", "Trillian configuration mismatch")
+				i.Recorder.Event(instance, corev1.EventTypeWarning, "CTLogConfigInvalid",
+					"Config secret has invalid Trillian configuration, will recreate")
+			} else {
+				// Check if root certificates match (for hot updates)
+				// Count fulcio-* keys in the secret
+				actualRootCertCount := 0
+				for key := range secret.Data {
+					if strings.HasPrefix(key, "fulcio-") {
+						actualRootCertCount++
+					}
+				}
+
+				// Compare with expected count from status
+				expectedRootCertCount := len(instance.Status.RootCertificates)
+				if actualRootCertCount == expectedRootCertCount && expectedRootCertCount > 0 {
+					// Everything matches - no need to recreate
+					return i.Continue()
+				}
+				// Root certificates changed - need to recreate for hot update
+				i.Logger.Info("Server config secret needs update for root certificate change",
+					"secret", secret.Name,
+					"expected_certs", expectedRootCertCount,
+					"actual_certs", actualRootCertCount)
+			}
+		}
+	}
+
 	configLabels := labels.ForResource(ComponentName, DeploymentName, instance.Name, serverConfigResourceName)
 
 	rootCerts, err := i.handleRootCertificates(instance)
@@ -151,7 +230,7 @@
 
 	instance.Status.ServerConfigRef = &rhtasv1alpha1.LocalObjectReference{Name: newConfig.Name}
 
-	i.Recorder.Eventf(instance, corev1.EventTypeNormal, "CTLogConfigCreated", "Secret with ctlog configuration created: %s", newConfig.Name)
+	i.Recorder.Event(instance, corev1.EventTypeNormal, "CTLogConfigCreated", "Config secret created successfully")
 	meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{
 		Type:               ConfigCondition,
 		Status:             metav1.ConditionTrue,
@@ -186,11 +265,11 @@
 		err = i.Client.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: partialConfig.Name, Namespace: partialConfig.Namespace}})
 		if err != nil {
 			i.Logger.Error(err, "unable to delete secret", "namespace", instance.Namespace, "name", partialConfig.Name)
-			i.Recorder.Eventf(instance, corev1.EventTypeWarning, "CTLogConfigDeleted", "Unable to delete secret: %s", partialConfig.Name)
+			i.Recorder.Event(instance, corev1.EventTypeWarning, "CTLogConfigCleanupFailed", "Unable to delete old config secret")
 			continue
 		}
 		i.Logger.Info("Remove invalid Secret with ctlog configuration", "Name", partialConfig.Name)
-		i.Recorder.Eventf(instance, corev1.EventTypeNormal, "CTLogConfigDeleted", "Secret with ctlog configuration deleted: %s", partialConfig.Name)
+		i.Recorder.Event(instance, corev1.EventTypeNormal, "CTLogConfigCleanedUp", "Old config secret deleted successfully")
 	}
 }
 

internal/controller/ctlog/actions/server_config_test.go

@@ -57,9 +57,11 @@ func TestServerConfig_CanHandle(t *testing.T) {
 		{
 			name:                  "ConditionTrue: spec.serverConfigRef is nil and status.serverConfigRef is not nil",
 			status:                metav1.ConditionTrue,
-			canHandle:             false,
+			canHandle:             true,
 			serverConfigRef:       nil,
 			statusServerConfigRef: &rhtasv1alpha1.LocalObjectReference{Name: "config"},
+			observedGeneration:    1,
+			generation:            1,
 		},
 		{
 			name:                  "ConditionTrue: spec.serverConfigRef is nil and status.serverConfigRef is nil",
@@ -85,7 +87,7 @@ func TestServerConfig_CanHandle(t *testing.T) {
 		{
 			name:                  "ConditionTrue: observedGeneration == generation",
 			status:                metav1.ConditionTrue,
-			canHandle:             false,
+			canHandle:             true,
 			statusServerConfigRef: &rhtasv1alpha1.LocalObjectReference{Name: "config"},
 			observedGeneration:    1,
 			generation:            1,
@@ -182,6 +184,17 @@ func TestServerConfig_Handle(t *testing.T) {
 				status: rhtasv1alpha1.CTlogStatus{
 					ServerConfigRef: nil,
 				},
+				objects: []client.Object{
+					&v1.Secret{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "config",
+							Namespace: "default",
+						},
+						Data: map[string][]byte{
+							ctlogUtils.ConfigKey: []byte("test-config"),
+						},
+					},
+				},
 			},
 			want: want{
 				result: testAction.StatusUpdate(),
@@ -238,6 +251,17 @@ func TestServerConfig_Handle(t *testing.T) {
 				status: rhtasv1alpha1.CTlogStatus{
 					ServerConfigRef: &rhtasv1alpha1.LocalObjectReference{Name: "old_config"},
 				},
+				objects: []client.Object{
+					&v1.Secret{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "new_config",
+							Namespace: "default",
+						},
+						Data: map[string][]byte{
+							ctlogUtils.ConfigKey: []byte("new-test-config"),
+						},
+					},
+				},
 			},
 			want: want{
 				result: testAction.StatusUpdate(),

internal/controller/ctlog/utils/ctlog_config.go

@@ -200,3 +200,50 @@ func CreateCtlogConfig(trillianUrl string, treeID int64, rootCerts []RootCertifi
 	}
 	return data, nil
 }
+
+// IsSecretDataValid validates that a CTLog config secret contains valid configuration
+// with the correct Trillian backend address.
+//
+// This function parses the protobuf text configuration and validates:
+// 1. The configuration can be unmarshalled into the expected structure
+// 2. At least one backend exists
+// 3. The backend's BackendSpec matches the expected Trillian address
+//
+// Parameters:
+//   - secretData: The Data field from a Kubernetes Secret containing CTLog configuration
+//   - expectedTrillianAddr: The Trillian address to validate against (e.g., "trillian-logserver.namespace.svc:8091")
+//
+// Returns true if the secret contains valid configuration with the correct Trillian address,
+// false otherwise. Used by the operator for self-healing to detect missing or invalid
+// configuration secrets that need to be recreated.
+func IsSecretDataValid(secretData map[string][]byte, expectedTrillianAddr string) bool {
+	if secretData == nil {
+		return false
+	}
+
+	configData, ok := secretData[ConfigKey]
+	if !ok || len(configData) == 0 {
+		return false
+	}
+
+	// Parse the protobuf text format configuration
+	var multiConfig configpb.LogMultiConfig
+	if err := prototext.Unmarshal(configData, &multiConfig); err != nil {
+		// Failed to parse - invalid configuration
+		return false
+	}
+
+	// Validate that at least one backend exists
+	if multiConfig.Backends == nil || multiConfig.Backends.Backend == nil || len(multiConfig.Backends.Backend) == 0 {
+		return false
+	}
+
+	// Check if any backend matches the expected Trillian address
+	for _, backend := range multiConfig.Backends.Backend {
+		if backend.BackendSpec == expectedTrillianAddr {
+			return true
+		}
+	}
+
+	return false
+}