diff --git a/README.md b/README.md index e819883..0b21e47 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,8 @@ More examples are available just below the spec that follows. | `image` | `string` | **Required**. Container image name. | | `imagePullPolicy` | `string` | Image pull policy. One of `Always`, `Never`, `IfNotPresent`. Defaults to `Always` if `:latest` tag is specified, or `IfNotPresent` otherwise. | | `imagePullSecrets` | `array` | Optional list of references to secrets in the same namespace to use for pulling the image. | +| `annotations` | `object` | Annotations to set on the Restate pod template. See note on merge ordering below. | +| `labels` | `object` | Labels to set on the Restate pod template. See note on merge ordering below. | | `resources` | `object` | Compute Resources for the Restate container. e.g., `requests` and `limits` for `cpu` and `memory`. | | `env` | `array` | List of environment variables to set in the container. | | `affinity` | `object` | Standard Kubernetes affinity rules. | @@ -118,6 +120,13 @@ More examples are available just below the spec that follows. | `dnsPolicy` | `string` | Pod DNS policy. | | `dnsConfig` | `object` | Pod DNS configuration. | +**Pod annotation and label merge ordering**: User-specified `annotations` and `labels` are +merged with values the operator sets internally (e.g. for workload identity hashes, trusted +CA cert hashes). If the same key appears in both, the operator's internal value wins. This +means operator-managed features like GCP Workload Identity annotations cannot be +accidentally overridden. If you need to set the same annotation that a built-in feature +uses, disable the built-in feature first — otherwise your value will be silently replaced. + --- #### `spec.storage` diff --git a/crd/restateclusters.yaml b/crd/restateclusters.yaml index e1ad001..65c3c40 100644 --- a/crd/restateclusters.yaml +++ b/crd/restateclusters.yaml @@ -561,6 +561,14 @@ spec: type: array type: object type: object + annotations: + additionalProperties: + type: string + description: |- + Annotations to set on the Restate pod template. These are merged with any annotations + the operator sets internally (e.g. for workload identity, trusted CA certs). + nullable: true + type: object args: description: Arguments to the entrypoint. The container image's CMD is used if this is not provided. items: @@ -733,6 +741,14 @@ spec: type: object nullable: true type: array + labels: + additionalProperties: + type: string + description: |- + Labels to set on the Restate pod template. These are merged with the standard labels + the operator sets (app.kubernetes.io/name, etc.). + nullable: true + type: object nodeSelector: additionalProperties: type: string diff --git a/release-notes/unreleased/45-pod-annotations-and-labels.md b/release-notes/unreleased/45-pod-annotations-and-labels.md new file mode 100644 index 0000000..e788c66 --- /dev/null +++ b/release-notes/unreleased/45-pod-annotations-and-labels.md @@ -0,0 +1,37 @@ +# Release Notes for Issue #45: Add support for custom pod annotations and labels + +## New Feature + +### What Changed +Added `spec.compute.annotations` and `spec.compute.labels` fields to the +RestateCluster CRD, allowing users to set custom annotations and labels on the +Restate StatefulSet pod template. + +User-specified annotations and labels are merged with any that the operator sets +internally (e.g. for workload identity, trusted CA certs). In case of conflict, +operator-managed values take precedence. + +### Why This Matters +Enables integrations that require pod-level metadata, such as GKE ComputeClass +scheduling (`cloud.google.com/compute-class`), Vault agent injection, Datadog, +Prometheus scraping, and custom scheduling constraints. + +### Impact on Users +- Existing deployments: No impact, both fields are optional +- New deployments: Can now set annotations and labels for integrations that + require them on the pod template + +### Migration Guidance +No migration required. To use the new fields: + +```yaml +spec: + compute: + annotations: + cloud.google.com/compute-class: "restate-workload" + labels: + team: "platform" +``` + +### Related Issues +- Issue #45: Add support for custom annotations on StatefulSet diff --git a/src/controllers/restatecluster/reconcilers/compute.rs b/src/controllers/restatecluster/reconcilers/compute.rs index 1d40dd1..42319d2 100644 --- a/src/controllers/restatecluster/reconcilers/compute.rs +++ b/src/controllers/restatecluster/reconcilers/compute.rs @@ -389,14 +389,32 @@ fn restate_statefulset( canary_image: &str, ) -> StatefulSet { let metadata = object_meta(base_metadata, RESTATE_STATEFULSET_NAME); - let labels = metadata.labels.clone(); - let pod_annotations = match (pod_annotations, metadata.annotations.clone()) { - (Some(pod_annotations), Some(mut base_annotations)) => { - base_annotations.extend(pod_annotations); - Some(base_annotations) + + // Merge pod labels: start with user-specified, then apply standard labels on top + // (standard labels take precedence in case of conflict) + let labels = { + let mut merged = spec.compute.labels.clone().unwrap_or_default(); + if let Some(standard) = metadata.labels.clone() { + merged.extend(standard); + } + Some(merged) + }; + + // Merge pod annotations: start with user-specified, then base metadata, then internal + // (internal annotations like WI/trusted-CA hashes take precedence) + let pod_annotations = { + let mut merged = spec.compute.annotations.clone().unwrap_or_default(); + if let Some(base) = metadata.annotations.clone() { + merged.extend(base); + } + if let Some(internal) = pod_annotations { + merged.extend(internal); + } + if merged.is_empty() { + None + } else { + Some(merged) } - (Some(annotations), None) | (None, Some(annotations)) => Some(annotations), - (None, None) => None, }; let mut volume_mounts = vec![ @@ -1564,6 +1582,7 @@ async fn apply_pod_disruption_budget( mod tests { use super::*; use crate::resources::iampolicymembers::{IAMPolicyMemberCondition, IAMPolicyMemberStatus}; + use crate::resources::restateclusters::RestateClusterCompute; use k8s_openapi::api::batch::v1::{JobCondition, JobStatus}; #[test] @@ -1958,4 +1977,116 @@ mod tests { assert_eq!(secret1.secret_name.as_deref(), Some("ca-two")); assert_eq!(secret1.items.as_ref().unwrap()[0].key, "root.pem"); } + + fn minimal_spec( + annotations: Option>, + labels: Option>, + ) -> RestateClusterSpec { + RestateClusterSpec { + compute: RestateClusterCompute { + image: "restate:test".into(), + annotations, + labels, + ..Default::default() + }, + storage: RestateClusterStorage { + storage_request_bytes: 1_000_000_000, + ..Default::default() + }, + ..Default::default() + } + } + + #[test] + fn test_statefulset_user_annotations_merged_with_internal() { + let user_annotations = BTreeMap::from([ + ( + "cloud.google.com/compute-class".into(), + "restate-workload".into(), + ), + ( + "restate.dev/trusted-ca-certs".into(), + "user-should-lose".into(), + ), + ]); + let internal_annotations = BTreeMap::from([( + "restate.dev/trusted-ca-certs".into(), + "internal-wins".into(), + )]); + let spec = minimal_spec(Some(user_annotations), None); + + let ss = restate_statefulset( + &test_base_metadata(), + &spec, + Some(internal_annotations), + None, + "config-abc".into(), + "busybox:uclibc", + ); + let pod_annotations = ss + .spec + .unwrap() + .template + .metadata + .unwrap() + .annotations + .unwrap(); + + // User annotation preserved + assert_eq!( + pod_annotations + .get("cloud.google.com/compute-class") + .unwrap(), + "restate-workload" + ); + // Internal annotation wins on conflict + assert_eq!( + pod_annotations.get("restate.dev/trusted-ca-certs").unwrap(), + "internal-wins" + ); + } + + #[test] + fn test_statefulset_user_labels_merged_with_standard() { + let user_labels = BTreeMap::from([ + ("team".into(), "platform".into()), + ("app.kubernetes.io/name".into(), "user-should-lose".into()), + ]); + let spec = minimal_spec(None, Some(user_labels)); + + let ss = restate_statefulset( + &test_base_metadata(), + &spec, + None, + None, + "config-abc".into(), + "busybox:uclibc", + ); + let pod_labels = ss.spec.unwrap().template.metadata.unwrap().labels.unwrap(); + + // User label preserved + assert_eq!(pod_labels.get("team").unwrap(), "platform"); + // Standard label wins on conflict + assert_eq!(pod_labels.get("app.kubernetes.io/name").unwrap(), "restate"); + } + + #[test] + fn test_statefulset_no_user_annotations_or_labels() { + let spec = minimal_spec(None, None); + + let ss = restate_statefulset( + &test_base_metadata(), + &spec, + None, + None, + "config-abc".into(), + "busybox:uclibc", + ); + let tmpl = ss.spec.unwrap().template.metadata.unwrap(); + + // Labels should still have standard labels + assert!(tmpl.labels.unwrap().contains_key("app.kubernetes.io/name")); + // Annotations should be None (no user, no base, no internal) + assert!(tmpl.annotations.is_none()); + } } diff --git a/src/resources/restateclusters.rs b/src/resources/restateclusters.rs index 24f2474..a950fa9 100644 --- a/src/resources/restateclusters.rs +++ b/src/resources/restateclusters.rs @@ -117,6 +117,12 @@ pub struct RestateClusterStorage { pub struct RestateClusterCompute { /// replicas is the desired number of Restate nodes. If unspecified, defaults to 1. pub replicas: Option, + /// Annotations to set on the Restate pod template. These are merged with any annotations + /// the operator sets internally (e.g. for workload identity, trusted CA certs). + pub annotations: Option>, + /// Labels to set on the Restate pod template. These are merged with the standard labels + /// the operator sets (app.kubernetes.io/name, etc.). + pub labels: Option>, /// Container image name. More info: https://kubernetes.io/docs/concepts/containers/images. pub image: String, /// Entrypoint array. Not executed within a shell. The container image's ENTRYPOINT is used if this is not provided.