Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ More examples are available just below the spec that follows.
| `image` | `string` | **Required**. Container image name. |
| `imagePullPolicy` | `string` | Image pull policy. One of `Always`, `Never`, `IfNotPresent`. Defaults to `Always` if `:latest` tag is specified, or `IfNotPresent` otherwise. |
| `imagePullSecrets` | `array` | Optional list of references to secrets in the same namespace to use for pulling the image. |
| `annotations` | `object` | Annotations to set on the Restate pod template. See note on merge ordering below. |
| `labels` | `object` | Labels to set on the Restate pod template. See note on merge ordering below. |
| `resources` | `object` | Compute Resources for the Restate container. e.g., `requests` and `limits` for `cpu` and `memory`. |
| `env` | `array` | List of environment variables to set in the container. |
| `affinity` | `object` | Standard Kubernetes affinity rules. |
Expand All @@ -118,6 +120,13 @@ More examples are available just below the spec that follows.
| `dnsPolicy` | `string` | Pod DNS policy. |
| `dnsConfig` | `object` | Pod DNS configuration. |

**Pod annotation and label merge ordering**: User-specified `annotations` and `labels` are
merged with values the operator sets internally (e.g. for workload identity hashes, trusted
CA cert hashes). If the same key appears in both, the operator's internal value wins. This
means operator-managed features like GCP Workload Identity annotations cannot be
accidentally overridden. If you need to set the same annotation that a built-in feature
uses, disable the built-in feature first — otherwise your value will be silently replaced.

---

#### `spec.storage`
Expand Down
16 changes: 16 additions & 0 deletions crd/restateclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,14 @@ spec:
type: array
type: object
type: object
annotations:
additionalProperties:
type: string
description: |-
Annotations to set on the Restate pod template. These are merged with any annotations
the operator sets internally (e.g. for workload identity, trusted CA certs).
nullable: true
type: object
args:
description: Arguments to the entrypoint. The container image's CMD is used if this is not provided.
items:
Expand Down Expand Up @@ -733,6 +741,14 @@ spec:
type: object
nullable: true
type: array
labels:
additionalProperties:
type: string
description: |-
Labels to set on the Restate pod template. These are merged with the standard labels
the operator sets (app.kubernetes.io/name, etc.).
nullable: true
type: object
nodeSelector:
additionalProperties:
type: string
Expand Down
37 changes: 37 additions & 0 deletions release-notes/unreleased/45-pod-annotations-and-labels.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Release Notes for Issue #45: Add support for custom pod annotations and labels

## New Feature

### What Changed
Added `spec.compute.annotations` and `spec.compute.labels` fields to the
RestateCluster CRD, allowing users to set custom annotations and labels on the
Restate StatefulSet pod template.

User-specified annotations and labels are merged with any that the operator sets
internally (e.g. for workload identity, trusted CA certs). In case of conflict,
operator-managed values take precedence.

### Why This Matters
Enables integrations that require pod-level metadata, such as GKE ComputeClass
scheduling (`cloud.google.com/compute-class`), Vault agent injection, Datadog,
Prometheus scraping, and custom scheduling constraints.

### Impact on Users
- Existing deployments: No impact, both fields are optional
- New deployments: Can now set annotations and labels for integrations that
require them on the pod template

### Migration Guidance
No migration required. To use the new fields:

```yaml
spec:
compute:
annotations:
cloud.google.com/compute-class: "restate-workload"
labels:
team: "platform"
```

### Related Issues
- Issue #45: Add support for custom annotations on StatefulSet
145 changes: 138 additions & 7 deletions src/controllers/restatecluster/reconcilers/compute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -389,14 +389,32 @@ fn restate_statefulset(
canary_image: &str,
) -> StatefulSet {
let metadata = object_meta(base_metadata, RESTATE_STATEFULSET_NAME);
let labels = metadata.labels.clone();
let pod_annotations = match (pod_annotations, metadata.annotations.clone()) {
(Some(pod_annotations), Some(mut base_annotations)) => {
base_annotations.extend(pod_annotations);
Some(base_annotations)

// Merge pod labels: start with user-specified, then apply standard labels on top
// (standard labels take precedence in case of conflict)
let labels = {
let mut merged = spec.compute.labels.clone().unwrap_or_default();
if let Some(standard) = metadata.labels.clone() {
merged.extend(standard);
}
Some(merged)
};

// Merge pod annotations: start with user-specified, then base metadata, then internal
// (internal annotations like WI/trusted-CA hashes take precedence)
let pod_annotations = {
let mut merged = spec.compute.annotations.clone().unwrap_or_default();
if let Some(base) = metadata.annotations.clone() {
merged.extend(base);
}
if let Some(internal) = pod_annotations {
merged.extend(internal);
}
if merged.is_empty() {
None
} else {
Some(merged)
}
(Some(annotations), None) | (None, Some(annotations)) => Some(annotations),
(None, None) => None,
};

let mut volume_mounts = vec![
Expand Down Expand Up @@ -1564,6 +1582,7 @@ async fn apply_pod_disruption_budget(
mod tests {
use super::*;
use crate::resources::iampolicymembers::{IAMPolicyMemberCondition, IAMPolicyMemberStatus};
use crate::resources::restateclusters::RestateClusterCompute;
use k8s_openapi::api::batch::v1::{JobCondition, JobStatus};

#[test]
Expand Down Expand Up @@ -1958,4 +1977,116 @@ mod tests {
assert_eq!(secret1.secret_name.as_deref(), Some("ca-two"));
assert_eq!(secret1.items.as_ref().unwrap()[0].key, "root.pem");
}

fn minimal_spec(
annotations: Option<BTreeMap<String, String>>,
labels: Option<BTreeMap<String, String>>,
) -> RestateClusterSpec {
RestateClusterSpec {
compute: RestateClusterCompute {
image: "restate:test".into(),
annotations,
labels,
..Default::default()
},
storage: RestateClusterStorage {
storage_request_bytes: 1_000_000_000,
..Default::default()
},
..Default::default()
}
}

#[test]
fn test_statefulset_user_annotations_merged_with_internal() {
let user_annotations = BTreeMap::from([
(
"cloud.google.com/compute-class".into(),
"restate-workload".into(),
),
(
"restate.dev/trusted-ca-certs".into(),
"user-should-lose".into(),
),
]);
let internal_annotations = BTreeMap::from([(
"restate.dev/trusted-ca-certs".into(),
"internal-wins".into(),
)]);
let spec = minimal_spec(Some(user_annotations), None);

let ss = restate_statefulset(
&test_base_metadata(),
&spec,
Some(internal_annotations),
None,
"config-abc".into(),
"busybox:uclibc",
);
let pod_annotations = ss
.spec
.unwrap()
.template
.metadata
.unwrap()
.annotations
.unwrap();

// User annotation preserved
assert_eq!(
pod_annotations
.get("cloud.google.com/compute-class")
.unwrap(),
"restate-workload"
);
// Internal annotation wins on conflict
assert_eq!(
pod_annotations.get("restate.dev/trusted-ca-certs").unwrap(),
"internal-wins"
);
}

#[test]
fn test_statefulset_user_labels_merged_with_standard() {
let user_labels = BTreeMap::from([
("team".into(), "platform".into()),
("app.kubernetes.io/name".into(), "user-should-lose".into()),
]);
let spec = minimal_spec(None, Some(user_labels));

let ss = restate_statefulset(
&test_base_metadata(),
&spec,
None,
None,
"config-abc".into(),
"busybox:uclibc",
);
let pod_labels = ss.spec.unwrap().template.metadata.unwrap().labels.unwrap();

// User label preserved
assert_eq!(pod_labels.get("team").unwrap(), "platform");
// Standard label wins on conflict
assert_eq!(pod_labels.get("app.kubernetes.io/name").unwrap(), "restate");
}

#[test]
fn test_statefulset_no_user_annotations_or_labels() {
let spec = minimal_spec(None, None);

let ss = restate_statefulset(
&test_base_metadata(),
&spec,
None,
None,
"config-abc".into(),
"busybox:uclibc",
);
let tmpl = ss.spec.unwrap().template.metadata.unwrap();

// Labels should still have standard labels
assert!(tmpl.labels.unwrap().contains_key("app.kubernetes.io/name"));
// Annotations should be None (no user, no base, no internal)
assert!(tmpl.annotations.is_none());
}
}
6 changes: 6 additions & 0 deletions src/resources/restateclusters.rs
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,12 @@ pub struct RestateClusterStorage {
pub struct RestateClusterCompute {
/// replicas is the desired number of Restate nodes. If unspecified, defaults to 1.
pub replicas: Option<i32>,
/// Annotations to set on the Restate pod template. These are merged with any annotations
/// the operator sets internally (e.g. for workload identity, trusted CA certs).
pub annotations: Option<BTreeMap<String, String>>,
/// Labels to set on the Restate pod template. These are merged with the standard labels
/// the operator sets (app.kubernetes.io/name, etc.).
pub labels: Option<BTreeMap<String, String>>,
/// Container image name. More info: https://kubernetes.io/docs/concepts/containers/images.
pub image: String,
/// Entrypoint array. Not executed within a shell. The container image's ENTRYPOINT is used if this is not provided.
Expand Down
Loading