diff --git a/examples/big-peer-deployment/.env.example b/examples/big-peer-deployment/.env.example new file mode 100644 index 0000000..8bd02e0 --- /dev/null +++ b/examples/big-peer-deployment/.env.example @@ -0,0 +1,82 @@ +# --------------------------------------------------------------------------- +# Example environment for the Makefile (Part 2 of this example). +# +# cp .env.example .env.local # then edit +# +# There are two files, and neither is committed: +# +# .env Generated by `terraform apply`. Holds DEPLOYMENT, CLUSTER and +# REGION only. Overwritten on every apply -- never edit it. +# Before the first apply the Makefile derives the same values +# from config.yaml instead. +# +# .env.local Yours. Loaded after .env, so anything here wins. Terraform +# never touches it. +# +# Anything you leave unset falls back to the default shown below. You can also +# override any of these per-invocation: make EXTERNAL=1 envoy +# --------------------------------------------------------------------------- + +# --- AWS ------------------------------------------------------------------- +# The profile used for `aws eks update-kubeconfig` and every AWS call. There is +# no sensible default, so set this unless you rely on ambient credentials. +AWS_PROFILE=my-aws-profile + +# Normally these come from .env (Terraform) or config.yaml. Set them only to +# target a cluster this example did not build. +#CLUSTER=big-peer-dev +#REGION=us-east-1 + +# --- Big Peer -------------------------------------------------------------- +# Namespace for the operator and the Big Peer instance. +NAMESPACE=ditto + +# Name of the BigPeer resource, and the Ditto version it runs. +BIGPEER_NAME=example +BIGPEER_VERSION=1.59.0 + +# Playground anonymous-auth token. The quickstart uses abc123; it grants full +# read/write, so change it for anything that is not a throwaway cluster. +SHARED_TOKEN=abc123 + +# --- Networking ------------------------------------------------------------ +# 0 = internal NLB only (default). 1 = also provision an internet-facing NLB. +# With no ACM certificate in the account the external gateway serves plaintext +# HTTP, so leave this off unless you understand the exposure. +EXTERNAL=0 + +# Envoy replicas per gateway. Raise once the cluster has more than one node. +ENVOY_REPLICAS=1 + +# Kafka's external listener is an `ingress`-type Strimzi listener, needed only +# for Kafka Data Bridges. This example installs Envoy Gateway, which implements +# the Gateway API and does NOT serve Ingress -- so with this on, Strimzi blocks +# forever waiting for an address and Big Peer never starts. Leave at 0 unless +# you have installed an Ingress controller. +KAFKA_EXTERNAL_LISTENER=0 + +# --- Portal (optional) ----------------------------------------------------- +# Self-managed portal UI, shipped in the ditto-operator chart. 0 = not +# installed (default). 1 = install it and route it through the *internal* +# gateway -- it never gets its own load balancer or public address. +# +# The chart's portal.ingress / portal.operatorApiIngress are left disabled: +# Envoy Gateway does not serve Ingress. An HTTPRoute is used instead, serving +# the portal, the operator API and the Big Peer API from this one hostname so +# the browser stays same-origin and needs no CORS policy. +PORTAL=0 + +# Hostname the portal is served on. Nothing registers this in DNS -- add it to +# /etc/hosts, or point a private zone at the internal NLB. +PORTAL_HOST=portal.ditto.local + +# Portal image tag (quay.io/ditto-external/portal-self-managed). +PORTAL_VERSION=0.3.0 + +# --- Chart versions -------------------------------------------------------- +# Pinned so a re-run installs the same thing. cert-manager and Strimzi track +# the Ditto operator quickstart; gateway-helm tracks cloud-infra dev. +CERT_MANAGER_VERSION=v1.21.1 +ENVOY_GATEWAY_VERSION=1.6.3 +STRIMZI_VERSION=0.49.0 +DITTO_OPERATOR_VERSION=0.17.2 diff --git a/examples/big-peer-deployment/.gitignore b/examples/big-peer-deployment/.gitignore new file mode 100644 index 0000000..ba7548e --- /dev/null +++ b/examples/big-peer-deployment/.gitignore @@ -0,0 +1,14 @@ +# Generated by terraform (env.tf) and scripts/config.sh +.env +# Your local overrides -- copy from .env.example +.env.local +# Self-contained kubeconfig written by `make kubeconfig` +.kube/ +# App ID and API key written by scripts/app.sh +.local/ + +# Terraform +.terraform/ +*.tfstate +*.tfstate.backup +*.tfplan diff --git a/examples/big-peer-deployment/Makefile b/examples/big-peer-deployment/Makefile new file mode 100644 index 0000000..1ed2d50 --- /dev/null +++ b/examples/big-peer-deployment/Makefile @@ -0,0 +1,194 @@ +# +# Part 2 of this example: deploy Big Peer and its dependencies onto the EKS +# cluster that Part 1 (Terraform) built. See README.md. +# +# make # list targets +# make all # kubeconfig -> bootstrap -> bigpeer -> app -> smoke +# make FORCE=1 … # reinstall components that are already healthy +# + +SHELL := /usr/bin/env bash +.SHELLFLAGS := -euo pipefail -c +.DEFAULT_GOAL := help + +# These steps are strictly ordered (CNI before nodes, gateways before routes, +# Big Peer before its app). -j would interleave them. +.NOTPARALLEL: + +S := ./scripts +KUBEDIR := $(CURDIR)/.kube +KUBECFG := $(KUBEDIR)/config + +# --------------------------------------------------------------------------- +# Configuration, in precedence order (lowest first): +# +# 1. the defaults below +# 2. .env - written by `terraform apply`, or derived from config.yaml +# 3. .env.local - yours, never touched by Terraform (see .env.example) +# 4. the command line, e.g. `make EXTERNAL=1 envoy` +# +# `-include` tolerates the files being absent. The `.env:` rule below creates +# it from config.yaml when Terraform has not run yet. +# --------------------------------------------------------------------------- + +# Written atomically: a partial .env would be silently swallowed by -include +# and leave `make` pointed at the placeholder defaults below. +.env: + @echo "==> .env not found; deriving it from config.yaml" + @$(S)/config.sh > $@.tmp && mv $@.tmp $@ \ + || { rm -f $@.tmp; echo "ERROR: could not derive .env from config.yaml" >&2; exit 1; } + +-include .env +-include .env.local + +# Nothing above is required to be set: every value has a working default here. +CLUSTER ?= big-peer-dev +REGION ?= us-east-1 +DEPLOYMENT ?= $(CLUSTER) +KUBE_CONTEXT ?= $(CLUSTER) + +NAMESPACE ?= ditto +BIGPEER_NAME ?= example +BIGPEER_VERSION ?= 1.59.0 +SHARED_TOKEN ?= abc123 + +EXTERNAL ?= 0 +ENVOY_REPLICAS ?= 1 + +# Kafka's ingress-type external listener. Needs an Ingress controller, which +# this example does not install (Envoy Gateway is Gateway API only). Only +# required for Kafka Data Bridges. +KAFKA_EXTERNAL_LISTENER ?= 0 + +# Self-managed portal UI. Ships in the ditto-operator chart, off by default. +# Served from the internal gateway only -- no public load balancer. +PORTAL ?= 0 +PORTAL_HOST ?= portal.ditto.local +PORTAL_VERSION ?= 0.3.0 + +CERT_MANAGER_VERSION ?= v1.21.1 +ENVOY_GATEWAY_VERSION ?= 1.6.3 +STRIMZI_VERSION ?= 0.49.0 +DITTO_OPERATOR_VERSION ?= 0.17.2 + +# Scripts read all of this from the environment. +export CLUSTER REGION DEPLOYMENT KUBE_CONTEXT NAMESPACE BIGPEER_NAME +export BIGPEER_VERSION SHARED_TOKEN EXTERNAL ENVOY_REPLICAS FORCE +export KAFKA_EXTERNAL_LISTENER PORTAL PORTAL_HOST PORTAL_VERSION +export CERT_MANAGER_VERSION ENVOY_GATEWAY_VERSION STRIMZI_VERSION DITTO_OPERATOR_VERSION + +# Self-contained kubeconfig: this example never touches ~/.kube/config, so it +# cannot disturb whatever cluster you have selected globally. +export KUBECONFIG := $(KUBECFG) + +ifdef AWS_PROFILE +export AWS_PROFILE +endif + +# --------------------------------------------------------------------------- + +.PHONY: help +help: ## Show this help + @echo "Big Peer on EKS - Part 2 (deploy)" + @echo + @grep -hE '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) \ + | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' + @echo + @echo "Cluster : $(CLUSTER) ($(REGION)) profile: $(or $(AWS_PROFILE),)" + @echo "Big Peer: $(BIGPEER_NAME) v$(BIGPEER_VERSION) in ns/$(NAMESPACE) EXTERNAL=$(EXTERNAL)" + @echo "Config : $(if $(wildcard .env),.env,) $(if $(wildcard .env.local),+ .env.local,)" + @echo "Kubecfg : $(KUBECFG)" + +.PHONY: config +config: ## Print the resolved configuration + @printf ' %-24s %s\n' \ + CLUSTER "$(CLUSTER)" REGION "$(REGION)" DEPLOYMENT "$(DEPLOYMENT)" \ + AWS_PROFILE "$(or $(AWS_PROFILE),)" KUBECONFIG "$(KUBECFG)" \ + NAMESPACE "$(NAMESPACE)" BIGPEER_NAME "$(BIGPEER_NAME)" \ + BIGPEER_VERSION "$(BIGPEER_VERSION)" EXTERNAL "$(EXTERNAL)" \ + ENVOY_REPLICAS "$(ENVOY_REPLICAS)" \ + KAFKA_EXTERNAL_LISTENER "$(KAFKA_EXTERNAL_LISTENER)" \ + PORTAL "$(PORTAL)" PORTAL_HOST "$(PORTAL_HOST)" \ + PORTAL_VERSION "$(PORTAL_VERSION)" \ + CERT_MANAGER_VERSION "$(CERT_MANAGER_VERSION)" \ + ENVOY_GATEWAY_VERSION "$(ENVOY_GATEWAY_VERSION)" \ + STRIMZI_VERSION "$(STRIMZI_VERSION)" \ + DITTO_OPERATOR_VERSION "$(DITTO_OPERATOR_VERSION)" + +# File target: written once, then reused. `make kubeconfig` forces a refresh. +$(KUBECFG): + @mkdir -p $(KUBEDIR) + @echo "==> writing kubeconfig for $(CLUSTER) -> $(KUBECFG)" + @aws eks update-kubeconfig --region "$(REGION)" --name "$(CLUSTER)" \ + --alias "$(KUBE_CONTEXT)" --kubeconfig "$(KUBECFG)" + +.PHONY: kubeconfig +kubeconfig: ## (Re)write the local kubeconfig + @rm -f $(KUBECFG) + @$(MAKE) --no-print-directory $(KUBECFG) + +.PHONY: preflight +preflight: $(KUBECFG) ## Verify tooling, cluster reachability and capacity + @$(S)/preflight.sh + +.PHONY: storage +storage: $(KUBECFG) ## Default gp3 StorageClass (EBS CSI driver comes from Terraform) + @$(S)/storage.sh + +.PHONY: cert-manager +cert-manager: $(KUBECFG) ## cert-manager + a self-signed ClusterIssuer + @$(S)/cert-manager.sh + +.PHONY: envoy +envoy: $(KUBECFG) ## Envoy Gateway + internal (and optional external) gateways + @$(S)/envoy-gateway.sh + +.PHONY: strimzi +strimzi: $(KUBECFG) ## Strimzi Kafka operator + @$(S)/strimzi.sh + +.PHONY: operator +operator: $(KUBECFG) ## Ditto Operator + @$(S)/ditto-operator.sh + +.PHONY: portal +portal: $(KUBECFG) ## Portal UI via the internal gateway (needs PORTAL=1) + @PORTAL=1 $(S)/ditto-operator.sh + @PORTAL=1 $(S)/portal.sh + +.PHONY: bootstrap +bootstrap: preflight storage cert-manager envoy strimzi operator ## All cluster dependencies + @$(S)/portal.sh + @echo "==> bootstrap complete" + +.PHONY: bigpeer +bigpeer: $(KUBECFG) ## BigPeer CR + its HTTPRoutes + @$(S)/bigpeer.sh + +.PHONY: app +app: $(KUBECFG) ## Create an app + API key (writes .local/app.env) + @$(S)/app.sh + +.PHONY: smoke +smoke: $(KUBECFG) ## Insert and query a document through the gateway + @$(S)/smoke.sh + +.PHONY: all +all: kubeconfig bootstrap bigpeer app smoke ## Everything, end to end + @echo "==> all done" + +.PHONY: status +status: $(KUBECFG) ## Show the state of every component + @$(S)/status.sh + +.PHONY: endpoint +endpoint: $(KUBECFG) ## Print the gateway endpoints + @$(S)/status.sh endpoints + +.PHONY: teardown +teardown: $(KUBECFG) ## Remove everything the Makefile installed (leaves Terraform alone) + @$(S)/teardown.sh + +.PHONY: clean +clean: ## Remove generated local files (.env, .kube/, .local/) + rm -rf $(KUBEDIR) .local .env diff --git a/examples/big-peer-deployment/README.md b/examples/big-peer-deployment/README.md new file mode 100644 index 0000000..bd8ef32 --- /dev/null +++ b/examples/big-peer-deployment/README.md @@ -0,0 +1,342 @@ +# Big Peer on Amazon EKS + +A complete, runnable example of standing up a Ditto **Big Peer** on AWS, in two +parts: + +| | What it does | Tool | +|---|---|---| +| **Part 1 — Infrastructure** | VPC, EKS cluster, node groups, core EKS add-ons | `terraform` | +| **Part 2 — Deployment** | cert-manager, Envoy Gateway, Strimzi, Ditto Operator, Big Peer | `make` | + +Part 1 hands off to Part 2 through a generated `.env`, so the cluster name and +region are configured in exactly one place: `config.yaml`. + +> This example is intended for an isolated AWS account. It is not yet wired into +> the `dittocloud` CLI. + +--- + +## Quick start + +```bash +cd examples/big-peer-deployment + +# Part 1 - infrastructure +terraform init +terraform apply # writes .env + +# Part 2 - deployment +cp .env.example .env.local # set AWS_PROFILE, then edit to taste +make all +``` + +`make all` runs `kubeconfig → bootstrap → bigpeer → app → smoke` and finishes by +inserting and reading back a document through the gateway. + +## Requirements + +`terraform` ≥ 1.5, `awscli` v2, `kubectl`, `helm` ≥ 3, `jq`. + +Part 2 also needs a YAML parser **only if you run `make` before +`terraform apply`** — `yq`, Python with PyYAML, or Ruby (ships with macOS). +After an apply, `.env` already exists and no parser is needed. + +--- + +# Part 1 — Infrastructure (Terraform) + +Edit `config.yaml`. Each top-level key under `deployments` is a complete +VPC + EKS stack. + +```yaml +deployments: + big-peer-dev: + region: us-east-1 + vpc: + create: true + cidr: 10.0.0.0/16 + availability_zones: ["us-east-1a", "us-east-1b", "us-east-1c"] + private_subnets: ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets: ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] + cluster: + name: big-peer-dev + version: "1.30" + # SSO/IAM roles granted cluster-admin via EKS access entries. Use the + # full ARN including the aws-reserved path - EKS rejects the + # path-stripped form as "invalid principal". + admin_principal_arns: + - arn:aws:iam::123456789012:role/aws-reserved/sso.amazonaws.com/us-east-1/AWSReservedSSO_Admin_0123456789abcdef + managed_node_groups: + default: + instance_types: ["m6i.large"] + min_size: 3 + max_size: 8 + desired_size: 3 +``` + +### Existing VPC + +Set `vpc.create: false` and supply `vpc_id`, `private_subnet_ids` and +`public_subnet_ids`. No VPC resources are created or modified. See +[`docs/bring-your-own-vpc.md`](../../docs/bring-your-own-vpc.md); at minimum the +VPC must have subnets in three AZs, `enableDnsSupport` + `enableDnsHostnames`, +`kubernetes.io/role/internal-elb=1` on private subnets and +`kubernetes.io/role/elb=1` on public ones, and a route to AWS APIs and +container registries. + +### Add-ons installed + +| Add-on | Notes | +|---|---| +| `vpc-cni`, `kube-proxy` | `before_compute = true` | +| `coredns` | after compute — it needs a node to schedule on | +| `eks-pod-identity-agent` | prerequisite for the EBS CSI driver's IAM | +| `aws-ebs-csi-driver` | pod identity + `AmazonEBSCSIDriverPolicyV2` | + +**Why `before_compute` matters.** The upstream module gives `aws_eks_addon` a +`depends_on` covering the node groups, so by default add-ons install *after* +compute. A node cannot reach `Ready` without a CNI, so the node group fails with +`NodeCreationFailure` and the apply dies before the CNI is ever installed. Marking +`vpc-cni` and `kube-proxy` as `before_compute` breaks that deadlock. + +**Why the EBS CSI driver is not optional.** Strimzi/Kafka and Big Peer both need +PersistentVolumes, and Kubernetes ≥ 1.31 has no in-tree EBS provisioner. Without +it every PVC stays `Pending` and Part 2 stalls at Strimzi. + +### Multiple deployments + +`.env` describes one cluster. With more than one deployment defined, pick which: + +```bash +terraform apply -var primary_deployment=big-peer-dev +``` + +To target a different deployment without re-running Terraform, override the +Makefile's cluster selection on the command line: + +```bash +make DEPLOYMENT=big-peer-prod CLUSTER=big-peer-prod REGION=us-west-2 all +``` + +--- + +# Part 2 — Deployment (Make) + +```bash +make # list targets and show resolved config +make all # everything, end to end +``` + +### Targets + +| Target | | +|---|---| +| `kubeconfig` | Write `./.kube/config` for the cluster | +| `preflight` | Verify tooling, reachability, node capacity, EBS CSI | +| `storage` | Default `gp3` StorageClass | +| `cert-manager` | cert-manager + self-signed `ClusterIssuer` | +| `envoy` | Envoy Gateway + internal (and optional external) gateways | +| `strimzi` | Strimzi Kafka operator | +| `operator` | Ditto Operator | +| `bootstrap` | All of the above | +| `bigpeer` | BigPeer CR + its HTTPRoutes | +| `portal` | Portal UI behind the internal gateway (`PORTAL=1`) | +| `app` | Create an app + API key → `.local/app.env` | +| `smoke` | Insert and query a document through the gateway | +| `status` / `endpoint` | Show component state / gateway addresses | +| `teardown` | Remove everything Make installed (leaves Terraform alone) | + +### Configuration + +Resolved lowest-precedence first: + +1. Defaults in the `Makefile` +2. **`.env`** — generated by `terraform apply`. Holds `DEPLOYMENT`, `CLUSTER`, + `REGION` only. Overwritten on every apply, so never edit it. Before the first + apply, `make` derives the same three values from `config.yaml`. +3. **`.env.local`** — yours. Copy from `.env.example`. Terraform never touches it. +4. The command line — `make EXTERNAL=1 envoy` + +Notable settings (all documented in `.env.example`): `AWS_PROFILE`, +`BIGPEER_NAME`, `BIGPEER_VERSION`, `SHARED_TOKEN`, `EXTERNAL`, +`ENVOY_REPLICAS`, `KAFKA_EXTERNAL_LISTENER`, and the four pinned chart versions. + +```bash +make config # print everything as resolved +``` + +### kubeconfig is self-contained + +Every target writes and reads `./.kube/config` and exports `KUBECONFIG` to it. +Nothing here touches `~/.kube/config`, so running this example cannot disturb +whatever cluster you have selected globally. Every `kubectl` call also pins +`--context` explicitly. + +### Re-running is cheap + +Each install target checks whether its component is already deployed *and +healthy*, and skips if so: + +``` + --> cert-manager v1.21.1 (already installed, skipping) +``` + +`make FORCE=1 bootstrap` reinstalls regardless. Toggling `EXTERNAL` is always +honoured — the Envoy target compares the installed gateways against the flag +rather than skipping blindly. + +### Networking + +Envoy Gateway, following the pattern used in `cloud-infra-apps`: an +`EnvoyProxy` → `GatewayClass` → `Gateway` triple per exposure. + +- **Internal** (`int-envoy-proxy`) — always created, internal NLB. +- **External** (`ext-envoy-proxy`) — only with `EXTERNAL=1`. Off by default: + without an ACM certificate it serves plaintext HTTP. `EXTERNAL=0` actively + removes it, so it is a real off switch. + +NLBs use the legacy cloud-controller-manager annotations +(`aws-load-balancer-type: nlb`), which EKS honours natively — the AWS Load +Balancer Controller is **not** required and is not installed by this example. + +Every service this example creates is either `ClusterIP` or an **internal** NLB. +`int-envoy-proxy` is the only load balancer at `EXTERNAL=0`, and Big Peer, the +portal and the operator API all share it. Nothing is reachable from the +internet unless you explicitly set `EXTERNAL=1`. + +Big Peer is reached through three `HTTPRoute` rules, ordered by longest path +prefix: + +| Path | Backend | +|---|---| +| `/_ditto/auth` | `ditto--auth-server:7070` | +| `/_ditto` | `ditto--subscription:7070` | +| `/` | `ditto--api:8080` | + +> The quickstart's `spec.network.ingress.host` makes the operator create a +> Kubernetes `Ingress`. Envoy Gateway implements the Gateway API and does not +> serve `Ingress`, so that field is omitted and routes are authored directly. +> Operator `0.17.2` serves only `v1alpha1`; the `spec.network.mode: GatewayApi` +> field referenced in the CRD docs arrives in `v1alpha2`. + +### Portal (optional) + +The self-managed portal UI ships inside the `ditto-operator` chart +(`portal.enabled`, default `false`) as +`quay.io/ditto-external/portal-self-managed`. It is **not** covered by the +operator quickstart. Enable it with: + +```bash +PORTAL=1 make portal # or: PORTAL=1 make bootstrap +``` + +Like everything else here it sits behind the **internal** gateway and gets no +load balancer of its own — the portal, operator API and Big Peer API are all +served from a single hostname on the existing internal NLB: + +| Path on `$PORTAL_HOST` | Backend | +|---|---| +| `/operator-api` | `ditto-operator:80` (prefix stripped) | +| `/big-peer` | `ditto--api:8080` (prefix stripped) | +| `/` | `ditto-operator-portal:80` | + +One hostname is deliberate. The chart's default topology puts the portal and +the operator API on separate hosts via two `Ingress` objects, which makes every +API call cross-origin and needs a CORS policy on the API. Serving both from one +origin avoids that entirely. Both chart `Ingress` templates are left disabled, +since Envoy Gateway implements the Gateway API and does not serve `Ingress`. + +Nothing registers `$PORTAL_HOST` in DNS. Reach it from inside the VPC, or +locally by port-forwarding the Envoy service and supplying the `Host` header +(`make portal` prints the exact commands). + +> **The operator API is unauthenticated by default.** The chart ships +> `operator.api.authMode: disabled` — *"API is open to all requests"* — and the +> portal's rendered config carries `AUTH_MODE: "disabled"`. That API can create +> apps and mint API keys. It is only reachable inside the VPC here, but anyone +> on the network can use it. The chart documents `k8s-token` as the alternative; +> the portal templates also branch on a `better-auth` mode that the values file +> does not document. Set an auth mode before treating this as anything but a +> lab. + +### Kafka and the Ingress trap + +Big Peer defaults to an **`ingress`-type external listener** for Kafka +(`spec.cdc.network.kafkaExternalListener.enabled` defaults to `true`). Strimzi +then blocks its entire reconcile waiting for that Ingress to become addressable. +Envoy Gateway implements the Gateway API and does **not** serve `Ingress`, so +nothing ever assigns an address: the Kafka broker is never created, its PVC sits +on `WaitForFirstConsumer`, and every Big Peer pod stays `0/1` waiting for Kafka. +The symptom looks like a storage or capacity problem, but is neither. + +`KAFKA_EXTERNAL_LISTENER=0` (the default here) disables it. The listener is only +needed for **Kafka Data Bridges**; set it to `1` only if you have also installed +an Ingress controller. + +### Credentials + +`make app` writes `.local/app.env` (mode `0600`, gitignored) with the app ID and +API key. `make smoke` reads it. Neither is printed in full. + +`SHARED_TOKEN` defaults to `abc123` from the quickstart and grants full +read/write. Change it for anything that is not a throwaway cluster. + +--- + +## Sizing and autoscaling + +**This cluster does not autoscale.** `max_size` is only the ceiling of the node +group's Auto Scaling Group — nothing raises the desired count when pods go +Pending. EKS managed node groups do not include an autoscaler, and this example +installs neither Cluster Autoscaler nor Karpenter. + +**`desired_size` applies only at creation.** The upstream EKS module sets +`ignore_changes = [scaling_config[0].desired_size]` so an autoscaler can own the +count without Terraform reverting it. With no autoscaler installed, editing +`desired_size` in `config.yaml` after the node group exists produces +`No changes` on plan — resize the node group through the EKS API or console. +`config.yaml` still governs the size of a newly created node group. + +The full stack — cert-manager, Envoy, Strimzi, Kafka, the operator and Big Peer — +is a tight fit on a single 2 vCPU node. `make preflight` warns below three Ready +nodes. If pods do not come up, `make bigpeer` distinguishes the three causes +(Kafka blocked, PVC Pending, insufficient capacity) rather than guessing. + +## Troubleshooting + +| Symptom | Cause | +|---|---| +| Nodes `NotReady`, node group `CREATE_FAILED` | No CNI. Check `vpc-cni` is installed and marked `before_compute`. | +| PVCs stuck `Pending` | EBS CSI driver missing, or no default StorageClass — run `make storage`. | +| `kubectl` returns 401 | Your IAM principal has no access entry. Add it to `admin_principal_arns` and re-apply. | +| Pods `Pending`, no PVC issue | Not enough capacity. Raise `desired_size` — there is no autoscaler. | +| Kafka never starts; `Exceeded timeout … waiting for Ingress … to be addressable` | Big Peer's Kafka external listener is on. See below. | +| Gateway address `` | NLB still provisioning, or subnets missing their `kubernetes.io/role/*elb` tags. | + +## Cleanup + +```bash +make teardown # in-cluster software (prompts for the cluster name) +terraform destroy # infrastructure +make clean # local .env, .kube/, .local/ +``` + +## Layout + +``` +big-peer-deployment/ +├── config.yaml # deployment definitions - the single source of truth +├── main.tf # parses config.yaml, orchestrates deployment modules +├── env.tf # writes .env for the Makefile +├── providers.tf variables.tf outputs.tf +├── modules/deployment/ # one VPC + EKS stack per deployment +│ ├── vpc.tf eks.tf vpc-endpoints.tf outputs.tf variables.tf +├── Makefile # Part 2 entry point +├── .env.example # template for .env.local +└── scripts/ + ├── lib.sh # shared helpers, skip checks, waits + ├── config.sh # derive .env from config.yaml (pre-apply fallback) + ├── preflight.sh storage.sh cert-manager.sh envoy-gateway.sh + ├── strimzi.sh ditto-operator.sh bigpeer.sh + └── app.sh smoke.sh status.sh teardown.sh +``` diff --git a/examples/big-peer-deployment/config.yaml b/examples/big-peer-deployment/config.yaml new file mode 100644 index 0000000..9d585ee --- /dev/null +++ b/examples/big-peer-deployment/config.yaml @@ -0,0 +1,82 @@ +# Deployment definitions - the single source of truth for Part 1. +# +# Each top-level key under "deployments" is one complete VPC + EKS stack. +# With more than one deployment defined, every deployment must target the +# same region (the root provider serves all of them), and `terraform apply` +# needs -var primary_deployment= to pick which one Part 2 targets. +# +# Quote every subnet/VPC ID. A stray character in an unquoted value reaches +# Terraform as a "real" string, and EKS rejects it as an invalid subnet. +--- +deployments: + # A development stack: this example creates the VPC for you. + big-peer-dev: + # Must match the root provider region (`terraform apply -var region=...`). + region: us-east-1 + + vpc: + # true = create a new VPC (cidr + subnet CIDRs below) + # false = use an existing VPC (vpc_id + subnet IDs; see big-peer-prod) + create: true + cidr: 10.0.0.0/16 + # EKS requires subnets in at least two AZs; three is the safe default. + availability_zones: ["us-east-1a", "us-east-1b", "us-east-1c"] + # Nodes run in private subnets (NAT for egress, no public IPs). + private_subnets: ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + # Public subnets host the NAT gateways (and ELBs, if exposed). + public_subnets: ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] + + cluster: + name: big-peer-dev + # Kubernetes version. Keep the quotes: YAML would read 1.30 as 1.3. + version: "1.30" + # REQUIRED: SSO/IAM roles granted cluster-admin via EKS access entries. + # At least one entry is validated at plan time - without it, no identity + # can reach the cluster API (the EKS module's cluster-creator admin + # permission defaults to off). + # Use the FULL role ARN including any path (e.g. the aws-reserved SSO + # path) - EKS rejects path-stripped ARNs as invalid principals. + admin_principal_arns: + - arn:aws:iam::123456789012:role/aws-reserved/sso.amazonaws.com/us-east-1/AWSReservedSSO_Admin_0123456789abcdef + managed_node_groups: + default: + instance_types: ["m6i.large"] + # desired_size applies only at creation. The EKS module then + # ignores it so an autoscaler can own the count; this example + # installs no autoscaler, so resize via the EKS API/console. + min_size: 2 + max_size: 4 + desired_size: 2 + + # A production-style stack in an existing, customer-managed VPC. + # Commented out by default: every deployment must share the root provider + # region, and the placeholder IDs below must be replaced with real ones. + # Uncomment, set region to match big-peer-dev (or pass -var region=...), + # and fill in your VPC/subnet IDs. + # + # big-peer-prod: + # region: us-west-2 + # + # vpc: + # create: false + # vpc_id: vpc-0123456789abcdef0 + # # All subnets must be in this VPC, in the target region, and spread + # # across at least three AZs. The VPC also needs DNS support + hostnames, + # # kubernetes.io/role/internal-elb=1 on private subnets, + # # kubernetes.io/role/elb=1 on public subnets, and routes to AWS APIs + # # and container registries. See docs/bring-your-own-vpc.md. + # private_subnet_ids: ["subnet-0123456789abcdef0", "subnet-1123456789abcdef0", "subnet-2123456789abcdef0"] + # public_subnet_ids: ["subnet-3123456789abcdef0", "subnet-4123456789abcdef0", "subnet-5123456789abcdef0"] + # + # cluster: + # name: big-peer-prod + # version: "1.30" + # admin_principal_arns: + # - arn:aws:iam::123456789012:role/your-admin-role + # managed_node_groups: + # default: + # instance_types: ["m6i.xlarge"] + # # The full stack (Kafka + operator + Big Peer) wants >= 3 nodes. + # min_size: 3 + # max_size: 6 + # desired_size: 3 diff --git a/examples/big-peer-deployment/env.tf b/examples/big-peer-deployment/env.tf new file mode 100644 index 0000000..7bae77b --- /dev/null +++ b/examples/big-peer-deployment/env.tf @@ -0,0 +1,49 @@ +# Terraform is the source of truth for what actually got built, so it writes the +# handful of values the Makefile needs to target the right cluster. Everything +# else the Makefile can discover from the cluster itself. +# +# This file is regenerated on every apply, so it holds nothing a human would +# want to edit. User-owned settings (AWS profile, Big Peer name, EXTERNAL, ...) +# belong in .env.local, which Terraform never touches. See .env.example. +# +# The Makefile falls back to scripts/config.sh, which parses config.yaml +# directly, so `make` is usable before the first apply. + +locals { + # With a single deployment the choice is unambiguous. With several, the caller + # picks one with -var primary_deployment=. + primary_deployment = ( + var.primary_deployment != null + ? var.primary_deployment + : (length(local.deployments) == 1 ? one(keys(local.deployments)) : null) + ) + + primary = local.primary_deployment != null ? module.deployment[local.primary_deployment] : null +} + +resource "local_file" "env" { + count = local.primary_deployment != null ? 1 : 0 + + filename = "${path.module}/.env" + file_permission = "0644" + + content = <<-EOT + # Generated by terraform (env.tf) -- do not edit; re-run `terraform apply`. + # Put your own overrides in .env.local instead. See .env.example. + DEPLOYMENT=${local.primary_deployment} + CLUSTER=${local.primary.cluster_name} + REGION=${local.primary.region} + EOT + + lifecycle { + precondition { + condition = local.primary_deployment != null + error_message = "Multiple deployments are defined; choose one with -var primary_deployment=." + } + } +} + +output "env_file" { + description = "Path to the generated .env consumed by the Makefile." + value = local.primary_deployment != null ? local_file.env[0].filename : null +} diff --git a/examples/big-peer-deployment/main.tf b/examples/big-peer-deployment/main.tf new file mode 100644 index 0000000..d3fce16 --- /dev/null +++ b/examples/big-peer-deployment/main.tf @@ -0,0 +1,35 @@ +# Parse the deployment definitions from YAML. Each top-level key under +# "deployments" is a complete VPC + EKS stack. +locals { + deployments = yamldecode(file(var.config_path)).deployments + + # The root provider serves all deployments, so every deployment must target + # the same region as the provider. Multi-region support can be added later by + # passing per-region provider configurations into each deployment module. + deployment_regions = distinct([for d in local.deployments : d.region]) + + precondition_regions_match = length(local.deployment_regions) == 1 && local.deployment_regions[0] == var.region +} + +resource "terraform_data" "validate_single_region" { + input = local.deployment_regions + + lifecycle { + precondition { + condition = local.precondition_regions_match + error_message = "All deployments must use the root provider region (${var.region}). Found regions: ${join(", ", local.deployment_regions)}." + } + } +} + +module "deployment" { + source = "./modules/deployment" + for_each = local.deployments + + deployment = each.value + tags = var.tags + + providers = { + aws = aws + } +} diff --git a/examples/big-peer-deployment/modules/deployment/eks.tf b/examples/big-peer-deployment/modules/deployment/eks.tf new file mode 100644 index 0000000..58baea7 --- /dev/null +++ b/examples/big-peer-deployment/modules/deployment/eks.tf @@ -0,0 +1,104 @@ +# Discover current partition/account for partition-safe ARNs. +data "aws_partition" "current" {} +data "aws_caller_identity" "current" {} + +# EKS cluster and managed node groups. The cluster control plane is created in +# the deployment's region and placed into the deployment's VPC subnets. +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "21.25.0" + + name = var.deployment.cluster.name + kubernetes_version = var.deployment.cluster.version + + vpc_id = local.vpc_id + subnet_ids = concat(local.private_subnet_ids, local.public_subnet_ids) + + endpoint_public_access = true + endpoint_private_access = true + + enable_irsa = true + + # Core add-ons required for a functional cluster. vpc-cni and kube-proxy must + # land before compute: without a CNI, nodes never reach Ready and the managed + # node group fails with NodeCreationFailure. coredns needs a node to schedule + # on, so it stays after. + addons = { + coredns = {} + kube-proxy = { + before_compute = true + } + vpc-cni = { + before_compute = true + } + # Pod Identity must exist before any addon that consumes it. + eks-pod-identity-agent = { + before_compute = true + } + # Strimzi/Kafka and Big Peer both need PVCs, and k8s >= 1.31 has no in-tree + # EBS provisioner, so without this every PVC stays Pending. + aws-ebs-csi-driver = { + pod_identity_association = [{ + role_arn = aws_iam_role.ebs_csi.arn + service_account = "ebs-csi-controller-sa" + }] + } + } + + eks_managed_node_groups = { + for name, ng in var.deployment.cluster.managed_node_groups : name => { + instance_types = ng.instance_types + min_size = ng.min_size + max_size = ng.max_size + desired_size = ng.desired_size + subnet_ids = local.private_subnet_ids + } + } + + # Admin access entries for human operators (e.g. SSO roles). The upstream + # module's cluster-creator permissions don't resolve for assumed-role + # sessions (SSO), so grant cluster admin explicitly. + access_entries = { + for idx, arn in var.deployment.cluster.admin_principal_arns : "admin_${idx}" => { + principal_arn = arn + policy_associations = { + admin = { + policy_arn = "arn:${data.aws_partition.current.partition}:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + access_scope = { + type = "cluster" + } + } + } + } + } + + # Explicit KMS key admins/owners prevent the upstream module from using + # aws_iam_session_context, which can resolve to an invalid assumed-role + # principal (e.g. an SSO session) and cause MalformedPolicyDocumentException. + kms_key_administrators = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root"] + kms_key_owners = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root"] + + tags = var.tags +} + +# Pod Identity role for the EBS CSI driver controller. +data "aws_iam_policy_document" "ebs_csi_trust" { + statement { + actions = ["sts:AssumeRole", "sts:TagSession"] + principals { + type = "Service" + identifiers = ["pods.eks.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "ebs_csi" { + name = "${var.deployment.cluster.name}-ebs-csi-driver" + assume_role_policy = data.aws_iam_policy_document.ebs_csi_trust.json + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "ebs_csi" { + role = aws_iam_role.ebs_csi.name + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEBSCSIDriverPolicyV2" +} diff --git a/examples/big-peer-deployment/modules/deployment/outputs.tf b/examples/big-peer-deployment/modules/deployment/outputs.tf new file mode 100644 index 0000000..c95b866 --- /dev/null +++ b/examples/big-peer-deployment/modules/deployment/outputs.tf @@ -0,0 +1,39 @@ +output "cluster_name" { + value = module.eks.cluster_name +} + +output "cluster_endpoint" { + value = module.eks.cluster_endpoint +} + +output "cluster_certificate_authority_data" { + value = module.eks.cluster_certificate_authority_data +} + +output "cluster_oidc_issuer_url" { + value = module.eks.cluster_oidc_issuer_url +} + +output "cluster_security_group_id" { + value = module.eks.cluster_security_group_id +} + +output "vpc_id" { + value = local.vpc_id +} + +output "private_subnet_ids" { + value = local.private_subnet_ids +} + +output "public_subnet_ids" { + value = local.public_subnet_ids +} + +output "node_group_names" { + value = keys(module.eks.eks_managed_node_groups) +} + +output "region" { + value = var.deployment.region +} diff --git a/examples/big-peer-deployment/modules/deployment/providers.tf b/examples/big-peer-deployment/modules/deployment/providers.tf new file mode 100644 index 0000000..66c2173 --- /dev/null +++ b/examples/big-peer-deployment/modules/deployment/providers.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + configuration_aliases = [aws] + } + } +} diff --git a/examples/big-peer-deployment/modules/deployment/variables.tf b/examples/big-peer-deployment/modules/deployment/variables.tf new file mode 100644 index 0000000..1ec949b --- /dev/null +++ b/examples/big-peer-deployment/modules/deployment/variables.tf @@ -0,0 +1,36 @@ +variable "deployment" { + description = "Single deployment definition from the YAML config." + type = object({ + region = string + vpc = object({ + create = bool + cidr = optional(string) + availability_zones = optional(list(string)) + private_subnets = optional(list(string)) + public_subnets = optional(list(string)) + vpc_id = optional(string) + private_subnet_ids = optional(list(string)) + public_subnet_ids = optional(list(string)) + }) + cluster = object({ + name = string + version = string + managed_node_groups = optional(map(any), {}) + admin_principal_arns = optional(list(string), []) + }) + }) + + # The upstream EKS module defaults enable_cluster_creator_admin_permissions + # to false, so without at least one admin ARN the access_entries map renders + # empty and no identity can reach the cluster API. Fail fast at plan. + validation { + condition = length(var.deployment.cluster.admin_principal_arns) > 0 + error_message = "cluster.admin_principal_arns must list at least one IAM role ARN (e.g. your SSO admin role). Without it, no identity is granted cluster access." + } +} + +variable "tags" { + description = "Tags applied to all resources." + type = map(string) + default = {} +} diff --git a/examples/big-peer-deployment/modules/deployment/vpc-endpoints.tf b/examples/big-peer-deployment/modules/deployment/vpc-endpoints.tf new file mode 100644 index 0000000..484d25e --- /dev/null +++ b/examples/big-peer-deployment/modules/deployment/vpc-endpoints.tf @@ -0,0 +1,66 @@ +# VPC endpoints so private subnets can reach AWS services without an internet path. +# Only created for managed VPCs; existing VPCs are expected to provide their own. +module "vpc_endpoints" { + source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" + version = "6.6.0" + count = var.deployment.vpc.create ? 1 : 0 + + vpc_id = local.vpc_id + + create_security_group = true + security_group_name_prefix = "${var.deployment.cluster.name}-vpc-endpoints-" + security_group_description = "VPC endpoint security group" + security_group_rules = { + ingress_https = { + description = "HTTPS from VPC" + cidr_blocks = [module.vpc[0].vpc_cidr_block] + } + } + + endpoints = { + s3 = { + service = "s3" + service_type = "Gateway" + route_table_ids = module.vpc[0].private_route_table_ids + private_dns_enabled = false + tags = { Name = "s3-vpc-endpoint" } + } + ecr_api = { + service = "ecr.api" + private_dns_enabled = true + subnet_ids = local.private_subnet_ids + tags = { Name = "ecr-api-endpoint" } + } + ecr_dkr = { + service = "ecr.dkr" + private_dns_enabled = true + subnet_ids = local.private_subnet_ids + tags = { Name = "ecr-dkr-endpoint" } + } + # SSM endpoints required for AL2023 node bootstrap and EKS node registration. + # Without these, nodes cannot register with the control plane (NodeCreationFailure). + ssm = { + service = "ssm" + private_dns_enabled = true + subnet_ids = local.private_subnet_ids + tags = { Name = "ssm-endpoint" } + } + ssmmessages = { + service = "ssmmessages" + private_dns_enabled = true + subnet_ids = local.private_subnet_ids + tags = { Name = "ssmmessages-endpoint" } + } + # STS endpoint required for AL2023 node bootstrap. Nodes use sts:GetCallerIdentity + # during kubelet startup to authenticate with EKS. Without this, nodes in private + # subnets cannot complete bootstrap (NodeCreationFailure). + sts = { + service = "sts" + private_dns_enabled = true + subnet_ids = local.private_subnet_ids + tags = { Name = "sts-endpoint" } + } + } + + tags = var.tags +} diff --git a/examples/big-peer-deployment/modules/deployment/vpc.tf b/examples/big-peer-deployment/modules/deployment/vpc.tf new file mode 100644 index 0000000..887eb86 --- /dev/null +++ b/examples/big-peer-deployment/modules/deployment/vpc.tf @@ -0,0 +1,54 @@ +locals { + azs = coalesce(var.deployment.vpc.availability_zones, [ + data.aws_availability_zones.available.names[0], + data.aws_availability_zones.available.names[1], + data.aws_availability_zones.available.names[2], + ]) +} + +data "aws_availability_zones" "available" { + state = "available" +} + +# Managed VPC: created only when vpc.create is true. +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "6.6.0" + count = var.deployment.vpc.create ? 1 : 0 + + name = "${var.deployment.cluster.name}-vpc" + cidr = var.deployment.vpc.cidr + + azs = local.azs + private_subnets = var.deployment.vpc.private_subnets + public_subnets = var.deployment.vpc.public_subnets + + enable_nat_gateway = true + single_nat_gateway = false + + enable_dns_hostnames = true + enable_dns_support = true + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = "1" + } + + public_subnet_tags = { + "kubernetes.io/role/elb" = "1" + } + + tags = var.tags +} + +# Existing VPC: subnet IDs are supplied directly. +data "aws_vpc" "existing" { + count = var.deployment.vpc.create ? 0 : 1 + id = var.deployment.vpc.vpc_id +} + +locals { + vpc_id = var.deployment.vpc.create ? module.vpc[0].vpc_id : data.aws_vpc.existing[0].id + + private_subnet_ids = var.deployment.vpc.create ? module.vpc[0].private_subnets : var.deployment.vpc.private_subnet_ids + public_subnet_ids = var.deployment.vpc.create ? module.vpc[0].public_subnets : var.deployment.vpc.public_subnet_ids +} diff --git a/examples/big-peer-deployment/outputs.tf b/examples/big-peer-deployment/outputs.tf new file mode 100644 index 0000000..42ab4f2 --- /dev/null +++ b/examples/big-peer-deployment/outputs.tf @@ -0,0 +1,14 @@ +output "deployments" { + description = "Summary of each deployment's cluster and VPC." + value = { + for name, dep in module.deployment : name => { + cluster_name = dep.cluster_name + cluster_endpoint = dep.cluster_endpoint + cluster_oidc_url = dep.cluster_oidc_issuer_url + vpc_id = dep.vpc_id + private_subnet_ids = dep.private_subnet_ids + public_subnet_ids = dep.public_subnet_ids + node_group_names = dep.node_group_names + } + } +} diff --git a/examples/big-peer-deployment/providers.tf b/examples/big-peer-deployment/providers.tf new file mode 100644 index 0000000..698ace8 --- /dev/null +++ b/examples/big-peer-deployment/providers.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + local = { + source = "hashicorp/local" + version = "~> 2.5" + } + } +} + +provider "aws" { + region = var.region +} diff --git a/examples/big-peer-deployment/scripts/app.sh b/examples/big-peer-deployment/scripts/app.sh new file mode 100755 index 0000000..4f9930f --- /dev/null +++ b/examples/big-peer-deployment/scripts/app.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Create an app and an API key through the operator's admin API (quickstart +# steps 8-10). +# +# The API is not exposed through the gateway -- it is an admin surface -- so we +# reach it with a short-lived port-forward. Credentials are written to +# .local/app.env, which is gitignored. They are never echoed in full. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +APP_NAME="${APP_NAME:-example-app}" +ENV_FILE="$LOCAL_DIR/app.env" + +port_forward "$NAMESPACE" deployments/ditto-operator 8080 +API="http://127.0.0.1:${PF_PORT}/namespace/${NAMESPACE}/bigPeer/${BIGPEER_NAME}" + +# Reuse the app if it already exists so the target is idempotent. +log "creating app '$APP_NAME'" +create=$(curl -s -H 'Content-Type: application/json' -X POST "${API}/app" \ + -d "{\"name\":\"${APP_NAME}\"}" || true) + +app_id=$(jq -r '.appId // .id // empty' <<<"$create" 2>/dev/null || true) + +if [[ -z "$app_id" ]]; then + log "create returned no appId; looking the app up instead" + app_id=$(curl -s "${API}/app" 2>/dev/null \ + | jq -r --arg n "$APP_NAME" '.. | objects | select(.name==$n) | (.appId // .id) // empty' \ + | head -1 || true) +fi + +[[ -n "$app_id" ]] || { + warn "operator response was: $create" + die "could not determine appId" +} +log "appId: $app_id" + +# An API key can only be read at creation time, and the operator rejects a +# duplicate key name -- so re-running with a fixed name fails. If we already +# hold a key for this app, keep it; otherwise mint one under a unique name. +if [[ -f "$ENV_FILE" ]] && ! force; then + existing_id=$(sed -n 's/^DITTO_APP_ID=//p' "$ENV_FILE" | head -1) + existing_key=$(sed -n 's/^DITTO_API_KEY=//p' "$ENV_FILE" | head -1) + if [[ "$existing_id" == "$app_id" && -n "$existing_key" ]]; then + skip "API key for $APP_NAME (in $ENV_FILE)" + log "appId ${app_id}, API key ${existing_key:0:6}... (truncated)" + exit 0 + fi +fi + +KEY_NAME="${KEY_NAME:-makefile-$(date +%Y%m%d%H%M%S)}" +log "generating API key '$KEY_NAME'" +key_json=$(curl -s -H 'Content-Type: application/json' \ + -X POST "${API}/app/${APP_NAME}/apiKey" \ + -d '{ + "name": "'"${KEY_NAME}"'", + "expiresAt": "2100-04-13T12:00:00Z", + "permissions": { + "remoteQuery": true, + "read": { "everything": true, "queriesByCollection": {} }, + "write": { "everything": true, "queriesByCollection": {} } + } + }') + +# The operator returns the key as a bare string, not JSON. Try JSON first in +# case that changes, then fall back to treating the body as the key itself. +api_key=$(jq -r '.apiKey // .key // .value // empty' <<<"$key_json" 2>/dev/null || true) +if [[ -z "$api_key" ]]; then + candidate=$(tr -d '[:space:]' <<<"$key_json") + # A bare token: no JSON punctuation, plausible length. + if [[ "$candidate" =~ ^[A-Za-z0-9._-]{20,}$ ]]; then + api_key="$candidate" + fi +fi + +[[ -n "$api_key" ]] || { + # Safe to show: we only reach here when the body did NOT look like a bare + # token, so it is an error payload rather than a credential. + warn "apiKey endpoint returned an unrecognised ${#key_json}-byte response:" + warn " $(head -c 300 <<<"$key_json")" + die "could not extract the API key" +} + +umask 077 +cat > "$ENV_FILE" </dev/null)" == "$BIGPEER_VERSION" ]] || return 1 + kc -n "$NAMESPACE" get pods -l "ditto.live/big-peer=${BIGPEER_NAME}" -o json 2>/dev/null \ + | jq -e '.items | length > 0 and (all(.[]; .status.conditions[]? | select(.type=="Ready") | .status == "True"))' \ + >/dev/null 2>&1 +} + +if already_running; then + skip "BigPeer/$BIGPEER_NAME v$BIGPEER_VERSION" + BIGPEER_READY=1 +fi + +# Big Peer defaults to an `ingress`-type external listener for Kafka, which +# only Data Bridges need. Strimzi then blocks its entire reconcile waiting for +# that Ingress to become addressable -- and with Envoy Gateway there is no +# Ingress controller to give it an address, so the Kafka broker is never +# created and the whole Big Peer stalls. Off unless explicitly requested. +if [[ "${KAFKA_EXTERNAL_LISTENER:-0}" == "1" ]]; then + kafka_listener="true" + warn "KAFKA_EXTERNAL_LISTENER=1 requires an Ingress controller (Envoy Gateway" + warn "does not serve Ingress). Kafka will not start without one." +else + kafka_listener="false" +fi + +log "applying BigPeer/$BIGPEER_NAME (version $BIGPEER_VERSION)" +kc apply -f - </dev/null || true) + if [[ "$kafka_msg" == *"Ingress"* ]]; then + warn " Kafka is blocked on an Ingress that nothing serves:" + warn " $kafka_msg" + warn " Envoy Gateway does not implement the Ingress API. Set" + warn " KAFKA_EXTERNAL_LISTENER=0 (the default) and re-run 'make bigpeer'," + warn " or install an Ingress controller if you need Kafka Data Bridges." + elif [[ -n "$kafka_msg" ]]; then + warn " Kafka not ready: $kafka_msg" + fi + + if kc -n "$NAMESPACE" get pods 2>/dev/null | grep -q Pending; then + warn " Pods Pending -> check node capacity; there is no autoscaler." + fi + warn " PVCs Pending -> check the gp3 StorageClass exists (make storage)." + + kc -n "$NAMESPACE" get pods,pvc || true + kc -n "$NAMESPACE" get kafka,kafkanodepool 2>/dev/null || true + exit 1 + } +fi + +# Backend services created by the operator, per cloud-infra-apps: +# ditto--api :8080 HTTP API +# ditto--auth-server :7070 auth +# ditto--subscription :7070 sync / WebSocket +SVC="ditto-${BIGPEER_NAME}" + +log "applying HTTPRoutes" +# One route with three rules. Gateway API resolves ties by longest path prefix, +# so /_ditto/auth beats /_ditto, which beats /. Splitting these across separate +# HTTPRoutes would make precedence depend on creation order instead. +for gw in $(kc -n envoy-gateway-system get gateway -o name 2>/dev/null | cut -d/ -f2); do + log " -> gateway $gw" + kc apply -f - <&2; exit 1; } + +# YAML has no portable CLI parser, so try what is most likely to be present. +# Order: yq (purpose-built) -> python+PyYAML -> ruby (ships with macOS). +emit() { + if command -v yq >/dev/null 2>&1; then + yq -r --arg want "$WANT" ' + .deployments as $d + | (if $want != "" then $want else ($d | keys | .[0]) end) as $name + | $d[$name] as $dep + | "DEPLOYMENT=\($name)", + "CLUSTER=\($dep.cluster.name)", + "REGION=\($dep.region)" + ' "$CONFIG" + return + fi + + if python3 -c 'import yaml' >/dev/null 2>&1; then + CONFIG="$CONFIG" WANT="$WANT" python3 - <<'PY' +import os, yaml +cfg = yaml.safe_load(open(os.environ["CONFIG"])) +deps = cfg["deployments"] +name = os.environ.get("WANT") or next(iter(deps)) +d = deps[name] +print(f"DEPLOYMENT={name}") +print(f"CLUSTER={d['cluster']['name']}") +print(f"REGION={d['region']}") +PY + return + fi + + if command -v ruby >/dev/null 2>&1 && ruby -ryaml -e '' >/dev/null 2>&1; then + CONFIG="$CONFIG" WANT="$WANT" ruby -ryaml -e ' + cfg = YAML.load_file(ENV["CONFIG"]) + deps = cfg["deployments"] + name = (ENV["WANT"].to_s.empty? ? deps.keys.first : ENV["WANT"]) + d = deps.fetch(name) { abort "deployment #{name} not found in config.yaml" } + puts "DEPLOYMENT=#{name}" + puts "CLUSTER=#{d["cluster"]["name"]}" + puts "REGION=#{d["region"]}" + ' + return + fi + + cat >&2 <<'MSG' +No YAML parser available. Install one of: + brew install yq (recommended) + pip3 install pyyaml +...or run `terraform apply` first, which generates .env directly. +MSG + exit 1 +} + +echo "# Generated by scripts/config.sh from config.yaml." +echo "# \`terraform apply\` overwrites this with the values it actually built." +emit diff --git a/examples/big-peer-deployment/scripts/ditto-operator.sh b/examples/big-peer-deployment/scripts/ditto-operator.sh new file mode 100755 index 0000000..c69cb53 --- /dev/null +++ b/examples/big-peer-deployment/scripts/ditto-operator.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Ditto Operator. Public OCI chart -- no pull secret required. +# +# PORTAL=1 additionally enables the self-managed portal UI, which ships in this +# same chart (portal.enabled). Its own Ingress templates stay disabled: Envoy +# Gateway does not serve Ingress, so scripts/portal.sh authors an HTTPRoute +# instead. Nothing here creates a load balancer. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +portal_wanted() { [[ "${PORTAL:-0}" == "1" ]]; } + +# Skip only when the installed state matches the requested one, so toggling +# PORTAL always takes effect (same approach as EXTERNAL in envoy-gateway.sh). +portal_installed() { kc -n "$NAMESPACE" get deploy ditto-operator-portal >/dev/null 2>&1; } +if release_healthy ditto-operator "$NAMESPACE" "app.kubernetes.io/name=ditto-operator"; then + if portal_wanted && portal_installed; then + skip "ditto-operator $DITTO_OPERATOR_VERSION (with portal)"; exit 0 + elif ! portal_wanted && ! portal_installed; then + skip "ditto-operator $DITTO_OPERATOR_VERSION"; exit 0 + fi + log "PORTAL=${PORTAL:-0} differs from what is installed; reconciling" +fi + +args=(--version "$DITTO_OPERATOR_VERSION") + +if portal_wanted; then + log "portal enabled (host $PORTAL_HOST, image tag $PORTAL_VERSION)" + # Same-origin URLs: both go through the one HTTPRoute portal.sh creates, so + # the browser never makes a cross-origin request and no CORS policy is needed. + args+=( + --set portal.enabled=true + --set portal.image.tag="$PORTAL_VERSION" + --set portal.config.bigPeerName="$BIGPEER_NAME" + --set portal.config.operatorApiUrl="http://${PORTAL_HOST}/operator-api" + --set portal.config.bigPeerBaseUrl="http://${PORTAL_HOST}/big-peer" + --set portal.ingress.enabled=false + --set portal.operatorApiIngress.enabled=false + ) +fi + +helm_install ditto-operator oci://quay.io/ditto-external/ditto-operator "$NAMESPACE" "${args[@]}" + +wait_pods "$NAMESPACE" "app.kubernetes.io/name=ditto-operator" 300 \ + || wait_pods "$NAMESPACE" "app=ditto-operator" 300 + +wait_crd bigpeers.ditto.live + +log "ditto operator ready" +kc api-resources --api-group=ditto.live 2>/dev/null || true diff --git a/examples/big-peer-deployment/scripts/envoy-gateway.sh b/examples/big-peer-deployment/scripts/envoy-gateway.sh new file mode 100755 index 0000000..4f2c910 --- /dev/null +++ b/examples/big-peer-deployment/scripts/envoy-gateway.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Envoy Gateway, shaped after cloud-infra-apps but sized for a test cluster. +# +# cloud-infra runs three proxies (ext/int/str) with 3 replicas, an HPA, +# zone topology spread, a dedicated `app.ditto.live/tier: infrastructure` +# node pool and TLS terminated at the NLB via ACM. None of that applies here: +# this cluster has one unlabelled node group and no ACM certificate. We keep +# the CR *shape* (EnvoyProxy -> GatewayClass -> Gateway) and drop the rest. +# +# The internal gateway is always created. The internet-facing one only when +# EXTERNAL=1, because without a certificate it would serve plaintext HTTP. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +EG_NS=envoy-gateway-system +REPLICAS="${ENVOY_REPLICAS:-1}" + +# Skip only when the installed state already matches the requested one. A change +# to EXTERNAL must still take effect, so check that the external gateway's +# presence agrees with the flag before bailing out. +ext_present() { kc -n "$EG_NS" get gateway ext-envoy-proxy >/dev/null 2>&1; } +if release_healthy envoy-gateway "$EG_NS" "app.kubernetes.io/name=gateway-helm" \ + && kc -n "$EG_NS" get gateway int-envoy-proxy >/dev/null 2>&1; then + if external_enabled && ext_present; then + skip "envoy gateway $ENVOY_GATEWAY_VERSION (internal + external)"; exit 0 + elif ! external_enabled && ! ext_present; then + skip "envoy gateway $ENVOY_GATEWAY_VERSION (internal)"; exit 0 + fi + log "EXTERNAL=$EXTERNAL differs from what is installed; reconciling" +fi + +helm_install envoy-gateway oci://docker.io/envoyproxy/gateway-helm "$EG_NS" \ + --version "$ENVOY_GATEWAY_VERSION" \ + --set config.envoyGateway.gateway.controllerName=gateway.envoyproxy.io/gatewayclass-controller \ + --set config.envoyGateway.provider.type=Kubernetes \ + --set config.envoyGateway.logging.level.default=info + +wait_pods "$EG_NS" "app.kubernetes.io/name=gateway-helm" 300 +wait_crd gatewayclasses.gateway.networking.k8s.io +wait_crd envoyproxies.gateway.envoyproxy.io + +# Emit one EnvoyProxy + GatewayClass + Gateway triple. +# $1 = name (int-envoy-proxy | ext-envoy-proxy) +# $2 = scheme (internal | internet-facing) +render_gateway() { + local name="$1" scheme="$2" internal_annotation="" + + # Legacy cloud-controller-manager annotations. The AWS Load Balancer + # Controller is not installed on this cluster, so we use the annotations + # EKS honours natively. + if [[ "$scheme" == "internal" ]]; then + internal_annotation='service.beta.kubernetes.io/aws-load-balancer-internal: "true"' + fi + + cat </dev/null 2>&1 || true + kc delete gatewayclass ext-envoy-proxy --ignore-not-found >/dev/null 2>&1 || true + kc delete envoyproxy ext-envoy-proxy -n "$EG_NS" --ignore-not-found >/dev/null 2>&1 || true +fi + +log "waiting for gateways to be programmed" +kc -n "$EG_NS" wait --for=condition=Programmed gateway/int-envoy-proxy --timeout=300s || { + kc -n "$EG_NS" describe gateway int-envoy-proxy || true + die "internal gateway never became Programmed" +} +external_enabled && kc -n "$EG_NS" wait --for=condition=Programmed gateway/ext-envoy-proxy --timeout=300s || true + +log "envoy gateway ready" +kc -n "$EG_NS" get gateway diff --git a/examples/big-peer-deployment/scripts/lib.sh b/examples/big-peer-deployment/scripts/lib.sh new file mode 100755 index 0000000..24f628b --- /dev/null +++ b/examples/big-peer-deployment/scripts/lib.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Shared helpers. Sourced by every script in this directory. + +set -euo pipefail + +: "${CLUSTER:?}" "${REGION:?}" "${NAMESPACE:?}" + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(dirname "$HERE")" +LOCAL_DIR="$ROOT/.local" + +# Self-contained kubeconfig in the example directory. Nothing here touches +# ~/.kube/config, so running this example cannot disturb whatever cluster the +# operator happens to have selected globally. +export KUBECONFIG="${KUBECONFIG:-$ROOT/.kube/config}" +KUBE_CONTEXT="${KUBE_CONTEXT:-$CLUSTER}" + +kc() { kubectl --context "$KUBE_CONTEXT" "$@"; } +helmc() { helm --kube-context "$KUBE_CONTEXT" "$@"; } + +log() { printf '\033[36m==>\033[0m %s\n' "$*"; } +skip() { printf '\033[32m -->\033[0m %s \033[2m(already installed, skipping)\033[0m\n' "$*"; } +warn() { printf '\033[33mWARN\033[0m %s\n' "$*" >&2; } +die() { printf '\033[31mERROR\033[0m %s\n' "$*" >&2; exit 1; } + +need() { + command -v "$1" >/dev/null 2>&1 || die "$1 is required but not installed" +} + +# --- idempotency ----------------------------------------------------------- +# Every target is safe to re-run, but re-running a healthy Helm release still +# costs a slow --wait. These let a target bail out early. FORCE=1 overrides. + +force() { [[ "${FORCE:-0}" == "1" ]]; } + +# True when a Helm release is deployed AND its pods are Ready. +release_healthy() { + local name="$1" ns="$2" selector="${3:-}" + force && return 1 + [[ "$(helmc status "$name" -n "$ns" -o json 2>/dev/null | jq -r '.info.status' 2>/dev/null)" == "deployed" ]] || return 1 + [[ -z "$selector" ]] && return 0 + kc -n "$ns" get pods -l "$selector" -o json 2>/dev/null \ + | jq -e '.items | length > 0 and (all(.[]; .status.conditions[]? | select(.type=="Ready") | .status == "True"))' \ + >/dev/null 2>&1 +} + +# True when a cluster-scoped object already exists. +object_exists() { + force && return 1 + kc get "$@" >/dev/null 2>&1 +} + +# --- waits ----------------------------------------------------------------- + +wait_crd() { + local crd="$1" timeout="${2:-120s}" + log "waiting for CRD $crd" + kc wait --for condition=established --timeout="$timeout" "crd/$crd" +} + +# Wait for pods matching a selector to be Ready, tolerating the window before +# they exist at all -- which `kubectl wait` alone does not. +wait_pods() { + local ns="$1" selector="$2" timeout="${3:-300}" + local deadline=$(( SECONDS + timeout )) + log "waiting for pods in $ns matching $selector" + while (( SECONDS < deadline )); do + if kc -n "$ns" get pods -l "$selector" -o name 2>/dev/null | grep -q .; then + if kc -n "$ns" wait --for=condition=Ready pod -l "$selector" --timeout=30s >/dev/null 2>&1; then + return 0 + fi + fi + sleep 5 + done + warn "timed out waiting for $selector in $ns; current state:" + kc -n "$ns" get pods -l "$selector" || true + return 1 +} + +ensure_ns() { + kc get namespace "$1" >/dev/null 2>&1 || kc create namespace "$1" +} + +helm_install() { + local name="$1" chart="$2" ns="$3"; shift 3 + log "helm upgrade --install $name ($chart) -> $ns" + helmc upgrade --install "$name" "$chart" \ + --namespace "$ns" --create-namespace \ + --wait --timeout 10m "$@" +} + +external_enabled() { [[ "${EXTERNAL:-0}" == "1" ]]; } + +mkdir -p "$LOCAL_DIR" + +# --- port-forward ---------------------------------------------------------- +# Hardcoding a port collides with anything already listening -- including a +# port-forward leaked by an earlier failed run. Pick a free one, and make +# cleanup reliable so we do not leak one ourselves. + +PF_PID="" +PF_PORT="" + +free_port() { + local p + for p in $(seq "${1:-18080}" "${2:-18180}"); do + if ! (exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null; then echo "$p"; return 0; fi + exec 3<&- 2>/dev/null || true + done + die "no free local port in ${1:-18080}-${2:-18180}" +} + +pf_stop() { + [[ -z "$PF_PID" ]] && return 0 + kill "$PF_PID" 2>/dev/null || true + # kubectl can spawn children; take those too. + pkill -P "$PF_PID" 2>/dev/null || true + PF_PID="" +} + +# port_forward -> sets PF_PORT +port_forward() { + local ns="$1" target="$2" remote="$3" + PF_PORT="$(free_port)" + log "port-forwarding $target on :$PF_PORT" + kc -n "$ns" port-forward "$target" "${PF_PORT}:${remote}" >/dev/null 2>&1 & + PF_PID=$! + # Detach from job control so killing it does not print "Terminated: 15". + disown "$PF_PID" 2>/dev/null || true + trap pf_stop EXIT + + local deadline=$(( SECONDS + 30 )) + while (( SECONDS < deadline )); do + kill -0 "$PF_PID" 2>/dev/null || die "port-forward to $target exited immediately" + # No -f: the API root legitimately returns 404, which still proves the + # tunnel is up. We only care that *something* answers. + if curl -s -o /dev/null --max-time 2 "http://127.0.0.1:${PF_PORT}/" 2>/dev/null; then + return 0 + fi + sleep 1 + done + die "port-forward to $target never became reachable" +} diff --git a/examples/big-peer-deployment/scripts/portal.sh b/examples/big-peer-deployment/scripts/portal.sh new file mode 100755 index 0000000..834b7cc --- /dev/null +++ b/examples/big-peer-deployment/scripts/portal.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Route the self-managed portal through the existing internal gateway. +# +# The portal itself is part of the ditto-operator chart (portal.enabled), so +# ditto-operator.sh turns it on. This script only handles the networking. +# +# The chart's own portal.ingress / portal.operatorApiIngress render `Ingress` +# objects with class nginx, which Envoy Gateway does not serve -- both are left +# disabled and we author an HTTPRoute instead. +# +# Everything is served from ONE hostname so the browser sees a single origin: +# the chart's default topology puts the portal and the operator API on separate +# hosts, which would need a CORS policy on the API. Same-origin avoids that. +# +# http://$PORTAL_HOST/ -> portal UI +# http://$PORTAL_HOST/operator-api/ -> operator admin API (prefix stripped) +# http://$PORTAL_HOST/big-peer/ -> Big Peer HTTP API (prefix stripped) +# +# No new load balancer: this attaches to int-envoy-proxy, which is an internal +# NLB. There is no public exposure. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +EG_NS=envoy-gateway-system +ROUTE="ditto-portal" +GW=int-envoy-proxy + +if [[ "${PORTAL:-0}" != "1" ]]; then + log "PORTAL=0; removing portal route if present" + kc -n "$NAMESPACE" delete httproute "$ROUTE" --ignore-not-found >/dev/null 2>&1 || true + exit 0 +fi + +kc -n "$EG_NS" get gateway "$GW" >/dev/null 2>&1 \ + || die "gateway $GW not found -- run 'make envoy' first" + +kc -n "$NAMESPACE" get deploy ditto-operator-portal >/dev/null 2>&1 \ + || die "portal deployment not found -- run 'PORTAL=1 make operator' first" + +log "applying portal HTTPRoute for host $PORTAL_HOST" +# Longest path prefix wins, so /operator-api and /big-peer beat the portal's +# catch-all. The portal is an SPA and needs to keep serving / itself. +kc apply -f - </dev/null 2>&1 \ + || die "cannot reach cluster via context '$KUBE_CONTEXT'. Run: make kubeconfig" + +# Nodes must be Ready. A NotReady node here almost always means the CNI add-on +# never landed -- the exact failure this example's Terraform now guards against. +ready=$(kc get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l | tr -d ' ') +total=$(kc get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ') +log "nodes Ready: $ready/$total" +(( ready > 0 )) || { + kc get nodes -o wide || true + die "no Ready nodes. If nodes are NotReady, check that the vpc-cni add-on is installed." +} + +# The full stack does not fit on a single m8i.large. Warn rather than fail so +# a deliberately small run can proceed. +if (( ready < 3 )); then + warn "only $ready Ready node(s). cert-manager + Envoy + Kafka + Big Peer will not" + warn "fit comfortably below 3 nodes. Consider desired_size: 3 in config.yaml." +fi + +# The EBS CSI driver is a hard dependency: Strimzi and Big Peer both need PVCs. +if ! kc get csidriver ebs.csi.aws.com >/dev/null 2>&1; then + die "EBS CSI driver not found. Run 'terraform apply' first -- aws-ebs-csi-driver + is declared in modules/deployment/eks.tf. Without it every PVC stays Pending." +fi +log "EBS CSI driver present" + +log "preflight OK" diff --git a/examples/big-peer-deployment/scripts/smoke.sh b/examples/big-peer-deployment/scripts/smoke.sh new file mode 100755 index 0000000..0c2fd12 --- /dev/null +++ b/examples/big-peer-deployment/scripts/smoke.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Quickstart steps 11-12: insert a document, then read it back. +# +# Routes through the internal gateway via a port-forward to the Envoy service, +# so this works whether or not EXTERNAL=1. That also exercises the HTTPRoutes +# rather than talking to the Big Peer service directly. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +ENV_FILE="$LOCAL_DIR/app.env" +[[ -f "$ENV_FILE" ]] || die "missing $ENV_FILE -- run 'make app' first" +# shellcheck disable=SC1090 +source "$ENV_FILE" + +EG_NS=envoy-gateway-system + +svc=$(kc -n "$EG_NS" get svc -l gateway.envoyproxy.io/owning-gateway-name=int-envoy-proxy \ + -o name 2>/dev/null | head -1) +[[ -n "$svc" ]] || die "could not find the Envoy service for int-envoy-proxy" + +port_forward "$EG_NS" "$svc" 80 +BASE="http://127.0.0.1:${PF_PORT}/${DITTO_APP_ID}/api/v4/store/execute" + +log "INSERT" +insert=$(curl -s -X POST "$BASE" \ + --header "Authorization: bearer ${DITTO_API_KEY}" \ + --header 'Content-Type: application/json' \ + --data-raw '{"statement":"INSERT INTO cars DOCUMENTS (:doc1)","args":{"doc1":{"_id":{"id":"777","locationId":"123456"},"color":"blue","timestamp":"1732192529"}}}') +echo "$insert" | jq . 2>/dev/null || echo "$insert" + +log "SELECT" +select=$(curl -s -X POST "$BASE" \ + --header "Authorization: bearer ${DITTO_API_KEY}" \ + --header 'Content-Type: application/json' \ + --data-raw '{"statement":"SELECT * FROM cars"}') +echo "$select" | jq . 2>/dev/null || echo "$select" + +if grep -q '"color"' <<<"$select"; then + log "smoke test PASSED" +else + die "smoke test FAILED -- no document returned" +fi diff --git a/examples/big-peer-deployment/scripts/status.sh b/examples/big-peer-deployment/scripts/status.sh new file mode 100755 index 0000000..630fff9 --- /dev/null +++ b/examples/big-peer-deployment/scripts/status.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Show what is installed and where to reach it. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +EG_NS=envoy-gateway-system + +endpoints() { + log "gateway endpoints" + for gw in int-envoy-proxy ext-envoy-proxy; do + kc -n "$EG_NS" get gateway "$gw" >/dev/null 2>&1 || continue + addr=$(kc -n "$EG_NS" get gateway "$gw" -o jsonpath='{.status.addresses[0].value}' 2>/dev/null) + printf ' %-18s %s\n' "$gw" "${addr:-}" + done + echo + echo " Nothing is published in DNS. Reach the internal gateway from inside the" + echo " VPC, or locally with:" + echo " kubectl --context $KUBE_CONTEXT -n $EG_NS port-forward \\" + echo " svc/ 18080:80" +} + +if [[ "${1:-}" == "endpoints" ]]; then + endpoints + exit 0 +fi + +log "nodes"; kc get nodes -o wide 2>/dev/null || true +echo +log "storage"; kc get storageclass 2>/dev/null || true +echo +log "helm releases" +helmc list --all-namespaces 2>/dev/null || true +echo +log "gateways"; kc -n "$EG_NS" get gateway 2>/dev/null || true +echo +log "ditto namespace" +kc -n "$NAMESPACE" get bigpeer,pods,pvc,httproute 2>/dev/null || true +echo +endpoints diff --git a/examples/big-peer-deployment/scripts/storage.sh b/examples/big-peer-deployment/scripts/storage.sh new file mode 100755 index 0000000..170c639 --- /dev/null +++ b/examples/big-peer-deployment/scripts/storage.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# EKS ships a legacy `gp2` StorageClass whose in-tree provisioner no longer +# exists in Kubernetes >= 1.31. Install a gp3 class backed by the EBS CSI +# driver and make it the cluster default. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if object_exists storageclass gp3; then + skip "gp3 StorageClass" + exit 0 +fi + +log "installing gp3 StorageClass" + +# Drop the default annotation from any other class first; two defaults is an +# error state and PVCs without an explicit class will fail to bind. +for sc in $(kc get storageclass -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null); do + [[ "$sc" == "gp3" ]] && continue + if [[ "$(kc get storageclass "$sc" -o jsonpath='{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}' 2>/dev/null)" == "true" ]]; then + log "removing default flag from StorageClass/$sc" + kc patch storageclass "$sc" -p \ + '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}' + fi +done + +kc apply -f - <<'YAML' +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gp3 + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: ebs.csi.aws.com +# WaitForFirstConsumer keeps the volume in the same AZ as the pod that claims +# it. With Immediate, a volume can land in an AZ with no schedulable node. +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +reclaimPolicy: Delete +parameters: + type: gp3 + encrypted: "true" +YAML + +log "storage ready" +kc get storageclass diff --git a/examples/big-peer-deployment/scripts/strimzi.sh b/examples/big-peer-deployment/scripts/strimzi.sh new file mode 100755 index 0000000..45bb835 --- /dev/null +++ b/examples/big-peer-deployment/scripts/strimzi.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Strimzi Kafka operator. Big Peer's CDC pipeline runs on Kafka, and the Ditto +# Operator expects Strimzi's CRDs to already exist. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +if release_healthy strimzi kafka "name=strimzi-cluster-operator"; then + skip "strimzi $STRIMZI_VERSION" + exit 0 +fi + +helm_install strimzi strimzi-kafka-operator kafka \ + --repo https://strimzi.io/charts \ + --version "$STRIMZI_VERSION" \ + --set watchAnyNamespace=true + +wait_pods kafka "name=strimzi-cluster-operator" 300 + +# The Ditto Operator creates Kafka CRs; without these registered it fails to +# reconcile a BigPeer. +wait_crd kafkas.kafka.strimzi.io +wait_crd kafkatopics.kafka.strimzi.io + +log "strimzi ready" diff --git a/examples/big-peer-deployment/scripts/teardown.sh b/examples/big-peer-deployment/scripts/teardown.sh new file mode 100755 index 0000000..ffee7a6 --- /dev/null +++ b/examples/big-peer-deployment/scripts/teardown.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Remove everything this Makefile installed. Terraform-managed resources (the +# cluster, node groups, EKS add-ons) are left alone -- use `terraform destroy` +# for those. +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +EG_NS=envoy-gateway-system + +cat </dev/null || true +helmc uninstall strimzi -n kafka 2>/dev/null || true + +log "deleting gateway resources" +for n in int-envoy-proxy ext-envoy-proxy; do + kc -n "$EG_NS" delete gateway "$n" --ignore-not-found || true + kc delete gatewayclass "$n" --ignore-not-found || true + kc -n "$EG_NS" delete envoyproxy "$n" --ignore-not-found || true +done +helmc uninstall envoy-gateway -n "$EG_NS" 2>/dev/null || true + +log "uninstalling cert-manager" +kc delete clusterissuer ditto-ca selfsigned-bootstrap --ignore-not-found || true +kc -n cert-manager delete certificate ditto-root-ca --ignore-not-found || true +helmc uninstall cert-manager -n cert-manager 2>/dev/null || true + +log "deleting gp3 StorageClass" +kc delete storageclass gp3 --ignore-not-found || true + +log "checking for leftover PVCs" +kc -n "$NAMESPACE" get pvc 2>/dev/null || true + +log "teardown complete. Local credentials remain at $LOCAL_DIR/app.env" diff --git a/examples/big-peer-deployment/variables.tf b/examples/big-peer-deployment/variables.tf new file mode 100644 index 0000000..19e9634 --- /dev/null +++ b/examples/big-peer-deployment/variables.tf @@ -0,0 +1,26 @@ +variable "config_path" { + description = "Path to the YAML file describing deployments." + type = string + default = "config.yaml" +} + +variable "region" { + description = "Default AWS region used by the root provider (overridden per deployment)." + type = string + default = "us-east-1" +} + +variable "tags" { + description = "Tags applied to all resources." + type = map(string) + default = { + GithubRepo = "dittocloud" + GithubOrg = "getditto" + } +} + +variable "primary_deployment" { + description = "Which deployment the generated .env describes. Optional when only one is defined." + type = string + default = null +}