Merge pull request #928 from Nordix/lentzi90/prow-kube-prometheus

Prow monitoring through kube-prometheus

Author: metal3-io-bot

.cspell-config.json

@@ -1,9 +1,7 @@
 {
   "version": "0.2",
   "language": "en",
-  "ignorePaths": [
-    "**/.github/workflows/**"
-  ],
+  "ignorePaths": ["**/.github/workflows/**"],
   "patterns": [
     {
       "name": "multiline-code-block",
@@ -14,10 +12,7 @@
       "pattern": "/`[^`].*`[^`]/g"
     }
   ],
-  "ignoreRegExpList": [
-    "multiline-code-block",
-    "inline-code"
-  ],
+  "ignoreRegExpList": ["multiline-code-block", "inline-code"],
   "words": [
     "autoscaler",
     "baremetal",
@@ -35,6 +30,7 @@
     "ghprb",
     "HMAC",
     "ipam",
+    "jsonnet",
     "keypair",
     "kubeadm",
     "kubeadmcontrolplane",

.gitignore

@@ -3,3 +3,12 @@
 *.tmp
 .DS_Store
 *.swp
+
+# Development containers
+.devcontainer
+
+# Zed
+.zed
+.zed_server
+
+vendor

prow/README.md

@@ -111,6 +111,10 @@ avoid deleting it even if the Service is deleted. See
 [cert-manager](https://cert-manager.io/) and the Let's Encrypt HTTP01 challenge,
 as seen in [infra/cluster-issuer-http.yaml](infra/cluster-issuer-http.yaml).
 
+#### Monitoring
+
+Please see [infra/kube-prometheus](infra/kube-prometheus).
+
 ## Building node images
 
 The Kubernetes cluster where Prow runs needs pre-built images for the Nodes. We

prow/capo-cluster/openstackcluster.yaml

@@ -32,7 +32,7 @@ spec:
       remoteManagedGroups:
       - controlplane
       - worker
-    allowAllInClusterTraffic: false
+    allowAllInClusterTraffic: true
   managedSubnets:
   - cidr: 10.6.0.0/24
     dnsNameservers:

prow/infra/kube-prometheus/Makefile

@@ -0,0 +1,11 @@
+export CONTAINER_RUNTIME ?= docker
+
+.phony: build
+
+build:
+	$(CONTAINER_RUNTIME) run --rm \
+		--volume "${PWD}:/workdir:rw,z" \
+		--workdir /workdir \
+		--entrypoint /workdir/build.sh \
+		docker.io/golang:1.23 \
+		metal3-kube-prometheus.jsonnet

prow/infra/kube-prometheus/README.md

@@ -0,0 +1,33 @@
+# Kube-prometheus for Metal3 Prow
+
+This monitoring stack is based on
+[kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/tree/main).
+We also took inspiration from how [k8s.io is monitoring
+ProwJobs](https://github.com/kubernetes/k8s.io/pull/5355).
+
+This is how you apply it in the cluster:
+
+```bash
+kubectl apply -f manifests/setup
+kubectl apply -f manifests
+kubectl apply -f prow-rules.yaml
+```
+
+The `manifests` are rendered using jsonnet based on
+`metal3-kube-prometheus.jsonnet`. Use the build script to render them after
+making changes:
+
+```bash
+make build
+```
+
+## How to access?
+
+For now, we have not exposed grafana or any other component. You can access them
+by using port-forward like this (after setting up access to the cluster itself):
+
+```bash
+kubectl -n monitoring port-forward svc/grafana 3000
+```
+
+Then go to <localhost:3000>.

prow/infra/kube-prometheus/build.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Based on https://github.com/prometheus-operator/kube-prometheus/blob/17aa6690a5739183c68efa048439739dce773827/build.sh
+# This script uses arg $1 (name of *.jsonnet file to use) to generate the manifests/*.yaml files.
+
+set -e
+set -x
+# only exit with zero if all commands of the pipeline exit successfully
+set -o pipefail
+
+# Make sure to use project tooling
+PATH="$(pwd)/tmp/bin:${PATH}"
+
+# Install needed tools
+go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/[email protected]
+go install github.com/google/go-jsonnet/cmd/[email protected]
+go install github.com/brancz/[email protected]
+
+
+# Make sure to start with a clean 'manifests' dir
+rm -rf manifests
+mkdir -p manifests/setup
+
+# Calling gojsontoyaml is optional, but we would like to generate yaml, not json
+jsonnet -J vendor -m manifests "${1-example.jsonnet}" | xargs -I{} sh -c 'cat {} | gojsontoyaml > {}.yaml' -- {}
+
+# Make sure to remove json files
+find manifests -type f ! -name '*.yaml' -delete
+rm -f kustomization