Test for operator custom metrics (#6025)

Author: assafad

hack/generate/samples/internal/go/memcached-with-webhooks/e2e_test_code.go

@@ -142,9 +142,13 @@ limitations under the License.
 package e2e
 
 import (
+	"bufio"
+	"encoding/json"
 	"fmt"
+	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -159,77 +163,105 @@ import (
 	"github.com/example/memcached-operator/test/utils"
 )
 
-// namespace store the ns where the Operator and Operand will be executed
-const namespace = "memcached-operator-system"
-
-var _ = Describe("memcached", func() {
-
-	Context("ensure that Operator and Operand(s) can run in restricted namespaces", func() {
-		BeforeEach(func() {
-			// The prometheus and the certmanager are installed in this test
-			// because the Memcached sample has this option enable and 
-            // when we try to apply the manifests both will be required to be installed
-			By("installing prometheus operator")
-			Expect(utils.InstallPrometheusOperator()).To(Succeed())
-
-			By("installing the cert-manager")
-			Expect(utils.InstallCertManager()).To(Succeed())
-
-			// The namespace can be created when we run make install
-			// However, in this test we want to ensure that the solution 
-			// can run in a ns labeled as restricted. Therefore, we are
-            // creating the namespace and labeling it. 
-			By("creating manager namespace")
-			cmd := exec.Command("kubectl", "create", "ns", namespace)
-			_, _ = utils.Run(cmd)
-
-			// Now, let's ensure that all namespaces can raise a Warn when we apply the manifests
-            // and that the namespace where the Operator and Operand will run are enforced as 
-            // restricted so that we can ensure that both can be admitted and run with the enforcement
-			By("labeling all namespaces to warn when we apply the manifest if it would violate the PodStandards")
-			cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all",
-				"pod-security.kubernetes.io/audit=restricted",
-				"pod-security.kubernetes.io/enforce-version=v1.24",
-				"pod-security.kubernetes.io/warn=restricted")
-			_, err := utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+// constant parts of the file
+const (
+	namespace                                      = "memcached-operator-system"
+	memcachedDeploymentSizeUndesiredCountTotalName = "memcached_deployment_size_undesired_count_total"
+	tokenRequestRawString                          = "{\"apiVersion\": \"authentication.k8s.io/v1\", \"kind\": \"TokenRequest\"}"
+)
 
-			By("labeling enforce the namespace where the Operator and Operand(s) will run")
-			cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace,
-				"pod-security.kubernetes.io/audit=restricted",
-				"pod-security.kubernetes.io/enforce-version=v1.24",
-				"pod-security.kubernetes.io/enforce=restricted")
-			_, err = utils.Run(cmd)
-			Expect(err).To(Not(HaveOccurred()))
-		})
+// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type
+// that we want to use for extracting the token.
+type tokenRequest struct {
+	Status struct {
+		Token string "json:\"token\""
+	} "json:\"status\""
+}
 
-		AfterEach(func() {
-			By("uninstalling the Prometheus manager bundle")
-			utils.UninstallPrometheusOperator()
+var _ = Describe("memcached", Ordered, func() {
+	BeforeAll(func() {
+		// The prometheus and the certmanager are installed in this test
+		// because the Memcached sample has this option enable and
+		// when we try to apply the manifests both will be required to be installed
+		By("installing prometheus operator")
+		Expect(utils.InstallPrometheusOperator()).To(Succeed())
+
+		By("installing the cert-manager")
+		Expect(utils.InstallCertManager()).To(Succeed())
+
+		// The namespace can be created when we run make install
+		// However, in this test we want ensure that the solution
+		// can run in a ns labeled as restricted. Therefore, we are
+		// creating the namespace an lebeling it.
+		By("creating manager namespace")
+		cmd := exec.Command("kubectl", "create", "ns", namespace)
+		_, _ = utils.Run(cmd)
+
+		// Now, let's ensure that all namespaces can raise an Warn when we apply the manifests
+		// and that the namespace where the Operator and Operand will run are enforced as
+		// restricted so that we can ensure that both can be admitted and run with the enforcement
+		By("labeling all namespaces to warn when we apply the manifest if would violate the PodStandards")
+		cmd = exec.Command("kubectl", "label", "--overwrite", "ns", "--all",
+			"pod-security.kubernetes.io/audit=restricted",
+			"pod-security.kubernetes.io/enforce-version=v1.24",
+			"pod-security.kubernetes.io/warn=restricted")
+		_, err := utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+		By("labeling enforce the namespace where the Operator and Operand(s) will run")
+		cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace,
+			"pod-security.kubernetes.io/audit=restricted",
+			"pod-security.kubernetes.io/enforce-version=v1.24",
+			"pod-security.kubernetes.io/enforce=restricted")
+		_, err = utils.Run(cmd)
+		Expect(err).To(Not(HaveOccurred()))
+
+		By("uncommenting all sections with 'monitoring' to enable operator custom metrics")
+		err = utils.ReplaceInFile("controllers/memcached_controller.go",
+			"//"+monitoringImportFragment, monitoringImportFragment)
+		Expect(err).To(Not(HaveOccurred()))
+
+		err = utils.ReplaceInFile("controllers/memcached_controller.go",
+			"//"+incMemcachedDeploymentSizeUndesiredCountTotalFragment, incMemcachedDeploymentSizeUndesiredCountTotalFragment)
+		Expect(err).To(Not(HaveOccurred()))
+
+		err = utils.ReplaceInFile("main.go",
+			"//"+monitoringImportFragment, monitoringImportFragment)
+		Expect(err).To(Not(HaveOccurred()))
+
+		err = utils.ReplaceInFile("main.go",
+			"//"+registerMetricsFragment, registerMetricsFragment)
+		Expect(err).To(Not(HaveOccurred()))
+	})
 
-			By("uninstalling the cert-manager bundle")
-			utils.UninstallCertManager()
+	AfterAll(func() {
+		By("uninstalling the Prometheus manager bundle")
+		utils.UninstallPrometheusOperator()
 
-			By("removing manager namespace")
-			cmd := exec.Command("kubectl", "create", "ns", namespace)
-			_, _ = utils.Run(cmd)
-		})
+		By("uninstalling the cert-manager bundle")
+		utils.UninstallCertManager()
+
+		By("removing manager namespace")
+		cmd := exec.Command("kubectl", "create", "ns", namespace)
+		_, _ = utils.Run(cmd)
+	})
 
-		It("should successfully run the Memcached Operator", func() {
+	Context("Memcached Operator", func() {
+		It("should run successfully", func() {
 			var controllerPodName string
 			var err error
 			projectDir, _ := utils.GetProjectDir()
 
-			// operatorImage store the name of the imahe used in the example
-			const operatorImage = "example.com/memcached-operator:v0.0.1"
+			// operatorImage stores the name of the image used in the example
+			var operatorImage = "example.com/memcached-operator:v0.0.1"
 
 			By("building the manager(Operator) image")
-			cmd := exec.Command("make", "docker-build", "IMG=example.com/memcached-operator:v0.0.1")
+			cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", operatorImage))
 			_, err = utils.Run(cmd)
 			ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 			By("loading the the manager(Operator) image on Kind")
-			err = utils.LoadImageToKindClusterWithName("example.com/memcached-operator:v0.0.1")
+			err = utils.LoadImageToKindClusterWithName(operatorImage)
 			ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 			By("installing CRDs")
@@ -287,7 +319,7 @@ var _ = Describe("memcached", func() {
 
 			By("validating that pod(s) status.phase=Running")
 			getMemcachedPodStatus := func() error {
-				cmd =  exec.Command("kubectl", "get", 
+				cmd = exec.Command("kubectl", "get",
 					"pods", "-l", "app.kubernetes.io/name=Memcached",
 					"-o", "jsonpath={.items[*].status}", "-n", namespace,
 				)
@@ -318,7 +350,172 @@ var _ = Describe("memcached", func() {
 			Eventually(getStatus, time.Minute, time.Second).Should(Succeed())
 		})
 	})
+
+	Context("Memcached Operator metrics", Ordered, func() {
+		BeforeAll(func() {
+			By("granting permissions to access the metrics")
+			cmd := exec.Command("kubectl",
+				"create", "clusterrolebinding", "metrics-memcached-operator",
+				"--clusterrole=memcached-operator-metrics-reader",
+				fmt.Sprintf("--serviceaccount=%s:memcached-operator-controller-manager", namespace))
+			_, err := utils.Run(cmd)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		AfterAll(func() {
+			By("removing permissions to access the metrics")
+			cmd := exec.Command("kubectl", "delete",
+				"clusterrolebinding", "metrics-memcached-operator")
+			_, err := utils.Run(cmd)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("MemcachedDeploymentSizeUndesiredCountTotal should be increased when scaling the Memcached deployment", func() {
+			initialMetricValue := getMetricValue(memcachedDeploymentSizeUndesiredCountTotalName)
+
+			numberOfScales := 5
+			By(fmt.Sprintf("scaling memcached-samle deployment %d times", numberOfScales))
+			scaleMemcachedSampleDeployment(numberOfScales)
+
+			By(fmt.Sprintf("validating MemcachedDeploymentSizeUndesiredCountTotal has increased by %d", numberOfScales))
+			finalMetricValue := getMetricValue(memcachedDeploymentSizeUndesiredCountTotalName)
+			Expect(finalMetricValue).To(Equal(initialMetricValue + numberOfScales))
+		})
+	})
 })
+
+// getMetricValue will reach the Memcached operator metrics endpoint, validate the metric and extract its value
+func getMetricValue(metricName string) int {
+	// reach the metrics endpoint and validate the metric exists
+	metricsEndpoint := curlMetrics()
+	ExpectWithOffset(1, metricsEndpoint).Should(ContainSubstring(metricName))
+
+	// extract the metric value
+	metricValue, err := strconv.Atoi(parseMetricValue(metricsEndpoint, metricName))
+	ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+	return metricValue
+}
+
+// curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned.
+func curlMetrics() string {
+	By("reading the metrics token")
+	// Filter token query by service account in case more than one exists in a namespace.
+	token, err := serviceAccountToken()
+	ExpectWithOffset(2, err).NotTo(HaveOccurred())
+	ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0))
+
+	By("creating a curl pod")
+	cmd := exec.Command("kubectl", "run", "curl", "--image=curlimages/curl:7.68.0",
+		"--restart=OnFailure", "-n", "default", "--", "curl", "-v", "-k", "-H",
+		fmt.Sprintf("Authorization: Bearer %s", strings.TrimSpace(token)),
+		fmt.Sprintf("https://memcached-operator-controller-manager-metrics-service.%s.svc:8443/metrics", namespace))
+	_, err = utils.Run(cmd)
+	ExpectWithOffset(2, err).NotTo(HaveOccurred())
+
+	By("validating that the curl pod is running as expected")
+	verifyCurlUp := func() error {
+		// Validate pod status
+		cmd := exec.Command("kubectl", "get", "pods", "curl",
+			"-o", "jsonpath={.status.phase}", "-n", "default")
+		statusOutput, err := utils.Run(cmd)
+		status := string(statusOutput)
+		ExpectWithOffset(3, err).NotTo(HaveOccurred())
+		if status != "Completed" && status != "Succeeded" {
+			return fmt.Errorf("curl pod in %s status", status)
+		}
+		return nil
+	}
+	EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed())
+
+	By("validating that the metrics endpoint is serving as expected")
+	var metricsEndpoint string
+	getCurlLogs := func() string {
+		cmd = exec.Command("kubectl", "logs", "curl", "-n", "default")
+		metricsEndpointOutput, err := utils.Run(cmd)
+		ExpectWithOffset(3, err).NotTo(HaveOccurred())
+		metricsEndpoint = string(metricsEndpointOutput)
+		return metricsEndpoint
+	}
+	EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/2 200"))
+
+	By("cleaning up the curl pod")
+	cmd = exec.Command("kubectl", "delete",
+		"pods/curl", "-n", "default")
+	_, err = utils.Run(cmd)
+	ExpectWithOffset(3, err).NotTo(HaveOccurred())
+
+	return metricsEndpoint
+}
+
+// serviceAccountToken provides a helper function that can provide you with a service account
+// token that you can use to interact with the service. This function leverages the k8s'
+// TokenRequest API in raw format in order to make it generic for all version of the k8s that
+// is currently being supported in kubebuilder test infra.
+// TokenRequest API returns the token in raw JWT format itself. There is no conversion required.
+func serviceAccountToken() (out string, err error) {
+	By("Creating the ServiceAccount token")
+	secretName := "memcached-operator-controller-manager-token-request"
+	projectDir, _ := utils.GetProjectDir()
+	tokenRequestFile := filepath.Join(projectDir, "/test/e2e/", secretName)
+	err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755))
+	if err != nil {
+		return out, err
+	}
+	var rawJson string
+	Eventually(func() error {
+		// Output of this is already a valid JWT token. No need to covert this from base64 to string format
+		cmd := exec.Command("kubectl", "create", "--raw",
+			fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/memcached-operator-controller-manager/token", namespace),
+			"-f", tokenRequestFile,
+		)
+		rawJsonOutput, err := utils.Run(cmd)
+		rawJson = string(rawJsonOutput)
+		if err != nil {
+			return err
+		}
+		var token tokenRequest
+		err = json.Unmarshal([]byte(rawJson), &token)
+		if err != nil {
+			return err
+		}
+		out = token.Status.Token
+		return nil
+	}, time.Minute, time.Second).Should(Succeed())
+
+	return out, err
+}
+
+// parseMetricValue will parse the metric value from the metrics endpoint
+func parseMetricValue(metricsEndpoint string, metricName string) string {
+	r := strings.NewReader(metricsEndpoint)
+	scan := bufio.NewScanner(r)
+	for scan.Scan() {
+		metricLine := scan.Text()
+		if strings.HasPrefix(metricLine, metricName) {
+			split := strings.Split(metricLine, " ")
+			return split[1]
+		}
+	}
+	return ""
+}
+
+// scaleMemcachedSampleDeployment will scale memcached-sample deployment 'numberOfScales' times
+func scaleMemcachedSampleDeployment(numberOfScales int) {
+	for i := 1; i <= numberOfScales; i++ {
+		cmd := exec.Command("kubectl", "scale", "--replicas=3",
+			"deployment", "memcached-sample", "-n", namespace)
+		_, err := utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+		time.Sleep(10 * time.Second)
+	}
+}
+
+const monitoringImportFragment = "\"github.com/example/memcached-operator/monitoring\""
+
+const incMemcachedDeploymentSizeUndesiredCountTotalFragment = "monitoring.MemcachedDeploymentSizeUndesiredCountTotal.Inc()"
+
+const registerMetricsFragment = "monitoring.RegisterMetrics()"
 `
 
 const utilsTemplate = `/*
@@ -340,6 +537,7 @@ limitations under the License.
 package utils
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -464,6 +662,29 @@ func GetProjectDir() (string, error) {
 	wd = strings.Replace(wd, "/test/e2e", "", -1)
 	return wd, nil
 }
+
+// ReplaceInFile replaces all instances of old with new in the file at path.
+func ReplaceInFile(path, old, new string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	// false positive
+	// nolint:gosec
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	if !strings.Contains(string(b), old) {
+		return errors.New("unable to find the content to be replaced")
+	}
+	s := strings.Replace(string(b), old, new, -1)
+	err = os.WriteFile(path, []byte(s), info.Mode())
+	if err != nil {
+		return err
+	}
+	return nil
+}
 `
 
 const targetTemplate = `