Skip to content

Commit b8c7431

Browse files
czeslavoczeslavo
committed
Add external Prometheus on OpenShift E2E
1 parent ddd390d commit b8c7431

File tree

3 files changed

+153
-7
lines changed

3 files changed

+153
-7
lines changed

‎hack/ci-deploy.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,10 @@ if [[ -n ${SO_INSTALL_XFSPROGS_ON_NODES:-} ]]; then
108108
kubectl_create -f "${ARTIFACTS_DEPLOY_DIR}"/install-xfsprogs.daemonset.yaml
109109
fi
110110

111+
if [[ "${SO_ENABLE_OPENSHIFT_USER_WORKLOAD_MONITORING:-}" == "true" ]]; then
112+
kubectl_create -f "${ARTIFACTS_DEPLOY_DIR}/openshift-uwm.cm.yaml"
113+
fi
114+
111115
kubectl_create -n haproxy-ingress -f "${ARTIFACTS_DEPLOY_DIR}/haproxy-ingress"
112116
kubectl_create -f "${ARTIFACTS_DEPLOY_DIR}"/cert-manager.yaml
113117

‎test/e2e/set/scylladbmonitoring/scylladbmonitoring.go‎

Lines changed: 126 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import (
3838
corev1 "k8s.io/api/core/v1"
3939
rbacv1 "k8s.io/api/rbac/v1"
4040
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
41-
"k8s.io/apimachinery/pkg/util/intstr"
41+
apimachineryutilintstr "k8s.io/apimachinery/pkg/util/intstr"
4242
"k8s.io/client-go/util/cert"
4343
"k8s.io/klog/v2"
4444
)
@@ -70,7 +70,7 @@ var _ = g.Describe("ScyllaDBMonitoring", func() {
7070
f := framework.NewFramework("scylladbmonitoring")
7171

7272
// Disabled on OpenShift because of https://github.com/scylladb/scylla-operator/issues/2319#issuecomment-2643287819
73-
g.DescribeTable("should setup monitoring stack TESTCASE_DISABLED_ON_OPENSHIFT", func(ctx g.SpecContext, e *scyllaDBMonitoringEntry) {
73+
g.DescribeTable("should setup monitoring stack", func(ctx g.SpecContext, e *scyllaDBMonitoringEntry) {
7474
framework.By("Creating a ScyllaCluster with a single node")
7575
sc := createTestScyllaCluster(ctx, f)
7676

@@ -88,8 +88,10 @@ var _ = g.Describe("ScyllaDBMonitoring", func() {
8888
framework.By("Waiting for the ScyllaDBMonitoring to roll out")
8989
awaitScyllaDBMonitoringRollout(ctx, f, sm)
9090

91-
framework.By("Verifying that Prometheus is configured correctly")
92-
e.VerifyPrometheusFn(ctx, f, sm)
91+
if e.VerifyPrometheusFn != nil {
92+
framework.By("Verifying that Prometheus is configured correctly")
93+
e.VerifyPrometheusFn(ctx, f, sm)
94+
}
9395

9496
framework.By("Verifying that Grafana is configured correctly")
9597
e.VerifyGrafanaFn(ctx, f, sm)
@@ -146,7 +148,7 @@ var _ = g.Describe("ScyllaDBMonitoring", func() {
146148
VerifyPrometheusFn: verifyExternalPrometheusWithoutTLS,
147149
VerifyGrafanaFn: verifyManagedGrafanaWithDashboards(getExpectedPlatformDashboards()),
148150
}),
149-
g.FEntry(describeEntry, &scyllaDBMonitoringEntry{
151+
g.Entry(describeEntry, &scyllaDBMonitoringEntry{
150152
Description: "Platform type with external Prometheus with TLS",
151153
ScyllaDBMonitoringModifierFn: func(sm *scyllav1alpha1.ScyllaDBMonitoring) {
152154
sm.Spec.Type = pointer.Ptr(scyllav1alpha1.ScyllaDBMonitoringTypePlatform)
@@ -172,6 +174,41 @@ var _ = g.Describe("ScyllaDBMonitoring", func() {
172174
VerifyPrometheusFn: verifyExternalPrometheusWithTLS,
173175
VerifyGrafanaFn: verifyManagedGrafanaWithDashboards(getExpectedPlatformDashboards()),
174176
}),
177+
g.FEntry(describeEntry, &scyllaDBMonitoringEntry{
178+
Description: "Platform type with Thanos Querier on OpenShift",
179+
ScyllaDBMonitoringModifierFn: func(sm *scyllav1alpha1.ScyllaDBMonitoring) {
180+
sm.Spec.Type = pointer.Ptr(scyllav1alpha1.ScyllaDBMonitoringTypePlatform)
181+
sm.Spec.Components.Prometheus.Mode = scyllav1alpha1.PrometheusModeExternal
182+
sm.Spec.Components.Grafana.Datasources = []scyllav1alpha1.GrafanaDatasourceSpec{
183+
{
184+
Name: "prometheus",
185+
Type: scyllav1alpha1.GrafanaDatasourceTypePrometheus,
186+
URL: "https://thanos-querier.openshift-monitoring.svc:9091",
187+
PrometheusOptions: &scyllav1alpha1.GrafanaPrometheusDatasourceOptions{
188+
TLS: &scyllav1alpha1.GrafanaDatasourceTLSSpec{
189+
InsecureSkipVerify: false,
190+
CACertConfigMapRef: &scyllav1alpha1.LocalObjectKeySelector{
191+
Name: "openshift-service-ca.crt",
192+
Key: "service-ca.crt",
193+
},
194+
},
195+
Auth: &scyllav1alpha1.GrafanaPrometheusDatasourceAuthSpec{
196+
Type: scyllav1alpha1.GrafanaPrometheusDatasourceAuthTypeBearerToken,
197+
BearerTokenOptions: &scyllav1alpha1.GrafanaPrometheusDatasourceBearerTokenAuthOptions{
198+
SecretRef: &scyllav1alpha1.LocalObjectKeySelector{
199+
Name: monitoringAccessServiceAccountNameOnOpenShift(sm),
200+
Key: "token",
201+
},
202+
},
203+
},
204+
},
205+
},
206+
}
207+
},
208+
PrepareExternalPrometheusFn: prepareOpenShiftMonitoring,
209+
VerifyPrometheusFn: nil, // Nothing to verify, we trust OpenShift.
210+
VerifyGrafanaFn: verifyManagedGrafanaWithDashboards(getExpectedPlatformDashboards()),
211+
}),
175212
)
176213
})
177214

@@ -400,6 +437,7 @@ func verifyManagedGrafanaWithDashboards(
400437
o.Expect(err).NotTo(o.HaveOccurred())
401438

402439
verifyGrafanaDashboards(grafanaClient, expectedDashboards, expectedHomeDashboardUID)
440+
verifyPrometheusGrafanaDataSource(grafanaClient)
403441
}
404442
}
405443

@@ -423,6 +461,18 @@ func verifyGrafanaDashboards(grafanaClient *grafana.Client, expectedDashboards [
423461
o.Expect(homeDashboardUID).To(o.Equal(expectedHomeDashboardUID))
424462
}
425463

464+
func verifyPrometheusGrafanaDataSource(grafanaClient *grafana.Client) {
465+
g.GinkgoHelper()
466+
467+
framework.By("Verifying 'prometheus' Grafana data source")
468+
o.Eventually(func(eo o.Gomega) {
469+
health, err := grafanaClient.DatasourceHealth("prometheus")
470+
framework.Infof("Checking 'prometheus' grafana data source health: err: %v, health: %v, message: %s", err, health.OK, health.Message)
471+
eo.Expect(err).NotTo(o.HaveOccurred())
472+
eo.Expect(health.OK).To(o.Equal(true))
473+
}).WithTimeout(10 * time.Minute).WithPolling(1 * time.Second).Should(o.Succeed())
474+
}
475+
426476
// getExpectedPlatformDashboards returns the expected grafana dashboards.
427477
// Platform dashboards come directly from ScyllaDB Monitoring, so we do not know the expected values
428478
// and given the size they are not feasible to be maintained as a duplicate.
@@ -685,7 +735,7 @@ func createExternalPrometheusInstanceWithoutTLS(ctx context.Context, f *framewor
685735
Alertmanagers: []monitoringv1.AlertmanagerEndpoints{
686736
{
687737
Name: "scylla-monitoring",
688-
Port: intstr.FromString("web"),
738+
Port: apimachineryutilintstr.FromString("web"),
689739
},
690740
},
691741
},
@@ -753,7 +803,7 @@ func createExternalPrometheusInstanceWithTLS(ctx context.Context, f *framework.F
753803
Alertmanagers: []monitoringv1.AlertmanagerEndpoints{
754804
{
755805
Name: "scylla-monitoring",
756-
Port: intstr.FromString("web"),
806+
Port: apimachineryutilintstr.FromString("web"),
757807
},
758808
},
759809
},
@@ -771,6 +821,71 @@ func createExternalPrometheusInstanceWithTLS(ctx context.Context, f *framework.F
771821
return prom
772822
}
773823

824+
func prepareOpenShiftMonitoring(ctx context.Context, f *framework.Framework, sm *scyllav1alpha1.ScyllaDBMonitoring) {
825+
g.GinkgoHelper()
826+
827+
framework.By("Creating a ServiceAccount for monitoring access on OpenShift")
828+
sa := createMonitoringAccessServiceAccountOnOpenShift(ctx, f, sm)
829+
830+
framework.By("Binding cluster-monitoring-view ClusterRole to the ServiceAccount")
831+
bindClusterMonitoringViewClusterRoleToServiceAccount(ctx, f, sa)
832+
833+
framework.By("Creating a Secret with the ServiceAccount token")
834+
createServiceAccountTokenSecret(ctx, f, sa)
835+
}
836+
837+
func createMonitoringAccessServiceAccountOnOpenShift(ctx context.Context, f *framework.Framework, sm *scyllav1alpha1.ScyllaDBMonitoring) *corev1.ServiceAccount {
838+
sa, err := f.KubeAdminClient().CoreV1().ServiceAccounts(f.Namespace()).Create(ctx, &corev1.ServiceAccount{
839+
ObjectMeta: metav1.ObjectMeta{
840+
Name: monitoringAccessServiceAccountNameOnOpenShift(sm),
841+
Namespace: f.Namespace(),
842+
},
843+
}, metav1.CreateOptions{
844+
FieldManager: f.FieldManager(),
845+
})
846+
o.Expect(err).NotTo(o.HaveOccurred())
847+
return sa
848+
}
849+
850+
func bindClusterMonitoringViewClusterRoleToServiceAccount(ctx context.Context, f *framework.Framework, sa *corev1.ServiceAccount) {
851+
_, err := f.KubeAdminClient().RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{
852+
ObjectMeta: metav1.ObjectMeta{
853+
Name: sa.Name,
854+
},
855+
Subjects: []rbacv1.Subject{
856+
{
857+
Kind: rbacv1.ServiceAccountKind,
858+
Name: sa.Name,
859+
Namespace: sa.Namespace,
860+
},
861+
},
862+
RoleRef: rbacv1.RoleRef{
863+
APIGroup: "rbac.authorization.k8s.io",
864+
Kind: "ClusterRole",
865+
Name: "cluster-monitoring-view",
866+
},
867+
}, metav1.CreateOptions{
868+
FieldManager: f.FieldManager(),
869+
})
870+
o.Expect(err).NotTo(o.HaveOccurred())
871+
}
872+
873+
func createServiceAccountTokenSecret(ctx context.Context, f *framework.Framework, sa *corev1.ServiceAccount) {
874+
_, err := f.KubeAdminClient().CoreV1().Secrets(f.Namespace()).Create(ctx, &corev1.Secret{
875+
ObjectMeta: metav1.ObjectMeta{
876+
Name: sa.Name,
877+
Namespace: f.Namespace(),
878+
Annotations: map[string]string{
879+
"kubernetes.io/service-account.name": sa.Name,
880+
},
881+
},
882+
Type: corev1.SecretTypeServiceAccountToken,
883+
}, metav1.CreateOptions{
884+
FieldManager: f.FieldManager(),
885+
})
886+
o.Expect(err).NotTo(o.HaveOccurred())
887+
}
888+
774889
func verifyExternalPrometheusWithoutTLS(ctx context.Context, f *framework.Framework, sm *scyllav1alpha1.ScyllaDBMonitoring) {
775890
g.GinkgoHelper()
776891

@@ -859,3 +974,7 @@ func prometheusCACertConfigMapNameForScyllaDBMonitoring(sm *scyllav1alpha1.Scyll
859974
func prometheusTLSSecretNameForScyllaDBMonitoring(sm *scyllav1alpha1.ScyllaDBMonitoring) string {
860975
return fmt.Sprintf("%s-prometheus-tls", sm.Name)
861976
}
977+
978+
func monitoringAccessServiceAccountNameOnOpenShift(sm *scyllav1alpha1.ScyllaDBMonitoring) string {
979+
return fmt.Sprintf("%s-monitoring-access", sm.Name)
980+
}

‎test/e2e/utils/grafana/client.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,26 @@ func (c *Client) HomeDashboardUID() (string, error) {
9595

9696
return "", fmt.Errorf("unexpected type for dashboard payload")
9797
}
98+
99+
type DatasourceHealth struct {
100+
Message string
101+
OK bool
102+
}
103+
104+
func (c *Client) DatasourceHealth(datasourceName string) (DatasourceHealth, error) {
105+
resp, err := c.c.Datasources.GetDataSourceByName(datasourceName)
106+
if err != nil {
107+
return DatasourceHealth{}, fmt.Errorf("failed to get datasource %q: %w", datasourceName, err)
108+
}
109+
110+
datasourceUID := resp.GetPayload().UID
111+
healthResp, err := c.c.Datasources.CheckDatasourceHealthWithUID(datasourceUID)
112+
if err != nil {
113+
return DatasourceHealth{}, fmt.Errorf("failed to check health for datasource %q (UID %s): %w", datasourceName, datasourceUID, err)
114+
}
115+
116+
return DatasourceHealth{
117+
Message: healthResp.GetPayload().Message,
118+
OK: healthResp.IsSuccess(),
119+
}, nil
120+
}

0 commit comments

Comments
 (0)