[WIP] Add external Prometheus E2Es

assets/monitoring/grafana/v1alpha1/deployment.yaml

@@ -121,10 +121,18 @@ spec:
           mountPath: /var/run/secrets/grafana-admin-credentials
         - name: grafana-serving-certs
           mountPath: /var/run/secrets/grafana-serving-certs
+        {{- if and .prometheusTLSSpec .prometheusTLSSpec.ClientTLSKeyPairSecretRef }}
         - name: prometheus-client-certs
           mountPath: /var/run/secrets/prometheus-client-certs
+        {{- end }}
+        {{- if and .prometheusTLSSpec .prometheusTLSSpec.ServingCAConfigMapRef }}
         - name: prometheus-serving-ca
           mountPath: /var/run/configmaps/prometheus-serving-ca
+        {{- end }}
+        {{- if and .prometheusAuthSpec .prometheusAuthSpec.BearerTokenSecretRef }}
+        - name: prometheus-bearer-token
+          mountPath: /var/run/secrets/prometheus-bearer-token
+        {{- end }}
         - name: grafana-storage
           mountPath: /var/lib/grafana
         securityContext:
@@ -159,12 +167,21 @@ spec:
       - name: grafana-serving-certs
         secret:
           secretName: "{{ .servingCertSecretName }}"
+      {{- if and .prometheusTLSSpec .prometheusTLSSpec.ClientTLSKeyPairSecretRef }}
       - name: prometheus-client-certs
         secret:
-          secretName: "{{ .scyllaDBMonitoringName }}-prometheus-client-grafana"
+          secretName: "{{ .prometheusTLSSpec.ClientTLSKeyPairSecretRef.Name }}"
+      {{- end }}
+      {{- if and .prometheusTLSSpec .prometheusTLSSpec.ServingCAConfigMapRef }}
       - name: prometheus-serving-ca
         configMap:
-          name: "{{ .scyllaDBMonitoringName }}-prometheus-serving-ca"
+          name: "{{ .prometheusTLSSpec.ServingCAConfigMapRef.Name }}"
+      {{- end }}
+      {{- if and .prometheusAuthSpec .prometheusAuthSpec.BearerTokenSecretRef }}
+      - name: prometheus-bearer-token
+        secret:
+          secretName: "{{ .prometheusAuthSpec.BearerTokenSecretRef.Name }}"
+      {{- end }}
       - name: grafana-storage
         emptyDir:
           sizeLimit: 100Mi

assets/monitoring/grafana/v1alpha1/provisioning.cm.yaml

@@ -20,16 +20,36 @@ data:
     - name: prometheus
       type: prometheus
       access: proxy
-      url: "https://{{ .scyllaDBMonitoringName }}-prometheus:9090"
+      url: "{{ .prometheusDatasource.URL }}"
       isDefault: true
       version: 1
       editable: false
       jsonData:
         timeInterval: "5s"
+        {{- if and .prometheusDatasource.TLS .prometheusDatasource.TLS.InsecureSkipVerify }}
+        tlsSkipVerify: true
+        {{- end }}
+        {{- if and .prometheusDatasource.TLS .prometheusDatasource.TLS.ServingCAConfigMapRef }}
         tlsAuthWithCACert: true
+        {{- end }}
+        {{- if and .prometheusDatasource.Auth .prometheusDatasource.Auth.BearerTokenSecretRef }}
+        httpHeaderName1: "Authorization"
+        {{- end }}
+      {{- $hasCACert := and .prometheusDatasource.TLS .prometheusDatasource.TLS.ServingCAConfigMapRef -}}
+      {{- $hasClientPair := and .prometheusDatasource.TLS .prometheusDatasource.TLS.ClientTLSKeyPairSecretRef -}}
+      {{- $hasBearer := and .prometheusDatasource.Auth .prometheusDatasource.Auth.BearerTokenSecretRef -}}
+      {{- if or $hasCACert $hasClientPair $hasBearer }}
       secureJsonData:
-        tlsCACert: "$__file{/var/run/configmaps/prometheus-serving-ca/ca-bundle.crt}"
-        tlsClientCert: "$__file{/var/run/secrets/prometheus-client-certs/tls.crt}"
+        {{- if $hasCACert }}
+        tlsCACert: "$__file{/var/run/configmaps/prometheus-serving-ca/{{ .prometheusDatasource.TLS.ServingCAConfigMapRef.Key }}}"
+        {{- end }}
+        {{- if $hasClientPair }}
+        tlsClientCert: "$__file{/var/run/secrets/prometheus-client-certs/tls.crt}" 
         tlsClientKey: "$__file{/var/run/secrets/prometheus-client-certs/tls.key}"
+        {{- end }}
+        {{- if $hasBearer }}
+        httpHeaderValue1: "Bearer $__file{/var/run/secrets/prometheus-bearer-token/{{ .prometheusDatasource.Auth.BearerTokenSecretRef.Key }}}"
+        {{- end }}
+      {{- end }}
   notifiers.yaml: ""
   plugins.yaml: ""

bundle/manifests/scylla-operator.clusterserviceversion.yaml

@@ -544,6 +544,7 @@ spec:
             - scylladbclusters
             - scylladbmanagerclusterregistrations
             - scylladbmanagertasks
+            - scylladbmonitorings
       sideEffects: None
       targetPort: 5000
       type: ValidatingAdmissionWebhook

bundle/manifests/scylla.scylladb.com_scylladbmonitorings.yaml

@@ -70,6 +70,106 @@ spec:
                               to Grafana without authentication.
                             type: boolean
                         type: object
+                      datasources:
+                        description: |-
+                          datasources is a list of Grafana datasources to configure.
+                          It's expected to be set when using Prometheus component in `External` mode.
+                          At most one datasource is allowed for now (only Prometheus is supported).
+                        items:
+                          properties:
+                            name:
+                              default: prometheus
+                              description: |-
+                                name is the name of the datasource as it will appear in Grafana.
+                                Only "prometheus" is supported as that's the datasource name expected by the ScyllaDB monitoring stack dashboards.
+                              enum:
+                              - prometheus
+                              type: string
+                            prometheusOptions:
+                              description: prometheusOptions defines Prometheus-specific
+                                options.
+                              properties:
+                                auth:
+                                  description: auth holds authentication options for
+                                    connecting to Prometheus.
+                                  properties:
+                                    bearerTokenOptions:
+                                      description: bearerToken holds options for Bearer
+                                        token authentication.
+                                      properties:
+                                        secretRef:
+                                          description: secretRef is a reference to
+                                            a key in a Secret holding a Bearer token
+                                            to use to authenticate with Prometheus.
+                                          properties:
+                                            key:
+                                              description: key within the selected
+                                                object.
+                                              minLength: 1
+                                              type: string
+                                            name:
+                                              description: name of the selected object.
+                                              minLength: 1
+                                              type: string
+                                          type: object
+                                      type: object
+                                    type:
+                                      default: NoAuthentication
+                                      description: type is the type of authentication
+                                        to use.
+                                      type: string
+                                  type: object
+                                tls:
+                                  description: tls holds TLS configuration for connecting
+                                    to Prometheus over HTTPS.
+                                  properties:
+                                    caCertConfigMapRef:
+                                      description: |-
+                                        caCert is a reference to a key within the CA bundle ConfigMap. The key should hold the CA cert in PEM format.
+                                        When not specified, system CAs are used.
+                                      properties:
+                                        key:
+                                          description: key within the selected object.
+                                          minLength: 1
+                                          type: string
+                                        name:
+                                          description: name of the selected object.
+                                          minLength: 1
+                                          type: string
+                                      type: object
+                                    clientTLSKeyPairSecretRef:
+                                      description: |-
+                                        clientTLSKeyPairSecretRef is a reference to a Secret holding client TLS certificate and key for mTLS authentication.
+                                        It's expected to be a standard Kubernetes TLS Secret with `tls.crt` and `tls.key` keys.
+                                      properties:
+                                        name:
+                                          description: Name of the referent.
+                                          type: string
+                                      type: object
+                                    insecureSkipVerify:
+                                      default: false
+                                      description: insecureSkipVerify controls whether
+                                        to skip server certificate verification.
+                                      type: boolean
+                                  type: object
+                              type: object
+                            type:
+                              allOf:
+                              - enum:
+                                - Prometheus
+                              - enum:
+                                - Prometheus
+                              default: Prometheus
+                              description: type is the type of the datasource. Only
+                                "prometheus" is supported.
+                              type: string
+                            url:
+                              description: url is the URL of the datasource.
+                              minLength: 1
+                              type: string
+                          type: object
+                        maxItems: 1
+                        type: array
                       exposeOptions:
                         description: exposeOptions specifies options for exposing
                           Grafana UI.
@@ -1172,6 +1272,13 @@ spec:
                                 type: object
                             type: object
                         type: object
+                      mode:
+                        default: Managed
+                        description: mode defines the mode of the Prometheus instance.
+                        enum:
+                        - Managed
+                        - External
+                        type: string
                       placement:
                         description: placement describes restrictions for the nodes
                           Prometheus is scheduled on.

bundle/manifests/scyllacluster-view_rbac.authorization.k8s.io_v1_clusterrole.yaml

@@ -17,6 +17,7 @@ rules:
   - scylladbclusters
   - scylladbmanagerclusterregistrations
   - scylladbmanagertasks
+  - scylladbmonitorings
   verbs:
   - get
   - list