Glassflow otel collector and helm values update

.github/workflows/lint.yaml

@@ -27,5 +27,8 @@ jobs:
       - name: Set up chart-testing
         uses: helm/[email protected]
 
+      - name: Update Helm dependencies
+        run: helm dependency update charts/glassflow-etl
+
       - name: Run chart-testing (lint)
         run: ct lint --config ct.yaml

charts/glassflow-etl/Chart.yaml

@@ -15,17 +15,17 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.8
+version: 0.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "2.1.2"
+appVersion: "2.1.3"
 
 dependencies:
   - name: glassflow-operator
-    version: "0.5.0"
+    version: "0.5.3"
     repository: https://glassflow.github.io/glassflow-etl-k8s-operator
   - name: nats
     version: "1.3.6"

charts/glassflow-etl/templates/deployment.yaml

@@ -112,11 +112,23 @@ spec:
             - name: GLASSFLOW_NATS_SERVER
               value: "nats://{{ .Release.Name }}-nats.{{ .Release.Namespace }}.svc.cluster.local:4222"
             - name: GLASSFLOW_LOG_LEVEL
-              value: "INFO"
+              value: "{{ .Values.api.logLevel }}"
             - name: GLASSFLOW_RUN_LOCAL
               value: "false"
             - name: GLASSFLOW_K8S_NAMESPACE
               value: {{ .Release.Namespace }}
+            - name: GLASSFLOW_OTEL_SERVICE_NAME
+              value: "glassflow-api"
+            - name: GLASSFLOW_OTEL_SERVICE_NAMESPACE
+              value: {{ .Release.Namespace }}
+            - name: GLASSFLOW_OTEL_SERVICE_VERSION
+              value: "{{ .Values.api.image.tag }}"          
+            - name: GLASSFLOW_OTEL_LOGS_ENABLED
+              value: "{{ .Values.global.observability.logs.enabled }}"
+            - name: GLASSFLOW_OTEL_METRICS_ENABLED
+              value: "{{ .Values.global.observability.metrics.enabled }}"
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: "http://{{ .Release.Name }}-otel-collector.{{ .Release.Namespace }}.svc.cluster.local:4318"
             {{- range .Values.api.env }}
             - name: {{ .name | quote }}
               value: {{ .value | quote }}

charts/glassflow-etl/templates/otel-collector-configmap.yaml

@@ -0,0 +1,64 @@
+{{- if or .Values.global.observability.metrics.enabled .Values.global.observability.logs.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-otel-collector-config
+  namespace: {{ .Release.Namespace }}
+data:
+  otel-collector-config.yaml: |
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+
+    processors:
+      batch:
+        timeout: 1s
+        send_batch_size: 1024
+      resource:
+        attributes:
+          - key: service.name
+            from_attribute: service.name
+            action: insert
+          - key: service.version
+            from_attribute: service.version
+            action: insert
+          - key: service.namespace
+            from_attribute: service.namespace
+            action: insert
+
+    exporters:
+{{- if .Values.global.observability.logs.enabled }}
+      otlp: {{ toYaml .Values.global.observability.logs.exporter.otlp | nindent 8 }}
+{{- end }}
+{{- if .Values.global.observability.metrics.enabled }}
+      prometheus:
+          endpoint: "0.0.0.0:9090"
+          namespace: {{ .Release.Namespace }}          
+          send_timestamps: true
+          enable_open_metrics: true
+{{- end }}
+
+    extensions:
+      health_check:
+        endpoint: 0.0.0.0:13133
+
+    service:
+      extensions: [health_check]
+      pipelines:
+{{- if .Values.global.observability.logs.enabled }}
+        logs:
+          receivers: [otlp]
+          processors: [batch, resource]
+          exporters: [otlp]
+{{- end }}
+{{- if .Values.global.observability.metrics.enabled }}
+        metrics:
+          receivers: [otlp]
+          processors: [batch, resource]
+          exporters: [prometheus]
+{{- end }}
+{{- end }}

charts/glassflow-etl/templates/otel-collector-deployment.yaml

@@ -0,0 +1,63 @@
+{{- if or .Values.global.observability.metrics.enabled .Values.global.observability.logs.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}-otel-collector
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-otel-collector
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-otel-collector
+    spec:
+      containers:
+      - name: {{ .Release.Name }}-otel-collector
+        image: otel/opentelemetry-collector-contrib:0.108.0
+        command:
+          - "/otelcol-contrib"
+          - "--config=/etc/otel-collector-config.yaml"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 128Mi
+          limits:
+            cpu: 500m
+            memory: 512Mi
+        ports:
+        - containerPort: 4317
+          name: otlp-grpc
+        - containerPort: 4318
+          name: otlp-http
+{{- if .Values.global.observability.metrics.enabled }}
+        - containerPort: 9090
+          name: prometheus
+{{- end }}
+        - containerPort: 13133
+          name: health-check
+        volumeMounts:
+        - name: config
+          mountPath: /etc/otel-collector-config.yaml
+          subPath: otel-collector-config.yaml
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 13133
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 13133
+          initialDelaySeconds: 5
+          periodSeconds: 10
+      volumes:
+      - name: config
+        configMap:
+          name: {{ .Release.Name }}-otel-collector-config
+{{- end }}

charts/glassflow-etl/templates/otel-collector-service.yaml

@@ -0,0 +1,24 @@
+{{- if or .Values.global.observability.metrics.enabled .Values.global.observability.logs.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}-otel-collector
+spec:
+  ports:
+  - name: otlp-grpc
+    port: 4317
+    targetPort: 4317
+  - name: otlp-http
+    port: 4318
+    targetPort: 4318
+{{- if .Values.global.observability.metrics.enabled }}
+  - name: prometheus
+    port: 9090
+    targetPort: 9090
+{{- end }}
+  selector:
+    app: {{ .Release.Name }}-otel-collector
+{{- end }}