Skip to content

Commit 6b01af9

Browse files
monraxmonrax
committed
add support for mmdb checksum, custom cron sched, post-install hook. Update README.md
1 parent d67bb1e commit 6b01af9

6 files changed

Lines changed: 229 additions & 137 deletions

File tree

‎README.md‎

Lines changed: 92 additions & 77 deletions
Large diffs are not rendered by default.

‎graylog/templates/_helpers.tpl‎

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,9 +278,45 @@ usage: {{ include "graylog.custom.env" .Values.{graylog|datanode} | indent N }}
278278
{{- end }}
279279

280280
{{/*
281-
Graylog plugins
281+
GeoIP update JobSpec
282+
usage: {{ list $geoSecretName $claimName $podIndex | include "graylog.geoip.job.spec" | indent }}
282283
*/}}
283-
{{- define "graylog.pluginURLs" }}
284+
{{- define "graylog.geoip.job.spec" }}
285+
backoffLimit: 2
286+
activeDeadlineSeconds: 900
287+
template:
288+
spec:
289+
securityContext:
290+
runAsUser: 1100
291+
runAsGroup: 1100
292+
fsGroup: 1100
293+
containers:
294+
- name: geoipupdate
295+
image: maxmindinc/geoipupdate:latest
296+
envFrom:
297+
- secretRef:
298+
name: {{ index . 0 }}
299+
env:
300+
- name: GEOIPUPDATE_EDITION_IDS
301+
value: "GeoLite2-City GeoLite2-ASN"
302+
- name: GEOIPUPDATE_FREQUENCY
303+
value: "0"
304+
- name: GEOIPUPDATE_DB_DIR
305+
value: "/usr/share/data/geolocation"
306+
volumeMounts:
307+
- name: geoip-db
308+
mountPath: /usr/share/data
309+
restartPolicy: OnFailure
310+
volumes:
311+
- name: geoip-db
312+
persistentVolumeClaim:
313+
claimName: {{ printf "%s-%d" (index . 1) (index . 2) }}
314+
{{- end }}
315+
316+
{{/*
317+
Graylog plugin URLs
318+
*/}}
319+
{{- define "graylog.plugin.URLs" }}
284320
{{- if and .Values.graylog.config.plugins.enabled .Values.graylog.config.init.assetFetch.enabled .Values.graylog.config.init.assetFetch.plugins.enabled .Values.graylog.plugins }}
285321
{{- $urls := list }}
286322
{{- $baseUrl := .Values.graylog.config.init.assetFetch.plugins.baseUrl | default "" }}
@@ -310,3 +346,40 @@ Graylog plugins
310346
{{- $urls | join "^" | quote }}
311347
{{- end }}
312348
{{- end }}
349+
350+
{{/*
351+
Geolocation mmdb URLs
352+
*/}}
353+
{{- define "graylog.mmdb.URLs" }}
354+
{{- if and .Values.graylog.config.geolocation.enabled .Values.graylog.config.init.assetFetch.geolocation.enabled .Values.graylog.config.geolocation.mmdbSources.city.url .Values.graylog.config.geolocation.mmdbSources.asn.url }}
355+
{{- $urls := list }}
356+
{{- $baseUrl := .Values.graylog.config.init.assetFetch.geolocation.baseUrl | default "" }}
357+
{{- $skipChecksum := .Values.graylog.config.init.assetFetch.skipChecksum | default false }}
358+
{{- $allowHttp := .Values.graylog.config.init.assetFetch.allowHttp | default false }}
359+
{{- if not $allowHttp | and (hasPrefix "http://" $baseUrl) }}
360+
{{- printf "Validation error: baseUrl is '%s' for geolocation mmdb sources. Only HTTPS is allowed for mmdb URLs." $baseUrl | fail }}
361+
{{- end }}
362+
{{- range $key, $vals := .Values.graylog.config.geolocation.mmdbSources }}
363+
{{- $name := eq $key "asn" | ternary ($key | upper) ($key | title) | printf "GeoLite2-%s" }}
364+
{{- with $vals }}
365+
{{- $url := .url }}
366+
{{- if $url }}
367+
{{- if and (not $skipChecksum) (empty .checksum) }}
368+
{{- printf "Validation error: checksum verification is enabled but no checksum hash has been provided for mmdb '%s'." $name | fail }}
369+
{{- end }}
370+
{{- if and (hasPrefix "http://" $url | not) (hasPrefix "https://" $url | not) }}
371+
{{- $url = printf "%s/%s" (trimSuffix "/" $baseUrl) (trimPrefix "/" $url) }}
372+
{{- end }}
373+
{{- if not $allowHttp | and (hasPrefix "http://" $url) }}
374+
{{- printf "Validation error: geolocation database '%s' is using URL '%s'. Only HTTPS is allowed for mmdb URLs." $name $url | fail }}
375+
{{- end }}
376+
{{- if not $skipChecksum }}
377+
{{- $url = printf "%s|%s" $url .checksum }}
378+
{{- end }}
379+
{{- $urls = printf "%s|%s" $name $url | append $urls }}
380+
{{- end }}
381+
{{- end }}
382+
{{- end }}
383+
{{- $urls | join "^" | quote }}
384+
{{- end }}
385+
{{- end }}

‎graylog/templates/config/init-graylog.yaml‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,18 @@ data:
1111
echo "Data already initialized, skipping copy."
1212
else
1313
cp -r /usr/share/graylog/data/* /mnt/data/
14-
mkdir -p /mnt/data/geolocation
1514
fi
1615
{{- if .Values.graylog.config.plugins.enabled }}
1716
# copy plugins
1817
[ -d /mnt/shared/plugins ] && find /mnt/shared/plugins/ -type f -name '*.jar' -exec cp {} /mnt/plugins/ \;
1918
2019
{{- if and .Values.graylog.config.init.assetFetch.enabled .Values.graylog.config.init.assetFetch.plugins.enabled }}
2120
# retrieve plugins directly
22-
for urlchecksum in $(echo "${GRAYLOG_PLUGIN_URLS}" | tr "^" "\n"); do
21+
for urlchecksum in $(echo "${INIT_GRAYLOG_PLUGIN_URLS}" | tr "^" "\n"); do
2322
name=$(echo "$urlchecksum" | cut -d'|' -f1)
2423
url=$(echo "$urlchecksum" | cut -d'|' -f2)
2524
checksum=$(echo "$urlchecksum" | cut -d'|' -f3)
26-
wget "$url" -O "$name.jar" || { echo "Failed to fetch plugin $name.jar at $url"; continue; }
25+
wget -T 5 -t 3 "$url" -O "$name.jar" || { echo "Failed to fetch plugin $name.jar at $url"; continue; }
2726
if [ -n "$checksum" ]; then
2827
actual=$(sha256sum "$name.jar" | awk '{print $1}')
2928
if [ "$checksum" = "$actual" ]; then
@@ -40,19 +39,34 @@ data:
4039
done
4140
{{- end }}
4241
{{- end }}
43-
# get geolocation city database
44-
if [ -n "$GRAYLOG_GEO_CITY_DB_URL" ]; then
45-
wget "$GRAYLOG_GEO_CITY_DB_URL" -O "/mnt/data/geolocation/city.mmdb" || echo "Failed to fetch geolocation database at $GRAYLOG_GEO_CITY_DB_URL"
46-
fi
47-
# get geolocation ASN database
48-
if [ -n "$GRAYLOG_GEO_ASN_DB_URL" ]; then
49-
wget "$GRAYLOG_GEO_ASN_DB_URL" -O "/mnt/data/geolocation/asn.mmdb" || echo "Failed to fetch geolocation database at $GRAYLOG_GEO_ASN_DB_URL"
50-
fi
42+
{{- if and .Values.graylog.config.geolocation.enabled .Values.graylog.config.init.assetFetch.enabled .Values.graylog.config.init.assetFetch.geolocation.enabled }}
43+
mkdir -p /mnt/data/geolocation
44+
# retrieve geolocation mmdb files directly
45+
for urlchecksum in $(echo "${INIT_GRAYLOG_GEO_MMDB_URLS}" | tr "^" "\n"); do
46+
name=$(echo "$urlchecksum" | cut -d'|' -f1)
47+
url=$(echo "$urlchecksum" | cut -d'|' -f2)
48+
checksum=$(echo "$urlchecksum" | cut -d'|' -f3)
49+
wget -T 5 -t 3 "$url" -O "$name.mmdb" || { echo "Failed to fetch geolocation database $name.mmdb at $url"; continue; }
50+
if [ -n "$checksum" ]; then
51+
actual=$(sha256sum "$name.mmdb" | awk '{print $1}')
52+
if [ "$checksum" = "$actual" ]; then
53+
echo "Checksum matches for $name.mmdb"
54+
cp "$name.mmdb" "/mnt/data/geolocation/" && rm "$name.mmdb"
55+
else
56+
echo "Checksum does NOT match for $name.mmdb. Skipping database."
57+
rm "$name.mmdb"
58+
fi
59+
else
60+
echo "Warning: no checksum validation has been performed for database $name.mmdb"
61+
cp "$name.mmdb" "/mnt/data/geolocation/" && rm "$name.mmdb"
62+
fi
63+
done
64+
{{- end }}
5165
# check mongo credentials
5266
if env | grep GRAYLOG_MONGODB_URI | grep -q "@"; then
5367
echo "MongoDB credentials set. We're good to go!"
5468
else
5569
echo "Error: MongoDB credentials not set in MongoDB URI. Make sure secrets are up to date."
5670
exit 1
5771
fi
58-
{{- end }}
72+
{{- end }}

‎graylog/templates/workload/cronjobs/geoip.yaml‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,38 +13,31 @@ data:
1313
GEOIPUPDATE_LICENSE_KEY: {{ .Values.graylog.config.geolocation.maxmindGeoIp.licenseKey | quote | trimAll "\"" | b64enc }}
1414
{{- $cronJobName := include "graylog.fullname" . | printf "%s-geoip-update" }}
1515
{{- $claimName := printf "%s-%s" (include "graylog.volumeName" .) (include "graylog.fullname" .) }}
16+
{{- $cronSchedule := .Values.graylog.config.geolocation.maxmindGeoIp.cronSchedule | default "0 0 * * *" }}
17+
{{- $hookIsEnabled := .Values.graylog.config.geolocation.maxmindGeoIp.postInstallRun | default true }}
1618
{{- range $i := include "graylog.replicas" . | default 2 | int | until }}
1719
---
1820
apiVersion: batch/v1
1921
kind: CronJob
2022
metadata:
2123
name: {{ printf "%s-%d" $cronJobName $i }}
2224
spec:
23-
schedule: "0 0 * * *" # daily
25+
schedule: {{ $cronSchedule }}
2426
jobTemplate:
2527
spec:
26-
template:
27-
spec:
28-
containers:
29-
- name: geoipupdate
30-
image: maxmindinc/geoipupdate:latest
31-
envFrom:
32-
- secretRef:
33-
name: {{ $geoSecretName }}
34-
env:
35-
- name: GEOIPUPDATE_EDITION_IDS
36-
value: "GeoLite2-City GeoLite2-ASN"
37-
- name: GEOIPUPDATE_FREQUENCY
38-
value: "0"
39-
- name: GEOIPUPDATE_DB_DIR
40-
value: "/usr/share/GeoIP/geolocation"
41-
volumeMounts:
42-
- name: geoip-db
43-
mountPath: /usr/share/GeoIP
44-
restartPolicy: OnFailure
45-
volumes:
46-
- name: geoip-db
47-
persistentVolumeClaim:
48-
claimName: {{ printf "%s-%d" $claimName $i }}
28+
{{- list $geoSecretName $claimName $i | include "graylog.geoip.job.spec" | nindent 6 }}
29+
{{- if $hookIsEnabled }}
30+
---
31+
apiVersion: batch/v1
32+
kind: Job
33+
metadata:
34+
name: {{ printf "%s-%d-hook" $cronJobName $i }}
35+
annotations:
36+
"helm.sh/hook": "post-install,post-upgrade"
37+
"helm.sh/hook-weight": "0"
38+
"helm.sh/hook-delete-policy": "before-hook-creation,hook-succeeded"
39+
spec:
40+
{{- list $geoSecretName $claimName $i | include "graylog.geoip.job.spec" | nindent 2 }}
41+
{{- end }}
4942
{{- end }}
5043
{{- end }}

‎graylog/templates/workload/statefulsets/graylog.yaml‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -61,20 +61,11 @@ spec:
6161
image: {{ include "graylog.image" . }}
6262
imagePullPolicy: {{ .Values.graylog.custom.image.imagePullPolicy }}
6363
command: [ "/bin/sh", "/scripts/init-script.sh" ]
64-
{{- $geoDirectDownload := and .Values.graylog.config.geolocation.enabled .Values.graylog.config.geolocation.directDownload.cityUrl .Values.graylog.config.geolocation.directDownload.asnUrl }}
65-
{{- if $geoDirectDownload | or .Values.graylog.config.plugins.enabled }}
6664
env:
67-
{{- if .Values.graylog.config.init.assetFetch.enabled .Values.graylog.config.init.assetFetch.plugins.enabled .Values.graylog.plugins }}
68-
- name: GRAYLOG_PLUGIN_URLS
69-
value: {{ include "graylog.pluginURLs" . }}
70-
{{- end }}
71-
{{- if $geoDirectDownload }}
72-
- name: GRAYLOG_GEO_CITY_DB_URL
73-
value: {{ .Values.graylog.config.geolocation.directDownload.cityUrl }}
74-
- name: GRAYLOG_GEO_ASN_DB_URL
75-
value: {{ .Values.graylog.config.geolocation.directDownload.asnUrl }}
76-
{{- end }}
77-
{{- end }}
65+
- name: INIT_GRAYLOG_PLUGIN_URLS
66+
value: {{ include "graylog.plugin.URLs" . | default "" }}
67+
- name: INIT_GRAYLOG_GEO_MMDB_URLS
68+
value: {{ include "graylog.mmdb.URLs" . | default "" }}
7869
envFrom:
7970
- configMapRef:
8071
name: {{ include "graylog.configmapName" . }}

‎graylog/values.yaml‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -84,15 +84,6 @@ graylog:
8484
outputBufferProcessorThreadsCorePoolSize: "3"
8585
outputBufferProcessors: ""
8686
processBufferProcessors: ""
87-
geolocation:
88-
enabled:
89-
directDownload:
90-
cityUrl:
91-
asnUrl:
92-
maxmindGeoIp:
93-
enabled:
94-
accountId:
95-
licenseKey:
9687
email:
9788
enabled: "false"
9889
senderAddress: "graylog@example.com"
@@ -106,6 +97,21 @@ graylog:
10697
webInterfaceUrl: "https://graylog.example.com"
10798
plugins:
10899
enabled: false
100+
geolocation:
101+
enabled: false
102+
maxmindGeoIp:
103+
enabled: true
104+
accountId:
105+
licenseKey:
106+
cronSchedule: "0 0 * * *"
107+
postInstallRun: true
108+
mmdbSources:
109+
city:
110+
url:
111+
checksum:
112+
asn:
113+
url:
114+
checksum:
109115
init:
110116
assetFetch:
111117
enabled: false

0 commit comments

Comments
 (0)