Skip to content

Commit 58ad87b

Browse files
battermannbattermann
committed
configure brig cassandra client for background worker
1 parent fb9ce89 commit 58ad87b

File tree

14 files changed

+116
-36
lines changed

14 files changed

+116
-36
lines changed

‎charts/background-worker/README.md‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
1-
Note that background-worker depends on some provisioned storage, namely:
1+
Note that background-worker depends on some provisioned storage/services, namely:
22

33
- rabbitmq
4+
- postgresql
5+
- cassandra (two clusters)
6+
7+
PostgreSQL configuration
8+
- Set connection parameters under `config.postgresql` (libpq keywords: `host`, `port`, `user`, `dbname`, etc.).
9+
- Provide the password via `secrets.pgPassword`; it is mounted at `/etc/wire/background-worker/secrets/pgPassword` and referenced from the configmap.
10+
11+
Cassandra configuration
12+
- Background-worker connects to two Cassandra clusters:
13+
- `config.cassandraGundeck` (keyspace: `gundeck`) for the dead user notification watcher.
14+
- `config.cassandraBrig` (keyspace: `brig`) for the user store.
15+
- TLS may be configured via either a reference (`tlsCaSecretRef`) or inline CA (`tlsCa`) for each cluster. Secrets mount under:
16+
- `/etc/wire/background-worker/cassandra-gundeck`
17+
- `/etc/wire/background-worker/cassandra-brig`
418

519
These are dealt with independently from this chart.

‎charts/background-worker/templates/_helpers.tpl‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,26 @@
88
{{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}}
99
{{- end -}}
1010

11-
{{- define "useCassandraTLS" -}}
12-
{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }}
11+
{{- define "useGundeckCassandraTLS" -}}
12+
{{ or (hasKey .cassandraGundeck "tlsCa") (hasKey .cassandraGundeck "tlsCaSecretRef") }}
1313
{{- end -}}
1414

15-
{{/* Return a Dict of TLS CA secret name and key
16-
This is used to switch between provided secret (e.g. by cert-manager) and
17-
created one (in case the CA is provided as PEM string.)
18-
*/}}
19-
{{- define "tlsSecretRef" -}}
20-
{{- if .cassandra.tlsCaSecretRef -}}
21-
{{ .cassandra.tlsCaSecretRef | toYaml }}
15+
{{- define "useBrigCassandraTLS" -}}
16+
{{ or (hasKey .cassandraBrig "tlsCa") (hasKey .cassandraBrig "tlsCaSecretRef") }}
17+
{{- end -}}
18+
19+
{{- define "gundeckTlsSecretRef" -}}
20+
{{- if .cassandraGundeck.tlsCaSecretRef -}}
21+
{{ .cassandraGundeck.tlsCaSecretRef | toYaml }}
22+
{{- else }}
23+
{{- dict "name" "background-worker-cassandra-gundeck" "key" "ca.pem" | toYaml -}}
24+
{{- end -}}
25+
{{- end -}}
26+
27+
{{- define "brigTlsSecretRef" -}}
28+
{{- if .cassandraBrig.tlsCaSecretRef -}}
29+
{{ .cassandraBrig.tlsCaSecretRef | toYaml }}
2230
{{- else }}
23-
{{- dict "name" "background-worker-cassandra" "key" "ca.pem" | toYaml -}}
31+
{{- dict "name" "background-worker-cassandra-brig" "key" "ca.pem" | toYaml -}}
2432
{{- end -}}
2533
{{- end -}}

‎charts/background-worker/templates/cassandra-secret.yaml‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,30 @@
1-
{{/* Secret for the provided Cassandra TLS CA. */}}
2-
{{- if not (empty .Values.config.cassandra.tlsCa) }}
1+
{{/* Secrets for provided Cassandra TLS CAs */}}
2+
{{- if not (empty .Values.config.cassandraGundeck.tlsCa) }}
33
apiVersion: v1
44
kind: Secret
55
metadata:
6-
name: background-worker-cassandra
6+
name: background-worker-cassandra-gundeck
77
labels:
88
app: background-worker
99
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
1010
release: "{{ .Release.Name }}"
1111
heritage: "{{ .Release.Service }}"
1212
type: Opaque
1313
data:
14-
ca.pem: {{ .Values.config.cassandra.tlsCa | b64enc | quote }}
14+
ca.pem: {{ .Values.config.cassandraGundeck.tlsCa | b64enc | quote }}
15+
{{- end }}
16+
{{- if not (empty .Values.config.cassandraBrig.tlsCa) }}
17+
---
18+
apiVersion: v1
19+
kind: Secret
20+
metadata:
21+
name: background-worker-cassandra-brig
22+
labels:
23+
app: background-worker
24+
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
25+
release: "{{ .Release.Name }}"
26+
heritage: "{{ .Release.Service }}"
27+
type: Opaque
28+
data:
29+
ca.pem: {{ .Values.config.cassandraBrig.tlsCa | b64enc | quote }}
1530
{{- end }}

‎charts/background-worker/templates/configmap.yaml‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,22 @@ data:
2121
host: federator
2222
port: 8080
2323
24-
cassandra:
24+
cassandraGundeck:
2525
endpoint:
26-
host: {{ .cassandra.host }}
26+
host: {{ .cassandraGundeck.host }}
2727
port: 9042
2828
keyspace: gundeck
29-
{{- if eq (include "useCassandraTLS" .) "true" }}
30-
tlsCa: /etc/wire/background-worker/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }}
29+
{{- if eq (include "useGundeckCassandraTLS" .) "true" }}
30+
tlsCa: /etc/wire/background-worker/cassandra-gundeck/{{- (include "gundeckTlsSecretRef" . | fromYaml).key }}
31+
{{- end }}
32+
33+
cassandraBrig:
34+
endpoint:
35+
host: {{ .cassandraBrig.host }}
36+
port: 9042
37+
keyspace: brig
38+
{{- if eq (include "useBrigCassandraTLS" .) "true" }}
39+
tlsCa: /etc/wire/background-worker/cassandra-brig/{{- (include "brigTlsSecretRef" . | fromYaml).key }}
3140
{{- end }}
3241
3342
{{- with .rabbitmq }}

‎charts/background-worker/templates/deployment.yaml‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,15 @@ spec:
3939
- name: "background-worker-secrets"
4040
secret:
4141
secretName: "background-worker"
42-
{{- if eq (include "useCassandraTLS" .Values.config) "true" }}
43-
- name: "background-worker-cassandra"
42+
{{- if eq (include "useGundeckCassandraTLS" .Values.config) "true" }}
43+
- name: "background-worker-cassandra-gundeck"
4444
secret:
45-
secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }}
45+
secretName: {{ (include "gundeckTlsSecretRef" .Values.config | fromYaml).name }}
46+
{{- end }}
47+
{{- if eq (include "useBrigCassandraTLS" .Values.config) "true" }}
48+
- name: "background-worker-cassandra-brig"
49+
secret:
50+
secretName: {{ (include "brigTlsSecretRef" .Values.config | fromYaml).name }}
4651
{{- end }}
4752
{{- if .Values.config.rabbitmq.tlsCaSecretRef }}
4853
- name: "rabbitmq-ca"
@@ -62,9 +67,13 @@ spec:
6267
mountPath: "/etc/wire/background-worker/secrets"
6368
- name: "background-worker-config"
6469
mountPath: "/etc/wire/background-worker/conf"
65-
{{- if eq (include "useCassandraTLS" .Values.config) "true" }}
66-
- name: "background-worker-cassandra"
67-
mountPath: "/etc/wire/background-worker/cassandra"
70+
{{- if eq (include "useGundeckCassandraTLS" .Values.config) "true" }}
71+
- name: "background-worker-cassandra-gundeck"
72+
mountPath: "/etc/wire/background-worker/cassandra-gundeck"
73+
{{- end }}
74+
{{- if eq (include "useBrigCassandraTLS" .Values.config) "true" }}
75+
- name: "background-worker-cassandra-brig"
76+
mountPath: "/etc/wire/background-worker/cassandra-brig"
6877
{{- end }}
6978
{{- if .Values.config.rabbitmq.tlsCaSecretRef }}
7079
- name: "rabbitmq-ca"

‎charts/background-worker/values.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,10 @@ config:
4040
# tlsCaSecretRef:
4141
# name: <secret-name>
4242
# key: <ca-attribute>
43-
cassandra:
43+
# Cassandra clusters used by background-worker
44+
cassandraGundeck:
45+
host: aws-cassandra
46+
cassandraBrig:
4447
host: aws-cassandra
4548

4649
backendNotificationPusher:

‎hack/helm_vars/wire-server/values.yaml.gotmpl‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ brig:
5858
teamCreatorWelcome: https://teams.wire.com/login
5959
teamMemberWelcome: https://wire.com/download
6060
accountPages: https://account.wire.com
61+
# Background-worker uses Brig's Cassandra keyspace.
6162
cassandra:
6263
host: {{ .Values.cassandraHost }}
6364
replicaCount: 1
@@ -607,7 +608,16 @@ background-worker:
607608
concurrency: 8
608609
jobTimeout: 60
609610
maxAttempts: 3
610-
cassandra:
611+
# Cassandra clusters used by background-worker
612+
cassandraGundeck:
613+
host: {{ .Values.cassandraHost }}
614+
replicaCount: 1
615+
{{- if .Values.useK8ssandraSSL.enabled }}
616+
tlsCaSecretRef:
617+
name: "cassandra-jks-keystore"
618+
key: "ca.crt"
619+
{{- end }}
620+
cassandraBrig:
611621
host: {{ .Values.cassandraHost }}
612622
replicaCount: 1
613623
{{- if .Values.useK8ssandraSSL.enabled }}

‎services/background-worker/background-worker.integration.yaml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,18 @@ federatorInternal:
88
host: 127.0.0.1
99
port: 8097
1010

11-
cassandra:
11+
cassandraGundeck:
1212
endpoint:
1313
host: 127.0.0.1
1414
port: 9042
1515
keyspace: gundeck_test
1616

17+
cassandraBrig:
18+
endpoint:
19+
host: 127.0.0.1
20+
port: 9042
21+
keyspace: brig_test
22+
1723
rabbitmq:
1824
host: 127.0.0.1
1925
port: 5671

‎services/background-worker/src/Wire/BackgroundWorker/Env.hs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ data Env = Env
5555
backgroundJobsConfig :: BackgroundJobsConfig,
5656
workerRunningGauge :: Vector Text Gauge,
5757
statuses :: IORef (Map Worker IsWorking),
58-
cassandra :: ClientState,
58+
gundeckCassandra :: ClientState,
59+
brigCassandra :: ClientState,
5960
hasqlPool :: HasqlPool.Pool,
6061
backgroundJobsQueue :: MVar Q.Channel
6162
}
@@ -80,7 +81,8 @@ mkWorkerRunningGauge =
8081
mkEnv :: Opts -> IO Env
8182
mkEnv opts = do
8283
logger <- Log.mkLogger opts.logLevel Nothing opts.logFormat
83-
cassandra <- defInitCassandra opts.cassandra logger
84+
gundeckCassandra <- defInitCassandra opts.cassandraGundeck logger
85+
brigCassandra <- defInitCassandra opts.cassandraBrig logger
8486
http2Manager <- initHttp2Manager
8587
httpManager <- newManager defaultManagerSettings
8688
let federatorInternal = opts.federatorInternal

‎services/background-worker/src/Wire/BackgroundWorker/Jobs/Registry.hs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ dispatchJob job = do
2828
. mapError @BackgroundJobError (T.pack . show)
2929
. mapError @UsageError (T.pack . show)
3030
. runInputConst @Pool env.hasqlPool
31-
. interpretUserStoreCassandra env.cassandra
31+
. interpretUserStoreCassandra env.brigCassandra
3232
. interpretUserGroupStoreToPostgres
3333
. runInputSem (readMVar env.backgroundJobsQueue)
3434
. interpretBackgroundJobsPublisherRabbitMQ job.requestId

0 commit comments

Comments
 (0)