From 737528134bc181d91561404f7d5c7e4fd96074f7 Mon Sep 17 00:00:00 2001
From: Sanjay Srikakulam <sanjaysrikakulam@gmail.com>
Date: Thu, 19 Jun 2025 18:04:39 +0000
Subject: [PATCH] Deploy TPV Brokers consumer script on the EU

to consumer messages produced and pushed to the MQ by the Pulsar endpoints
---
 group_vars/maintenance.yml | 20 ++++++++++++++++++++
 maintenance.yml            |  1 +
 requirements.yaml          |  3 +++
 3 files changed, 24 insertions(+)

diff --git a/group_vars/maintenance.yml b/group_vars/maintenance.yml
index 32318e99e..78ca2c2eb 100644
--- a/group_vars/maintenance.yml
+++ b/group_vars/maintenance.yml
@@ -8,6 +8,7 @@ galaxy_config_dir: "{{ galaxy_root }}/config"
 galaxy_config_file: "{{ galaxy_config_dir }}/galaxy.yml"
 galaxy_mutable_config_dir: "{{ galaxy_root }}/mutable-config"
 galaxy_log_dir: "/var/log/galaxy"
+galaxy_virtualenv_python: "{{ galaxy_venv_dir }}/bin/python"
 
 galaxy_group:
   name: galaxy
@@ -345,6 +346,13 @@ telegraf_plugins_extra:
       - timeout = "10s"
       - data_format = "influx"
       - interval = "24h"
+  tpv_broker_pulsar_consumer:
+    plugin: "exec"
+    config:
+      - commands = ["{{ consumer_venv_dir }} {{ pulsar_consumer_dir }}/pulsar_metric_consumer.py {{ consumer_galaxy_job_conf }}"]
+      - timeout = "10s"
+      - data_format = "influx"
+      - interval = "1m"
 
 # Role: hxr.monitor-cluster
 monitor_condor: true
@@ -412,3 +420,15 @@ walle_envs_database:
 walle_cron_day: "*"
 walle_cron_hour: "*"
 walle_cron_minute: "0"
+
+# Role: usegalaxy_eu.ansible-pulsar-util
+pulsar_metric_role: "consumer"
+consumer_influx_address: '{{ influxdb.host }}'
+consumer_influx_port: 8086
+consumer_influx_db: '{{ influxdb.node.database }}'
+consumer_influx_username: '{{ influxdb.node.username }}'
+consumer_influx_password: '{{ influxdb.node.password }}'
+consumer_influx_measurement: "pulsars_htcondor_cluster_usage"
+pulsar_consumer_dir: "/opt/tpv_broker_pulsar_consumer"
+consumer_venv_dir: "{{ pulsar_consumer_dir }}/venv"
+consumer_galaxy_job_conf: "{{ galaxy_config_dir }}/job_conf.yml"
diff --git a/maintenance.yml b/maintenance.yml
index 9701ec546..5b2fcb49f 100644
--- a/maintenance.yml
+++ b/maintenance.yml
@@ -141,6 +141,7 @@
     - usegalaxy-eu.remove-orphan-condor-jobs
     - ssh_hardening
     - usegalaxy-eu.job-radar-stats-influxdb
+    - usegalaxy_eu.ansible-pulsar-util
     - dj-wasabi.telegraf
     # - usegalaxy-eu.fix-stop-ITs
     - usegalaxy-eu.logrotate
diff --git a/requirements.yaml b/requirements.yaml
index 26459b818..b799b356a 100644
--- a/requirements.yaml
+++ b/requirements.yaml
@@ -168,5 +168,8 @@ roles:
   - name: usegalaxy_eu.ansible_tpv_broker
     src: https://github.com/usegalaxy-eu/ansible-tpv-broker
     version: main
+  - name: usegalaxy_eu.ansible-pulsar-util
+    src: https://github.com/usegalaxy-eu/ansible-pulsar-util
+    version: main
   - name: geerlingguy.swap
     version: 1.2.0