Update xcache list internally

Author: ponyisi

helm/servicex/templates/app/configmap.yaml

@@ -94,6 +94,11 @@ data:
     {{- else }}
     TRANSFORMER_CACHE_PREFIX = ""
     {{- end }}
+    {{- if .Values.transformer.cacheVPSSite }}
+    TRANSFORMER_CACHE_VPS_SITE = "{{ .Values.transformer.cacheVPSSite }}"
+    TRANSFORMER_CACHE_VPS_INTERVAL = {{ .Values.transformer. cacheVPSCheckInterval }}
+    TRANSFORMER_CACHE_VPS_LIVENESS_URL = "{{ .Values.transformer.cacheVPSLivenessURL }}"
+    {{- end }}
     TRANSFORMER_AUTOSCALE_ENABLED = {{- ternary "True" "False" .Values.transformer.autoscaler.enabled }}
     TRANSFORMER_CPU_LIMIT = {{ .Values.transformer.cpuLimit }}
     TRANSFORMER_MEMORY_LIMIT = "{{ .Values.transformer.memoryLimit }}"

helm/servicex/values.yaml

@@ -201,6 +201,12 @@ transformer:
   # comma-separated list of values.
   # Do not put root:// in these values.
   cachePrefix: null
+  # Alternatively, provide the virtual placement service site name to look up
+  # xCache servers from
+  cacheVPSSite: null
+  cacheVPSCheckInterval: 1800
+  cacheVPSLivenessURL: https://vps.cern.ch/liveness
+
 
   autoscaler:
     cpuScaleThreshold: 30

servicex_app/servicex_app/transformer_manager.py

@@ -220,7 +220,9 @@ def create_job_object(
             env = env + [env_var_instance_name]
 
         # provide each pod with an environment var holding cache prefix path
-        if "TRANSFORMER_CACHE_PREFIX" in current_app.config:
+        if ("TRANSFORMER_CACHE_PREFIX" in current_app.config
+                or "TRANSFORMER_CACHE_VPS_SITE" in current_app.config):
+            TransformerManager.validate_caches()
             env += [
                 client.V1EnvVar(
                     "CACHE_PREFIX", value=current_app.config["TRANSFORMER_CACHE_PREFIX"]
@@ -410,6 +412,29 @@ def persistent_volume_claim_exists(self, claim_name, namespace):
                     return False
         return False
 
+    @staticmethod
+    def validate_caches():
+        import time
+        import urllib3
+        if not (thissite := current_app.config.get("TRANSFORMER_CACHE_VPS_SITE", None)):
+            return
+
+        lastchecktime = current_app.config.get("TRANSFORMER_CACHE_VPS_LASTCHECK", 0)
+        ttl = current_app.config.get("TRANSFORMER_CACHE_VPS_INTERVAL", 30*60)
+        if ((now := time.time()) - lastchecktime > ttl):
+            current_app.config["TRANSFORMER_CACHE_VPS_LASTCHECK"] = now
+            vps_server = current_app.config["TRANSFORMER_CACHE_VPS_LIVENESS_URL"]
+            try:
+                sitedata = urllib3.request("GET", vps_server).json()
+                if thissite not in sitedata:
+                    current_app.logger.error(f"{thissite} is not in VPS liveness data")
+                servers = sorted([_["address"] for _ in sitedata[thissite].values() if _["live"]])
+                current_app.logger.info(f"Live xCache servers are {servers}")
+                current_app.config["TRANSFORMER_CACHE_PREFIX"] = ",".join(servers)
+            except Exception as e:
+                current_app.logger.error(f"Exception retrieving site data: {e}")
+
+
     @staticmethod
     def create_hpa_object(request_id, max_workers):
         target = client.V1CrossVersionObjectReference(