feat: log debug pod information before deleting pods by agent

Author: mb-ii

clearml_agent/glue/k8s.py

@@ -1037,20 +1037,73 @@ def _process_bash_lines_response(self, bash_cmd: str, raise_error=True):
         ]
         return lines
 
-    def _delete_pods(self, selectors: List[str], namespace: str, msg: str = None) -> List[str]:
-        kubectl_cmd = \
-            "kubectl delete pod -l={agent_label} " \
-            "--namespace={namespace} --field-selector={selector} --output name".format(
+    def _delete_pods(
+        self, selectors: List[str], namespace: str, msg: str = None
+    ) -> List[str]:
+        kubectl_get_cmd = (
+            "kubectl get pod -l={agent_label} "
+            "--namespace={namespace} --field-selector={selector} --output json".format(
                 selector=",".join(selectors),
                 agent_label=self._get_agent_label(),
                 namespace=namespace,
             )
-        self.log.debug("Deleting old/failed pods{} for ns {}: {}".format(
-            msg or "", namespace, kubectl_cmd
-        ))
-        lines = self._process_bash_lines_response(kubectl_cmd)
-        self.log.debug(" - deleted pods %s", ", ".join(lines))
-        return lines
+        )
+        pods_to_delete = json.loads(get_bash_output(kubectl_get_cmd))
+
+        pod_names = []
+        for pod in pods_to_delete.get("items", []):
+            pod_name = get_path(pod, "metadata", "name", default="unknown")
+            exit_code = get_path(
+                pod,
+                "status",
+                "containerStatuses",
+                "0",
+                "state",
+                "terminated",
+                "exitCode",
+                default="unknown",
+            )
+            reason = get_path(
+                pod,
+                "status",
+                "containerStatuses",
+                "0",
+                "state",
+                "terminated",
+                "reason",
+                default="unknown",
+            )
+            message = get_path(
+                pod,
+                "status",
+                "containerStatuses",
+                "0",
+                "state",
+                "terminated",
+                "message",
+                default="unknown",
+            )
+            self.log.debug(
+                "Pod {pod_name} exited with code {exit_code}, reason: {reason}, message: {message} - {msg}".format(
+                    pod_name=pod_name,
+                    exit_code=exit_code,
+                    reason=reason,
+                    message=message,
+                    msg=msg or "",
+                )
+            )
+
+            _ = get_bash_output(
+                "kubectl delete pod {pod_name} --namespace={namespace}".format(
+                    pod_name=pod_name,
+                    namespace=namespace,
+                ),
+                raise_error=False,
+            )
+            self.log.debug("Deleted pod {} - {}".format(pod_name, msg or ""))
+            pod_names.append(pod_name)
+
+        return pod_names
 
     def _delete_jobs_by_names(self, names_to_ns: Dict[str, str], msg: str = None) -> List[str]:
         if not names_to_ns: