From e5ca819c43595a3771a84c5cffbb6545c73f16f9 Mon Sep 17 00:00:00 2001 From: Davide Date: Thu, 6 Jul 2023 16:46:35 +0200 Subject: [PATCH 1/2] Allow selecting which celery worker to ping --- docs/settings.rst | 23 ++++++++++ health_check/contrib/celery_ping/backends.py | 16 +++++-- tests/test_celery_ping.py | 46 +++++++++++++++++++- 3 files changed, 80 insertions(+), 5 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index 18892f7d..0065f3dd 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -117,3 +117,26 @@ Using `django.settings` you may exert more fine-grained control over the behavio - Number - `None` - Specifies the healthcheck task priority. + + +Celery-Ping Health Check +------------------------ + +Using `django.settings` you may exert more fine-grained control over the behavior of the celery-ping health check + +.. list-table:: Additional Settings + :widths: 25 10 10 55 + :header-rows: 1 + + * - Name + - Type + - Default + - Description + * - `HEALTHCHECK_CELERY_PING_TIMEOUT` + - Number + - `1` + - Specifies the maximum total time (in seconds) for which "pong" responses are awaited. + * - `HEALTHCHECK_CELERY_PING_DESTINATION` + - List of Strings + - `None` + - Specifies the list of workers which will receive the "ping" request. diff --git a/health_check/contrib/celery_ping/backends.py b/health_check/contrib/celery_ping/backends.py index 95d2de7f..5ffa615a 100644 --- a/health_check/contrib/celery_ping/backends.py +++ b/health_check/contrib/celery_ping/backends.py @@ -10,9 +10,10 @@ class CeleryPingHealthCheck(BaseHealthCheckBackend): def check_status(self): timeout = getattr(settings, "HEALTHCHECK_CELERY_PING_TIMEOUT", 1) + destination = getattr(settings, "HEALTHCHECK_CELERY_PING_DESTINATION", None) try: - ping_result = app.control.ping(timeout=timeout) + ping_result = app.control.ping(destination=destination, timeout=timeout) except OSError as e: self.add_error(ServiceUnavailable("IOError"), e) except NotImplementedError as exc: @@ -28,9 +29,9 @@ def check_status(self): ServiceUnavailable("Celery workers unavailable"), ) else: - self._check_ping_result(ping_result) + self._check_ping_result(ping_result, destination) - def _check_ping_result(self, ping_result): + def _check_ping_result(self, ping_result, destination): active_workers = [] for result in ping_result: @@ -42,6 +43,15 @@ def _check_ping_result(self, ping_result): continue active_workers.append(worker) + if destination: + inactive_workers = set(destination) - set(active_workers) + if inactive_workers: + self.add_error( + ServiceUnavailable( + f"Celery workers {inactive_workers} did not respond" + ) + ) + if not self.errors: self._check_active_queues(active_workers) diff --git a/tests/test_celery_ping.py b/tests/test_celery_ping.py index 0dda4871..82365cc4 100644 --- a/tests/test_celery_ping.py +++ b/tests/test_celery_ping.py @@ -2,7 +2,6 @@ import pytest from django.apps import apps -from django.conf import settings from health_check.contrib.celery_ping.apps import HealthCheckConfig from health_check.contrib.celery_ping.backends import CeleryPingHealthCheck @@ -18,7 +17,9 @@ class TestCeleryPingHealthCheck: def health_check(self): return CeleryPingHealthCheck() - def test_check_status_doesnt_add_errors_when_ping_successful(self, health_check): + def test_check_status_doesnt_add_errors_when_ping_successful( + self, health_check, settings + ): celery_worker = "celery@4cc150a7b49b" with ( @@ -54,6 +55,7 @@ def test_check_status_reports_errors_if_ping_responses_are_incorrect(self, healt def test_check_status_adds_errors_when_ping_successfull_but_not_all_defined_queues_have_consumers( self, health_check, + settings, ): celery_worker = "celery@4cc150a7b49b" queues = list(settings.CELERY_QUEUES) @@ -111,6 +113,46 @@ def test_check_status_add_error_when_ping_result_failed(self, ping_result, healt assert len(health_check.errors) == 1 assert "workers unavailable" in health_check.errors[0].message.lower() + def test_check_status_reports_errors_if_ping_responses_are_missing( + self, + health_check, + settings, + ): + settings.HEALTHCHECK_CELERY_PING_DESTINATION = [ + "celery1@4cc150a7b49b", + "celery2@4cc150a7b49b", + ] + with patch( + self.CELERY_APP_CONTROL_PING, + return_value=[ + {"celery1@4cc150a7b49b": CeleryPingHealthCheck.CORRECT_PING_RESPONSE}, + ], + ): + health_check.check_status() + + assert len(health_check.errors) == 1 + + def test_check_status_reports_destinations( + self, + health_check, + settings, + ): + settings.HEALTHCHECK_CELERY_PING_DESTINATION = [ + "celery1@4cc150a7b49b", + "celery2@4cc150a7b49b", + ] + with patch( + self.CELERY_APP_CONTROL_PING, + return_value=[ + {"celery1@4cc150a7b49b": CeleryPingHealthCheck.CORRECT_PING_RESPONSE}, + {"celery2@4cc150a7b49b": CeleryPingHealthCheck.CORRECT_PING_RESPONSE}, + {"celery3@4cc150a7b49b": CeleryPingHealthCheck.CORRECT_PING_RESPONSE}, + ], + ): + health_check.check_status() + + assert len(health_check.errors) == 1 + class TestCeleryPingHealthCheckApps: def test_apps(self): From b4b03eeb185e90ce456b2a2893916bb1c6f7f721 Mon Sep 17 00:00:00 2001 From: Davide Date: Thu, 14 Mar 2024 12:55:39 +0100 Subject: [PATCH 2/2] Fix test with multiple destionations --- tests/test_celery_ping.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_celery_ping.py b/tests/test_celery_ping.py index 82365cc4..b9b95250 100644 --- a/tests/test_celery_ping.py +++ b/tests/test_celery_ping.py @@ -148,10 +148,18 @@ def test_check_status_reports_destinations( {"celery2@4cc150a7b49b": CeleryPingHealthCheck.CORRECT_PING_RESPONSE}, {"celery3@4cc150a7b49b": CeleryPingHealthCheck.CORRECT_PING_RESPONSE}, ], + ), patch( + self.CELERY_APP_CONTROL_INSPECT_ACTIVE_QUEUES, + return_value={ + celery_worker: [ + {"name": queue.name} for queue in settings.CELERY_QUEUES + ] + for celery_worker in ("celery1@4cc150a7b49b", "celery2@4cc150a7b49b", "celery3@4cc150a7b49b") + }, ): health_check.check_status() - assert len(health_check.errors) == 1 + assert not health_check.errors class TestCeleryPingHealthCheckApps: