ref(uptime): move uptime helper functions to utility file (#102105)

Sorry. But these are functions that have moved around a couple of times
due to circular dependency issues, so let's just push them into their
own file. I also looked at most of the major files in uptime to see if
there were any other obvious candidates to be moved here, but from what
I can tell, it's relatively tidy.

(I also put back the get_cluster() call I had to inline, in a preceding
PR.)

Author: klochek

src/sentry/uptime/autodetect/ranking.py

@@ -4,18 +4,15 @@
 from datetime import datetime, timedelta
 from typing import TYPE_CHECKING
 
-from django.conf import settings
-from redis.client import StrictRedis
-from rediscluster import RedisCluster
-
 from sentry.constants import UPTIME_AUTODETECTION
 from sentry.uptime.models import get_active_auto_monitor_count_for_org
 from sentry.uptime.subscriptions.subscriptions import (
     MAX_AUTO_SUBSCRIPTIONS_PER_ORG,
     MaxUrlsForDomainReachedException,
     check_url_limits,
 )
-from sentry.utils import metrics, redis
+from sentry.uptime.utils import get_cluster
+from sentry.utils import metrics
 
 if TYPE_CHECKING:
     from sentry.models.organization import Organization
@@ -37,10 +34,6 @@
 KEY_EXPIRY = ORGANIZATION_FLUSH_FREQUENCY * 2
 
 
-def _get_cluster() -> RedisCluster | StrictRedis:
-    return redis.redis_clusters.get(settings.SENTRY_UPTIME_DETECTOR_CLUSTER)
-
-
 def add_base_url_to_rank(project: Project, base_url: str):
     """
     Takes a project and valid base url and stores ranking information about it in Redis.
@@ -59,7 +52,7 @@ def add_base_url_to_rank(project: Project, base_url: str):
     larger than `RANKED_MAX_SIZE`. That shouldn't cause us problems, and is preferable to
     trimming it on every call.
     """
-    cluster = _get_cluster()
+    cluster = get_cluster()
     org_projects_key = build_org_projects_key(project.organization)
     pipeline = cluster.pipeline()
     pipeline.zincrby(org_projects_key, 1, str(project.id))
@@ -91,7 +84,7 @@ def get_candidate_projects_for_org(org: Organization) -> list[tuple[int, int]]:
     Project ids are sorted by `total_urls_seen` desc.
     """
     key = build_org_projects_key(org)
-    cluster = _get_cluster()
+    cluster = get_cluster()
     return [
         (int(project_id), count)
         for project_id, count in cluster.zrange(
@@ -105,7 +98,7 @@ def delete_candidate_projects_for_org(org: Organization) -> None:
     Deletes candidate projects related to the organization that have seen urls.
     """
     key = build_org_projects_key(org)
-    cluster = _get_cluster()
+    cluster = get_cluster()
     cluster.delete(key)
 
 
@@ -115,7 +108,7 @@ def get_candidate_urls_for_project(project: Project, limit=5) -> list[tuple[str,
     `times_url_seen` desc.
     """
     key = get_project_base_url_rank_key(project)
-    cluster = _get_cluster()
+    cluster = get_cluster()
     candidate_urls = cluster.zrange(key, 0, -1, desc=True, withscores=True, score_cast_func=int)
     urls = []
     for candidate_url, url_count in candidate_urls:
@@ -134,7 +127,7 @@ def delete_candidate_urls_for_project(project: Project) -> None:
     Deletes all current candidate rules for a project.
     """
     key = get_project_base_url_rank_key(project)
-    cluster = _get_cluster()
+    cluster = get_cluster()
     cluster.delete(key)
 
 
@@ -166,7 +159,7 @@ def get_organization_bucket(bucket: datetime) -> set[int]:
     that have projects that have seen urls.
     """
     key = get_organization_bucket_key_for_datetime(bucket)
-    cluster = _get_cluster()
+    cluster = get_cluster()
     return {int(organization_id) for organization_id in cluster.smembers(key)}
 
 
@@ -175,7 +168,7 @@ def delete_organization_bucket(bucket: datetime) -> None:
     Delete all organizations from a specific datetime bucket.
     """
     key = get_organization_bucket_key_for_datetime(bucket)
-    cluster = _get_cluster()
+    cluster = get_cluster()
     cluster.delete(key)
 
 

src/sentry/uptime/autodetect/result_handler.py

@@ -11,7 +11,6 @@
 
 from sentry import audit_log
 from sentry.uptime.autodetect.notifications import send_auto_detected_notifications
-from sentry.uptime.autodetect.ranking import _get_cluster
 from sentry.uptime.autodetect.tasks import set_failed_url
 from sentry.uptime.models import UptimeSubscription, get_audit_log_data
 from sentry.uptime.subscriptions.subscriptions import (
@@ -20,6 +19,7 @@
     update_uptime_detector,
 )
 from sentry.uptime.types import UptimeMonitorMode
+from sentry.uptime.utils import get_cluster
 from sentry.utils import metrics
 from sentry.utils.audit import create_system_audit_entry
 from sentry.workflow_engine.models.detector import Detector
@@ -52,7 +52,7 @@ def handle_onboarding_result(
     metric_tags: dict[str, str],
 ) -> None:
     if result["status"] == CHECKSTATUS_FAILURE:
-        redis = _get_cluster()
+        redis = get_cluster()
         key = build_onboarding_failure_key(detector)
         pipeline = redis.pipeline()
         pipeline.incr(key)

src/sentry/uptime/autodetect/tasks.py

@@ -14,7 +14,6 @@
 from sentry.tasks.base import instrumented_task
 from sentry.taskworker.namespaces import uptime_tasks
 from sentry.uptime.autodetect.ranking import (
-    _get_cluster,
     delete_candidate_projects_for_org,
     delete_candidate_urls_for_project,
     delete_organization_bucket,
@@ -31,6 +30,7 @@
     is_url_auto_monitored_for_project,
 )
 from sentry.uptime.types import UptimeMonitorMode
+from sentry.uptime.utils import get_cluster
 from sentry.utils import metrics
 from sentry.utils.hashlib import md5_text
 from sentry.utils.locking import UnableToAcquireLock
@@ -67,7 +67,7 @@ def schedule_autodetections():
     )
     try:
         with lock.acquire():
-            cluster = _get_cluster()
+            cluster = get_cluster()
             last_processed = cluster.get(LAST_PROCESSED_KEY)
             if last_processed is None:
                 last_processed = timezone.now().replace(second=0, microsecond=0)
@@ -260,7 +260,7 @@ def monitor_url_for_project(project: Project, url: str) -> Detector:
 
 def is_failed_url(url: str) -> bool:
     key = get_failed_url_key(url)
-    return _get_cluster().exists(key) == 1
+    return get_cluster().exists(key) == 1
 
 
 def set_failed_url(url: str) -> None:
@@ -269,7 +269,7 @@ def set_failed_url(url: str) -> None:
     """
     key = get_failed_url_key(url)
     # TODO: Jitter the expiry here, so we don't retry all at the same time.
-    _get_cluster().set(key, 1, ex=FAILED_URL_RETRY_FREQ)
+    get_cluster().set(key, 1, ex=FAILED_URL_RETRY_FREQ)
 
 
 def get_failed_url_key(url: str) -> str:

src/sentry/uptime/consumers/results_consumer.py

@@ -18,7 +18,6 @@
     ResultProcessor,
     ResultsStrategyFactory,
 )
-from sentry.uptime.autodetect.ranking import _get_cluster
 from sentry.uptime.autodetect.result_handler import handle_onboarding_result
 from sentry.uptime.consumers.eap_producer import produce_eap_uptime_result
 from sentry.uptime.grouptype import UptimePacketValue
@@ -30,8 +29,6 @@
     load_regions_for_uptime_subscription,
 )
 from sentry.uptime.subscriptions.subscriptions import (
-    build_last_seen_interval_key,
-    build_last_update_key,
     check_and_update_regions,
     disable_uptime_detector,
     remove_uptime_subscription_if_unused,
@@ -41,6 +38,7 @@
     update_remote_uptime_subscription,
 )
 from sentry.uptime.types import UptimeMonitorMode
+from sentry.uptime.utils import build_last_seen_interval_key, build_last_update_key, get_cluster
 from sentry.utils import metrics
 from sentry.workflow_engine.models.data_source import DataPacket
 from sentry.workflow_engine.models.detector import Detector
@@ -277,7 +275,7 @@ def handle_result(self, subscription: UptimeSubscription | None, result: CheckRe
             sample_rate=1.0,
         )
 
-        cluster = _get_cluster()
+        cluster = get_cluster()
         last_update_key = build_last_update_key(detector)
         last_update_raw: str | None = cluster.get(last_update_key)
         last_update_ms = 0 if last_update_raw is None else int(last_update_raw)

src/sentry/uptime/grouptype.py

@@ -15,8 +15,8 @@
 from sentry.types.group import PriorityLevel
 from sentry.uptime.endpoints.validators import UptimeDomainCheckFailureValidator
 from sentry.uptime.models import UptimeSubscription
-from sentry.uptime.subscriptions.subscriptions import build_fingerprint
 from sentry.uptime.types import GROUP_TYPE_UPTIME_DOMAIN_CHECK_FAILURE, UptimeMonitorMode
+from sentry.uptime.utils import build_fingerprint
 from sentry.utils import metrics
 from sentry.workflow_engine.handlers.detector.base import DetectorOccurrence, EventData
 from sentry.workflow_engine.handlers.detector.stateful import (

src/sentry/uptime/subscriptions/subscriptions.py

@@ -1,7 +1,6 @@
 import logging
 from collections.abc import Sequence
 
-from django.conf import settings
 from django.db import router, transaction
 from sentry_kafka_schemas.schema_types.uptime_results_v1 import (
     CHECKSTATUS_FAILURE,
@@ -40,7 +39,7 @@
     GROUP_TYPE_UPTIME_DOMAIN_CHECK_FAILURE,
     UptimeMonitorMode,
 )
-from sentry.utils import redis
+from sentry.uptime.utils import build_fingerprint, build_last_update_key, get_cluster
 from sentry.utils.db import atomic_transaction
 from sentry.utils.not_set import NOT_SET, NotSet, default_if_not_set
 from sentry.utils.outcomes import Outcome
@@ -58,22 +57,6 @@
 MAX_MONITORS_PER_DOMAIN = 100
 
 
-def build_last_update_key(detector: Detector) -> str:
-    return f"project-sub-last-update:detector:{detector.id}"
-
-
-def build_last_seen_interval_key(detector: Detector) -> str:
-    return f"project-sub-last-seen-interval:detector:{detector.id}"
-
-
-def build_detector_fingerprint_component(detector: Detector) -> str:
-    return f"uptime-detector:{detector.id}"
-
-
-def build_fingerprint(detector: Detector) -> list[str]:
-    return [build_detector_fingerprint_component(detector)]
-
-
 def resolve_uptime_issue(detector: Detector) -> None:
     """
     Sends an update to the issue platform to resolve the uptime issue for this
@@ -474,7 +457,7 @@ def disable_uptime_detector(detector: Detector, skip_quotas: bool = False):
             # start from a good state
             detector_state.update(state=DetectorPriorityLevel.OK, is_triggered=False)
 
-        cluster = redis.redis_clusters.get(settings.SENTRY_UPTIME_DETECTOR_CLUSTER)
+        cluster = get_cluster()
         last_update_key = build_last_update_key(detector)
         cluster.delete(last_update_key)
 

src/sentry/uptime/utils.py

@@ -0,0 +1,26 @@
+from django.conf import settings
+from redis.client import StrictRedis
+from rediscluster import RedisCluster
+
+from sentry.utils import redis
+from sentry.workflow_engine.models.detector import Detector
+
+
+def build_last_update_key(detector: Detector) -> str:
+    return f"project-sub-last-update:detector:{detector.id}"
+
+
+def build_last_seen_interval_key(detector: Detector) -> str:
+    return f"project-sub-last-seen-interval:detector:{detector.id}"
+
+
+def build_detector_fingerprint_component(detector: Detector) -> str:
+    return f"uptime-detector:{detector.id}"
+
+
+def build_fingerprint(detector: Detector) -> list[str]:
+    return [build_detector_fingerprint_component(detector)]
+
+
+def get_cluster() -> RedisCluster | StrictRedis:
+    return redis.redis_clusters.get(settings.SENTRY_UPTIME_DETECTOR_CLUSTER)

tests/sentry/tasks/test_post_process.py

@@ -75,7 +75,8 @@
 from sentry.testutils.skips import requires_snuba
 from sentry.types.activity import ActivityType
 from sentry.types.group import GroupSubStatus, PriorityLevel
-from sentry.uptime.autodetect.ranking import _get_cluster, get_organization_bucket_key
+from sentry.uptime.autodetect.ranking import get_organization_bucket_key
+from sentry.uptime.utils import get_cluster
 from sentry.users.services.user.service import user_service
 from sentry.utils import json
 from sentry.utils.cache import cache
@@ -2314,7 +2315,7 @@ def test_user_reports_no_shim_if_group_exists_on_report(
 class DetectBaseUrlsForUptimeTestMixin(BasePostProgressGroupMixin):
     def assert_organization_key(self, organization: Organization, exists: bool) -> None:
         key = get_organization_bucket_key(organization)
-        cluster = _get_cluster()
+        cluster = get_cluster()
         assert exists == cluster.sismember(key, str(organization.id))
 
     def test_uptime_detection_feature_url(self) -> None:

tests/sentry/uptime/autodetect/test_detector.py

@@ -2,13 +2,14 @@
 from sentry.testutils.cases import UptimeTestCase
 from sentry.testutils.helpers.options import override_options
 from sentry.uptime.autodetect.detector import autodetect_base_url_for_project
-from sentry.uptime.autodetect.ranking import _get_cluster, get_organization_bucket_key
+from sentry.uptime.autodetect.ranking import get_organization_bucket_key
+from sentry.uptime.utils import get_cluster
 
 
 class DetectBaseUrlForProjectTest(UptimeTestCase):
     def assert_organization_key(self, organization: Organization, exists: bool) -> None:
         key = get_organization_bucket_key(organization)
-        cluster = _get_cluster()
+        cluster = get_cluster()
         assert exists == cluster.sismember(key, str(organization.id))
 
     def test(self) -> None:

tests/sentry/uptime/autodetect/test_ranking.py

@@ -5,7 +5,6 @@
 from sentry.models.project import Project
 from sentry.testutils.cases import UptimeTestCase
 from sentry.uptime.autodetect.ranking import (
-    _get_cluster,
     add_base_url_to_rank,
     build_org_projects_key,
     delete_candidate_urls_for_project,
@@ -17,14 +16,15 @@
     should_autodetect_for_organization,
     should_autodetect_for_project,
 )
+from sentry.uptime.utils import get_cluster
 
 
 class AddBaseUrlToRankTest(UptimeTestCase):
     def assert_project_count(
         self, project: Project, count: int | None, expiry: int | None
     ) -> int | None:
         key = build_org_projects_key(project.organization)
-        cluster = _get_cluster()
+        cluster = get_cluster()
         if count is None:
             assert not cluster.zscore(key, str(project.id))
             return None
@@ -36,7 +36,7 @@ def assert_url_count(
         self, project: Project, url: str, count: int | None, expiry: int | None
     ) -> int | None:
         key = get_project_base_url_rank_key(project)
-        cluster = _get_cluster()
+        cluster = get_cluster()
         if count is None:
             assert cluster.zscore(key, url) is None
             return None
@@ -45,7 +45,7 @@ def assert_url_count(
             return self.check_expiry(key, expiry)
 
     def check_expiry(self, key: str, expiry: int | None) -> int:
-        cluster = _get_cluster()
+        cluster = get_cluster()
         ttl = cluster.ttl(key)
         if expiry is None:
             assert ttl > 0
@@ -94,7 +94,7 @@ def test_trim(self) -> None:
             url_1 = "https://sentry.io"
             url_2 = "https://sentry.sentry.io"
             url_3 = "https://santry.sentry.io"
-            cluster = _get_cluster()
+            cluster = get_cluster()
             add_base_url_to_rank(self.project, url_1)
             add_base_url_to_rank(self.project, url_1)
             add_base_url_to_rank(self.project, url_1)