diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bf8a56c91c0d63..c8149d51b81b4a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -165,9 +165,6 @@ pnpm-lock.yaml @getsentry/owners-js-de /static/app/views/settings/projectAlerts/ @getsentry/alerts-notifications /static/app/views/alerts/ @getsentry/alerts-notifications /static/app/views/alerts/rules/uptime @getsentry/crons -/static/app/views/releases/ @getsentry/replay-frontend -/static/app/views/releases/drawer/ @getsentry/replay-frontend -/static/app/views/releases/releaseBubbles/ @getsentry/replay-frontend /src/sentry/rules/processing/delayed_processing.py @getsentry/alerts-notifications /static/app/views/settings/account/notifications/ @getsentry/alerts-notifications @@ -448,10 +445,19 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /tests/sentry/auth/test_superuser.py @getsentry/enterprise /tests/sentry/middleware/test_staff.py @getsentry/enterprise /tests/sentry/integrations/github/ @getsentry/enterprise -/tests/sentry/models/releases/ @getsentry/replay-backend ## End of Enterprise +## Releases +/tests/sentry/models/releases/ @getsentry/replay-backend +/src/sentry/releases/ @getsentry/replay-backend +/tests/sentry/releases/ @getsentry/replay-backend +/static/app/views/releases/ @getsentry/replay-frontend +/static/app/views/releases/drawer/ @getsentry/replay-frontend +/static/app/views/releases/releaseBubbles/ @getsentry/replay-frontend +## End of Releases + + ## APIs /src/sentry/apidocs/ @getsentry/owners-api /src/sentry/api/urls.py @getsentry/owners-api diff --git a/src/sentry/models/release.py b/src/sentry/models/release.py index 0cb7e77a3a0a3a..b2c935b6a3eaed 100644 --- a/src/sentry/models/release.py +++ b/src/sentry/models/release.py @@ -1,820 +1,17 @@ -# Sort commit list in reverse order -from __future__ import annotations - -import logging -from collections.abc import Mapping, Sequence -from typing import ClassVar, Literal, TypedDict - -import orjson -import sentry_sdk -from django.contrib.postgres.fields.array import ArrayField -from django.db import IntegrityError, models, router -from django.db.models import Case, Exists, F, Func, OuterRef, Sum, When -from django.utils import timezone -from django.utils.functional import cached_property -from django.utils.translation import gettext_lazy as _ -from sentry_relay.exceptions import RelayError -from sentry_relay.processing import parse_release - -from sentry.backup.scopes import RelocationScope -from sentry.constants import BAD_RELEASE_CHARS, COMMIT_RANGE_DELIMITER -from sentry.db.models import ( - BoundedBigIntegerField, - BoundedPositiveIntegerField, - FlexibleForeignKey, - JSONField, - Model, - region_silo_model, - sane_repr, -) -from sentry.db.models.fields.hybrid_cloud_foreign_key import HybridCloudForeignKey -from sentry.db.models.indexes import IndexWithPostgresNameLimits -from sentry.db.models.manager.base import BaseManager -from sentry.models.artifactbundle import ArtifactBundle -from sentry.models.commitauthor import CommitAuthor -from sentry.models.releases.constants import ( - DB_VERSION_LENGTH, - ERR_RELEASE_HEALTH_DATA, - ERR_RELEASE_REFERENCED, +from sentry.releases.models.release import ( + Release, + ReleaseStatus, + _get_cache_key, + follows_semver_versioning_scheme, + get_artifact_counts, + get_previous_release, ) -from sentry.models.releases.exceptions import UnsafeReleaseDeletion -from sentry.models.releases.release_project import ReleaseProject -from sentry.models.releases.util import ReleaseQuerySet, SemverFilter, SemverVersion -from sentry.utils import metrics -from sentry.utils.cache import cache -from sentry.utils.db import atomic_transaction -from sentry.utils.hashlib import hash_values, md5_text -from sentry.utils.numbers import validate_bigint -from sentry.utils.sdk import set_span_attribute - -logger = logging.getLogger(__name__) - - -class _CommitDataKwargs(TypedDict, total=False): - author: CommitAuthor - message: str - date_added: str - - -class ReleaseStatus: - OPEN = 0 - ARCHIVED = 1 - - @classmethod - def from_string(cls, value): - if value == "open": - return cls.OPEN - elif value == "archived": - return cls.ARCHIVED - else: - raise ValueError(repr(value)) - - @classmethod - def to_string(cls, value): - # XXX(markus): Since the column is nullable we need to handle `null` here. - # However `null | undefined` in request payloads means "don't change - # status of release". This is why `from_string` does not consider - # `null` valid. - # - # We could remove `0` as valid state and only have `null` but I think - # that would make things worse. - # - # Eventually we should backfill releasestatus to 0 - if value is None or value == ReleaseStatus.OPEN: - return "open" - elif value == ReleaseStatus.ARCHIVED: - return "archived" - else: - raise ValueError(repr(value)) - - -def _get_cache_key(project_id: int, group_id: int, first: bool) -> str: - return f"g-r:{group_id}-{project_id}-{first}" - - -class ReleaseModelManager(BaseManager["Release"]): - def get_queryset(self) -> ReleaseQuerySet: - return ReleaseQuerySet(self.model, using=self._db) - - def annotate_prerelease_column(self): - return self.get_queryset().annotate_prerelease_column() - - def filter_to_semver(self) -> ReleaseQuerySet: - return self.get_queryset().filter_to_semver() - - def filter_by_semver_build( - self, - organization_id: int, - operator: str, - build: str, - project_ids: Sequence[int] | None = None, - negated: bool = False, - ) -> models.QuerySet: - return self.get_queryset().filter_by_semver_build( - organization_id, - operator, - build, - project_ids, - negated=negated, - ) - - def filter_by_semver( - self, - organization_id: int, - semver_filter: SemverFilter, - project_ids: Sequence[int] | None = None, - ) -> models.QuerySet: - return self.get_queryset().filter_by_semver(organization_id, semver_filter, project_ids) - - def filter_by_stage( - self, - organization_id: int, - operator: str, - value, - project_ids: Sequence[int] | None = None, - environments: list[str] | None = None, - ) -> models.QuerySet: - return self.get_queryset().filter_by_stage( - organization_id, operator, value, project_ids, environments - ) - - def order_by_recent(self): - return self.get_queryset().order_by_recent() - - def _get_group_release_version(self, group_id: int, orderby: str) -> str: - from sentry.models.grouprelease import GroupRelease - - # Using `id__in()` because there is no foreign key relationship. - return self.get( - id__in=GroupRelease.objects.filter(group_id=group_id) - .order_by(orderby) - .values("release_id")[:1] - ).version - - def get_group_release_version( - self, project_id: int, group_id: int, first: bool = True, use_cache: bool = True - ) -> str | None: - cache_key = _get_cache_key(project_id, group_id, first) - - release_version: Literal[False] | str | None = cache.get(cache_key) if use_cache else None - if release_version is False: - # We've cached the fact that no rows exist. - return None - - if release_version is None: - # Cache miss or not use_cache. - orderby = "first_seen" if first else "-last_seen" - try: - release_version = self._get_group_release_version(group_id, orderby) - except Release.DoesNotExist: - release_version = False - cache.set(cache_key, release_version, 3600) - - # Convert the False back into a None. - return release_version or None - - -@region_silo_model -class Release(Model): - """ - A release is generally created when a new version is pushed into a - production state. - - A commit is generally a git commit. See also releasecommit.py - """ - - __relocation_scope__ = RelocationScope.Excluded - - organization = FlexibleForeignKey("sentry.Organization") - projects = models.ManyToManyField( - "sentry.Project", related_name="releases", through=ReleaseProject - ) - status = BoundedPositiveIntegerField( - default=ReleaseStatus.OPEN, - null=True, - choices=( - (ReleaseStatus.OPEN, _("Open")), - (ReleaseStatus.ARCHIVED, _("Archived")), - ), - ) - - version = models.CharField(max_length=DB_VERSION_LENGTH) - # ref might be the branch name being released - ref = models.CharField(max_length=DB_VERSION_LENGTH, null=True, blank=True) - url = models.URLField(null=True, blank=True) - date_added = models.DateTimeField(default=timezone.now) - # DEPRECATED - not available in UI or editable from API - date_started = models.DateTimeField(null=True, blank=True) - date_released = models.DateTimeField(null=True, blank=True) - # arbitrary data recorded with the release - data = JSONField(default=dict) - # generally the release manager, or the person initiating the process - owner_id = HybridCloudForeignKey("sentry.User", on_delete="SET_NULL", null=True, blank=True) - - # materialized stats - commit_count = BoundedPositiveIntegerField(null=True, default=0) - last_commit_id = BoundedBigIntegerField(null=True) - authors = ArrayField(models.TextField(), default=list, null=True) - total_deploys = BoundedPositiveIntegerField(null=True, default=0) - last_deploy_id = BoundedPositiveIntegerField(null=True) - - # Denormalized semver columns. These will be filled if `version` matches at least - # part of our more permissive model of semver: - # `@...-+ - package = models.TextField(null=True) - major = models.BigIntegerField(null=True) - minor = models.BigIntegerField(null=True) - patch = models.BigIntegerField(null=True) - revision = models.BigIntegerField(null=True) - prerelease = models.TextField(null=True) - build_code = models.TextField(null=True) - # If `build_code` can be parsed as a 64 bit int we'll store it here as well for - # sorting/comparison purposes - build_number = models.BigIntegerField(null=True) - - # HACK HACK HACK - # As a transitional step we permit release rows to exist multiple times - # where they are "specialized" for a specific project. The goal is to - # later split up releases by project again. This is for instance used - # by the org release listing. - _for_project_id: int | None = None - # the user agent that set the release - user_agent = models.TextField(null=True) - - # Custom Model Manager required to override create method - objects: ClassVar[ReleaseModelManager] = ReleaseModelManager() - - class Meta: - app_label = "sentry" - db_table = "sentry_release" - unique_together = (("organization", "version"),) - indexes = [ - models.Index( - fields=["organization", "version"], - opclasses=["", "text_pattern_ops"], - name="sentry_release_version_btree", - ), - # We also use a functional index to order `prerelease` according to semver rules, - IndexWithPostgresNameLimits( - "organization", - "package", - F("major").desc(), - F("minor").desc(), - F("patch").desc(), - F("revision").desc(), - Case(When(prerelease="", then=1), default=0).desc(), - F("prerelease").desc(), - name="sentry_release_semver_by_package_idx", - ), - models.Index( - "organization", - F("major").desc(), - F("minor").desc(), - F("patch").desc(), - F("revision").desc(), - Case(When(prerelease="", then=1), default=0).desc(), - F("prerelease").desc(), - name="sentry_release_semver_idx", - ), - models.Index(fields=("organization", "build_code")), - models.Index(fields=("organization", "build_number")), - models.Index(fields=("organization", "date_added")), - models.Index(fields=("organization", "status")), - ] - - __repr__ = sane_repr("organization_id", "version") - - SEMVER_COLS = ["major", "minor", "patch", "revision", "prerelease_case", "prerelease"] - - def __eq__(self, other: object) -> bool: - """Make sure that specialized releases are only comparable to the same - other specialized release. This for instance lets us treat them - separately for serialization purposes. - """ - return ( - # don't treat `NotImplemented` as truthy - Model.__eq__(self, other) is True - and isinstance(other, Release) - and self._for_project_id == other._for_project_id - ) - - def __hash__(self): - # https://code.djangoproject.com/ticket/30333 - return super().__hash__() - - @staticmethod - def is_valid_version(value): - if value is None: - return False - - if any(c in value for c in BAD_RELEASE_CHARS): - return False - - value_stripped = str(value).strip() - return not ( - not value_stripped - or value_stripped in (".", "..") - or value_stripped.lower() == "latest" - ) - - @property - def is_semver_release(self): - return self.package is not None - - def get_previous_release(self, project): - """Get the release prior to this one. None if none exists""" - return ( - ReleaseProject.objects.filter(project=project, release__date_added__lt=self.date_added) - .order_by("-release__date_added") - .first() - ) - - @staticmethod - def is_semver_version(version): - """ - Method that checks if a version follows semantic versioning - """ - # If version is not a valid release version, or it has no package then we return False - if not Release.is_valid_version(version) or "@" not in version: - return False - - try: - version_info = parse_release(version, json_loads=orjson.loads) - version_parsed = version_info.get("version_parsed") - return version_parsed is not None and all( - validate_bigint(version_parsed[field]) - for field in ("major", "minor", "patch", "revision") - ) - except RelayError: - # This can happen on invalid legacy releases - return False - - @staticmethod - def is_release_newer_or_equal(org_id, release, other_release): - if release is None: - return False - - if other_release is None: - return True - - if release == other_release: - return True - - releases = { - release.version: float(release.date_added.timestamp()) - for release in Release.objects.filter( - organization_id=org_id, version__in=[release, other_release] - ) - } - release_date = releases.get(release) - other_release_date = releases.get(other_release) - - if release_date is not None and other_release_date is not None: - return release_date > other_release_date - - return False - - @property - def semver_tuple(self) -> SemverVersion: - return SemverVersion( - self.major, - self.minor, - self.patch, - self.revision, - 1 if self.prerelease == "" else 0, - self.prerelease, - ) - - @classmethod - def get_cache_key(cls, organization_id, version): - return f"release:3:{organization_id}:{md5_text(version).hexdigest()}" - - @classmethod - def get_lock_key(cls, organization_id, release_id): - return f"releasecommits:{organization_id}:{release_id}" - - @classmethod - def get(cls, project, version): - cache_key = cls.get_cache_key(project.organization_id, version) - - release = cache.get(cache_key) - if release is None: - try: - release = cls.objects.get( - organization_id=project.organization_id, projects=project, version=version - ) - except cls.DoesNotExist: - release = -1 - cache.set(cache_key, release, 300) - - if release == -1: - return - return release - - @classmethod - def get_or_create(cls, project, version, date_added=None): - with metrics.timer("models.release.get_or_create") as metric_tags: - return cls._get_or_create_impl(project, version, date_added, metric_tags) - - @classmethod - def _get_or_create_impl(cls, project, version, date_added, metric_tags): - from sentry.models.project import Project - - if date_added is None: - date_added = timezone.now() - - cache_key = cls.get_cache_key(project.organization_id, version) - - release = cache.get(cache_key) - - if release in (None, -1): - # TODO(dcramer): if the cache result is -1 we could attempt a - # default create here instead of default get - project_version = (f"{project.slug}-{version}")[:DB_VERSION_LENGTH] - releases = list( - cls.objects.filter( - organization_id=project.organization_id, - version__in=[version, project_version], - projects=project, - ) - ) - - if releases: - try: - release = [r for r in releases if r.version == project_version][0] - except IndexError: - release = releases[0] - metric_tags["created"] = "false" - else: - try: - with atomic_transaction(using=router.db_for_write(cls)): - release = cls.objects.create( - organization_id=project.organization_id, - version=version, - date_added=date_added, - total_deploys=0, - ) - - metric_tags["created"] = "true" - except IntegrityError: - metric_tags["created"] = "false" - release = cls.objects.get( - organization_id=project.organization_id, version=version - ) - - # NOTE: `add_project` creates a ReleaseProject instance - release.add_project(project) - if not project.flags.has_releases: - project.flags.has_releases = True - project.update(flags=F("flags").bitor(Project.flags.has_releases)) - - # TODO(dcramer): upon creating a new release, check if it should be - # the new "latest release" for this project - cache.set(cache_key, release, 3600) - metric_tags["cache_hit"] = "false" - else: - metric_tags["cache_hit"] = "true" - - return release - - @cached_property - def version_info(self): - try: - return parse_release(self.version, json_loads=orjson.loads) - except RelayError: - # This can happen on invalid legacy releases - return None - - @classmethod - def merge(cls, to_release, from_releases): - # The following models reference release: - # ReleaseCommit.release - # ReleaseEnvironment.release_id - # ReleaseProject.release - # GroupRelease.release_id - # GroupResolution.release - # Group.first_release - # ReleaseFile.release - - from sentry.models.group import Group - from sentry.models.grouprelease import GroupRelease - from sentry.models.groupresolution import GroupResolution - from sentry.models.releasecommit import ReleaseCommit - from sentry.models.releaseenvironment import ReleaseEnvironment - from sentry.models.releasefile import ReleaseFile - from sentry.models.releaseprojectenvironment import ReleaseProjectEnvironment - from sentry.models.releases.release_project import ReleaseProject - - model_list = ( - ReleaseCommit, - ReleaseEnvironment, - ReleaseFile, - ReleaseProject, - ReleaseProjectEnvironment, - GroupRelease, - GroupResolution, - ) - for release in from_releases: - for model in model_list: - if hasattr(model, "release"): - update_kwargs = {"release": to_release} - else: - update_kwargs = {"release_id": to_release.id} - try: - with atomic_transaction(using=router.db_for_write(model)): - model.objects.filter(release_id=release.id).update(**update_kwargs) - except IntegrityError: - for item in model.objects.filter(release_id=release.id): - try: - with atomic_transaction(using=router.db_for_write(model)): - model.objects.filter(id=item.id).update(**update_kwargs) - except IntegrityError: - item.delete() - - Group.objects.filter(first_release=release).update(first_release=to_release) - - release.delete() - - def add_dist(self, name, date_added=None): - from sentry.models.distribution import Distribution - - if date_added is None: - date_added = timezone.now() - return Distribution.objects.get_or_create( - release=self, - name=name, - defaults={"date_added": date_added, "organization_id": self.organization_id}, - )[0] - - def add_project(self, project): - """ - Add a project to this release. - - Returns True if the project was added and did not already exist. - """ - from sentry.models.project import Project - - try: - with atomic_transaction(using=router.db_for_write(ReleaseProject)): - obj, created = ReleaseProject.objects.get_or_create(project=project, release=self) - if not project.flags.has_releases: - project.flags.has_releases = True - project.update(flags=F("flags").bitor(Project.flags.has_releases)) - except IntegrityError: - obj = None - created = False - - return obj, created - - def handle_commit_ranges(self, refs): - """ - Takes commit refs of the form: - [ - { - 'previousCommit': None, - 'commit': 'previous_commit..commit', - } - ] - Note: Overwrites 'previousCommit' and 'commit' - """ - for ref in refs: - if COMMIT_RANGE_DELIMITER in ref["commit"]: - ref["previousCommit"], ref["commit"] = ref["commit"].split(COMMIT_RANGE_DELIMITER) - - def set_refs(self, refs, user_id, fetch=False): - with sentry_sdk.start_span(op="set_refs"): - from sentry.api.exceptions import InvalidRepository - from sentry.models.commit import Commit - from sentry.models.releaseheadcommit import ReleaseHeadCommit - from sentry.models.repository import Repository - from sentry.tasks.commits import fetch_commits - - names = {r["repository"] for r in refs} - repos = list( - Repository.objects.filter(organization_id=self.organization_id, name__in=names) - ) - repos_by_name = {r.name: r for r in repos} - invalid_repos = names - set(repos_by_name.keys()) - if invalid_repos: - raise InvalidRepository(f"Invalid repository names: {','.join(invalid_repos)}") - - self.handle_commit_ranges(refs) - - for ref in refs: - repo = repos_by_name[ref["repository"]] - - commit = Commit.objects.get_or_create( - organization_id=self.organization_id, repository_id=repo.id, key=ref["commit"] - )[0] - # update head commit for repo/release if exists - ReleaseHeadCommit.objects.create_or_update( - organization_id=self.organization_id, - repository_id=repo.id, - release=self, - values={"commit": commit}, - ) - if fetch: - prev_release = get_previous_release(self) - fetch_commits.apply_async( - kwargs={ - "release_id": self.id, - "user_id": user_id, - "refs": refs, - "prev_release_id": prev_release and prev_release.id, - } - ) - - @sentry_sdk.trace - def set_commits(self, commit_list): - """ - Bind a list of commits to this release. - - This will clear any existing commit log and replace it with the given - commits. - """ - set_span_attribute("release.set_commits", len(commit_list)) - - from sentry.models.releases.set_commits import set_commits - - set_commits(self, commit_list) - - def safe_delete(self): - """Deletes a release if possible or raises a `UnsafeReleaseDeletion` - exception. - """ - from sentry import release_health - from sentry.models.group import Group - from sentry.models.releasefile import ReleaseFile - - # we don't want to remove the first_release metadata on the Group, and - # while people might want to kill a release (maybe to remove files), - # removing the release is prevented - if Group.objects.filter(first_release=self).exists(): - raise UnsafeReleaseDeletion(ERR_RELEASE_REFERENCED) - - # We do not allow releases with health data to be deleted because - # the upserting from snuba data would create the release again. - # We would need to be able to delete this data from snuba which we - # can't do yet. - project_ids = list(self.projects.values_list("id").all()) - if release_health.backend.check_has_health_data( - [(p[0], self.version) for p in project_ids] - ): - raise UnsafeReleaseDeletion(ERR_RELEASE_HEALTH_DATA) - - # TODO(dcramer): this needs to happen in the queue as it could be a long - # and expensive operation - file_list = ReleaseFile.objects.filter(release_id=self.id).select_related("file") - for releasefile in file_list: - releasefile.file.delete() - releasefile.delete() - self.delete() - - def count_artifacts(self): - """Sum the artifact_counts of all release files. - - An artifact count of NULL is interpreted as 1. - """ - counts = get_artifact_counts([self.id]) - return counts.get(self.id, 0) - - def count_artifacts_in_artifact_bundles(self, project_ids: Sequence[int]): - """ - Counts the number of artifacts in the artifact bundles associated with this release and a set of projects. - """ - qs = ( - ArtifactBundle.objects.filter( - organization_id=self.organization.id, - releaseartifactbundle__release_name=self.version, - projectartifactbundle__project_id__in=project_ids, - ) - .annotate(count=Sum(Func(F("artifact_count"), 1, function="COALESCE"))) - .values_list("releaseartifactbundle__release_name", "count") - ) - - qs.query.group_by = ["releaseartifactbundle__release_name"] - - if len(qs) == 0: - return None - - return qs[0] - - def clear_commits(self): - """ - Delete all release-specific commit data associated to this release. We will not delete the Commit model values because other releases may use these commits. - """ - with sentry_sdk.start_span(op="clear_commits"): - from sentry.models.releasecommit import ReleaseCommit - from sentry.models.releaseheadcommit import ReleaseHeadCommit - - ReleaseHeadCommit.objects.get( - organization_id=self.organization_id, release=self - ).delete() - ReleaseCommit.objects.filter( - organization_id=self.organization_id, release=self - ).delete() - - self.authors = [] - self.commit_count = 0 - self.last_commit_id = None - self.save() - - -def get_artifact_counts(release_ids: list[int]) -> Mapping[int, int]: - """Get artifact count grouped by IDs""" - from sentry.models.releasefile import ReleaseFile - - qs = ( - ReleaseFile.objects.filter(release_id__in=release_ids) - .annotate(count=Sum(Func(F("artifact_count"), 1, function="COALESCE"))) - .values_list("release_id", "count") - ) - qs.query.group_by = ["release_id"] - return dict(qs) - - -def follows_semver_versioning_scheme(org_id, project_id, release_version=None): - """ - Checks if we should follow semantic versioning scheme for ordering based on - 1. Latest ten releases of the project_id passed in all follow semver - 2. provided release version argument is a valid semver version - - Inputs: - * org_id - * project_id - * release_version - Returns: - Boolean that indicates if we should follow semantic version or not - """ - # TODO(ahmed): Move this function else where to be easily accessible for re-use - # TODO: this method could be moved to the Release model manager - cache_key = "follows_semver:1:%s" % hash_values([org_id, project_id]) - follows_semver = cache.get(cache_key) - - if follows_semver is None: - # Check if the latest ten releases are semver compliant - releases_list = list( - Release.objects.filter( - organization_id=org_id, projects__id__in=[project_id], status=ReleaseStatus.OPEN - ) - .using_replica() - .order_by("-date_added")[:10] - ) - - if not releases_list: - cache.set(cache_key, False, 3600) - return False - - # TODO(ahmed): re-visit/replace these conditions once we enable project wide `semver` setting - # A project is said to be following semver versioning schemes if it satisfies the following - # conditions:- - # 1: At least one semver compliant in the most recent 3 releases - # 2: At least 3 semver compliant releases in the most recent 10 releases - if len(releases_list) <= 2: - # Most recent release is considered to decide if project follows semver - follows_semver = releases_list[0].is_semver_release - elif len(releases_list) < 10: - # We forego condition 2 and it is enough if condition 1 is satisfied to consider this - # project to have semver compliant releases - follows_semver = any(release.is_semver_release for release in releases_list[0:3]) - else: - # Count number of semver releases in the last ten - semver_matches = sum(map(lambda release: release.is_semver_release, releases_list)) - - at_least_three_in_last_ten = semver_matches >= 3 - at_least_one_in_last_three = any( - release.is_semver_release for release in releases_list[0:3] - ) - - follows_semver = at_least_one_in_last_three and at_least_three_in_last_ten - cache.set(cache_key, follows_semver, 3600) - - # Check release_version that is passed is semver compliant - if release_version: - follows_semver = follows_semver and Release.is_semver_version(release_version) - return follows_semver - - -def get_previous_release(release: Release) -> Release | None: - # NOTE: Keeping the below todo. Just optimizing the query. - # - # TODO: this does the wrong thing unless you are on the most - # recent release. Add a timestamp compare? - return ( - Release.objects.filter(organization_id=release.organization_id) - .filter( - Exists( - ReleaseProject.objects.filter( - release=OuterRef("pk"), - project_id__in=ReleaseProject.objects.filter(release=release).values_list( - "project_id", flat=True - ), - ) - ) - ) - .extra(select={"sort": "COALESCE(date_released, date_added)"}) - .exclude(version=release.version) - .order_by("-sort") - .first() - ) +__all__ = ( + "Release", + "ReleaseStatus", + "follows_semver_versioning_scheme", + "_get_cache_key", + "get_previous_release", + "get_artifact_counts", +) diff --git a/src/sentry/models/release_threshold/constants.py b/src/sentry/models/release_threshold/constants.py index e4cd871db21494..288e3ec4aca00f 100644 --- a/src/sentry/models/release_threshold/constants.py +++ b/src/sentry/models/release_threshold/constants.py @@ -1,93 +1 @@ -class ReleaseThresholdType: - TOTAL_ERROR_COUNT = 0 - NEW_ISSUE_COUNT = 1 - UNHANDLED_ISSUE_COUNT = 2 - REGRESSED_ISSUE_COUNT = 3 - FAILURE_RATE = 4 - CRASH_FREE_SESSION_RATE = 5 - CRASH_FREE_USER_RATE = 6 - - TOTAL_ERROR_COUNT_STR = "total_error_count" - NEW_ISSUE_COUNT_STR = "new_issue_count" - UNHANDLED_ISSUE_COUNT_STR = "unhandled_issue_count" - REGRESSED_ISSUE_COUNT_STR = "regressed_issue_count" - FAILURE_RATE_STR = "failure_rate" - CRASH_FREE_SESSION_RATE_STR = "crash_free_session_rate" - CRASH_FREE_USER_RATE_STR = "crash_free_user_rate" - - @classmethod - def as_choices(cls): - return ( - (cls.TOTAL_ERROR_COUNT_STR, cls.TOTAL_ERROR_COUNT), - (cls.NEW_ISSUE_COUNT_STR, cls.NEW_ISSUE_COUNT), - (cls.UNHANDLED_ISSUE_COUNT_STR, cls.UNHANDLED_ISSUE_COUNT), - (cls.REGRESSED_ISSUE_COUNT_STR, cls.REGRESSED_ISSUE_COUNT), - (cls.FAILURE_RATE_STR, cls.FAILURE_RATE), - (cls.CRASH_FREE_SESSION_RATE_STR, cls.CRASH_FREE_SESSION_RATE), - (cls.CRASH_FREE_USER_RATE_STR, cls.CRASH_FREE_USER_RATE), - ) - - @classmethod - def as_str_choices(cls): - return ( - (cls.TOTAL_ERROR_COUNT_STR, cls.TOTAL_ERROR_COUNT_STR), - (cls.NEW_ISSUE_COUNT_STR, cls.NEW_ISSUE_COUNT_STR), - (cls.UNHANDLED_ISSUE_COUNT_STR, cls.UNHANDLED_ISSUE_COUNT_STR), - (cls.REGRESSED_ISSUE_COUNT_STR, cls.REGRESSED_ISSUE_COUNT_STR), - (cls.FAILURE_RATE_STR, cls.FAILURE_RATE_STR), - (cls.CRASH_FREE_SESSION_RATE_STR, cls.CRASH_FREE_SESSION_RATE_STR), - (cls.CRASH_FREE_USER_RATE_STR, cls.CRASH_FREE_USER_RATE_STR), - ) - - -class TriggerType: - OVER = 0 - UNDER = 1 - - OVER_STR = "over" - UNDER_STR = "under" - - @classmethod - def as_choices(cls): # choices for model column - return ( - (cls.OVER_STR, cls.OVER), - (cls.UNDER_STR, cls.UNDER), - ) - - @classmethod - def as_str_choices(cls): # choices for serializer - return ( - (cls.OVER_STR, cls.OVER_STR), - (cls.UNDER_STR, cls.UNDER_STR), - ) - - -THRESHOLD_TYPE_INT_TO_STR = { - ReleaseThresholdType.TOTAL_ERROR_COUNT: ReleaseThresholdType.TOTAL_ERROR_COUNT_STR, - ReleaseThresholdType.NEW_ISSUE_COUNT: ReleaseThresholdType.NEW_ISSUE_COUNT_STR, - ReleaseThresholdType.UNHANDLED_ISSUE_COUNT: ReleaseThresholdType.UNHANDLED_ISSUE_COUNT_STR, - ReleaseThresholdType.REGRESSED_ISSUE_COUNT: ReleaseThresholdType.REGRESSED_ISSUE_COUNT_STR, - ReleaseThresholdType.FAILURE_RATE: ReleaseThresholdType.FAILURE_RATE_STR, - ReleaseThresholdType.CRASH_FREE_SESSION_RATE: ReleaseThresholdType.CRASH_FREE_SESSION_RATE_STR, - ReleaseThresholdType.CRASH_FREE_USER_RATE: ReleaseThresholdType.CRASH_FREE_USER_RATE_STR, -} - -THRESHOLD_TYPE_STR_TO_INT = { - ReleaseThresholdType.TOTAL_ERROR_COUNT_STR: ReleaseThresholdType.TOTAL_ERROR_COUNT, - ReleaseThresholdType.NEW_ISSUE_COUNT_STR: ReleaseThresholdType.NEW_ISSUE_COUNT, - ReleaseThresholdType.UNHANDLED_ISSUE_COUNT_STR: ReleaseThresholdType.UNHANDLED_ISSUE_COUNT, - ReleaseThresholdType.REGRESSED_ISSUE_COUNT_STR: ReleaseThresholdType.REGRESSED_ISSUE_COUNT, - ReleaseThresholdType.FAILURE_RATE_STR: ReleaseThresholdType.FAILURE_RATE, - ReleaseThresholdType.CRASH_FREE_SESSION_RATE_STR: ReleaseThresholdType.CRASH_FREE_SESSION_RATE, - ReleaseThresholdType.CRASH_FREE_USER_RATE_STR: ReleaseThresholdType.CRASH_FREE_USER_RATE, -} - -TRIGGER_TYPE_INT_TO_STR = { - TriggerType.OVER: TriggerType.OVER_STR, - TriggerType.UNDER: TriggerType.UNDER_STR, -} - -TRIGGER_TYPE_STRING_TO_INT = { - TriggerType.OVER_STR: TriggerType.OVER, - TriggerType.UNDER_STR: TriggerType.UNDER, -} +from sentry.releases.models.release_threshold.constants import * # noqa: F401,F403 diff --git a/src/sentry/models/release_threshold/release_threshold.py b/src/sentry/models/release_threshold/release_threshold.py index a9acb5f2d19ac6..f4aa56fe552cb8 100644 --- a/src/sentry/models/release_threshold/release_threshold.py +++ b/src/sentry/models/release_threshold/release_threshold.py @@ -1,34 +1 @@ -from django.db import models -from django.utils import timezone - -from sentry.backup.scopes import RelocationScope -from sentry.db.models import FlexibleForeignKey, Model, region_silo_model -from sentry.db.models.fields.bounded import BoundedPositiveIntegerField -from sentry.models.release_threshold.constants import ReleaseThresholdType -from sentry.models.release_threshold.constants import TriggerType as ReleaseThresholdTriggerType - - -@region_silo_model -class ReleaseThreshold(Model): - """ - NOTE: - To transition to utilizing AlertRules, there are some duplicated attrs we'll want to dedup. - AlertRule model should house metadata on the AlertRule itself (eg. type of alert rule) - AlertRuleTrigger model should house the trigger requirements (eg. value, over/under trigger type) - - TODO: Will need to determine how this translates to release_threshold evaluation - QuerySubscription model subscribes the AlertRule to specific query in Snuba - SnubaQuery model represents the actual query run in Snuba - - TODO: replace query constructed in release_thresholds api with activated SnubaQuery / determine whether we're constructing the same query or not - """ - - __relocation_scope__ = RelocationScope.Excluded - - threshold_type = BoundedPositiveIntegerField(choices=ReleaseThresholdType.as_choices()) - trigger_type = BoundedPositiveIntegerField(choices=ReleaseThresholdTriggerType.as_choices()) - - value = models.IntegerField() - window_in_seconds = models.PositiveIntegerField() - - project = FlexibleForeignKey("sentry.Project", db_index=True, related_name="release_thresholds") - environment = FlexibleForeignKey("sentry.Environment", null=True, db_index=True) - date_added = models.DateTimeField(default=timezone.now) +from sentry.releases.models.release_threshold.release_threshold import * # noqa: F401,F403 diff --git a/src/sentry/models/releaseactivity.py b/src/sentry/models/releaseactivity.py index 12b91f0c558a2c..bc543cc2d2de3b 100644 --- a/src/sentry/models/releaseactivity.py +++ b/src/sentry/models/releaseactivity.py @@ -1,25 +1 @@ -from django.db import models -from django.utils import timezone - -from sentry.backup.scopes import RelocationScope -from sentry.db.models import ( - BoundedPositiveIntegerField, - FlexibleForeignKey, - Model, - region_silo_model, -) -from sentry.types.releaseactivity import CHOICES - - -@region_silo_model -class ReleaseActivity(Model): - __relocation_scope__ = RelocationScope.Excluded - - release = FlexibleForeignKey("sentry.Release", db_index=True) - type = BoundedPositiveIntegerField(null=False, choices=CHOICES) - data = models.JSONField(default=dict) - date_added = models.DateTimeField(default=timezone.now) - - class Meta: - app_label = "sentry" - db_table = "sentry_releaseactivity" +from sentry.releases.models.releaseactivity import * # noqa: F401,F403 diff --git a/src/sentry/models/releasecommit.py b/src/sentry/models/releasecommit.py index b6c42c36d0240b..462e91fd1fb842 100644 --- a/src/sentry/models/releasecommit.py +++ b/src/sentry/models/releasecommit.py @@ -1,28 +1 @@ -from sentry.backup.scopes import RelocationScope -from sentry.db.models import ( - BoundedBigIntegerField, - BoundedPositiveIntegerField, - FlexibleForeignKey, - Model, - region_silo_model, - sane_repr, -) - - -@region_silo_model -class ReleaseCommit(Model): - __relocation_scope__ = RelocationScope.Excluded - - organization_id = BoundedBigIntegerField(db_index=True) - # DEPRECATED - project_id = BoundedBigIntegerField(null=True) - release = FlexibleForeignKey("sentry.Release") - commit = FlexibleForeignKey("sentry.Commit") - order = BoundedPositiveIntegerField() - - class Meta: - app_label = "sentry" - db_table = "sentry_releasecommit" - unique_together = (("release", "commit"), ("release", "order")) - - __repr__ = sane_repr("release_id", "commit_id", "order") +from sentry.releases.models.releasecommit import * # noqa: F401,F403 diff --git a/src/sentry/models/releaseenvironment.py b/src/sentry/models/releaseenvironment.py index e88dd90727d70a..bf3eb970154538 100644 --- a/src/sentry/models/releaseenvironment.py +++ b/src/sentry/models/releaseenvironment.py @@ -1,79 +1 @@ -from datetime import timedelta - -from django.db import models -from django.utils import timezone - -from sentry.backup.scopes import RelocationScope -from sentry.db.models import ( - BoundedBigIntegerField, - FlexibleForeignKey, - Model, - region_silo_model, - sane_repr, -) -from sentry.utils import metrics -from sentry.utils.cache import cache - - -@region_silo_model -class ReleaseEnvironment(Model): - __relocation_scope__ = RelocationScope.Excluded - - organization = FlexibleForeignKey("sentry.Organization", db_index=True, db_constraint=False) - # DEPRECATED - project_id = BoundedBigIntegerField(null=True) - release = FlexibleForeignKey("sentry.Release", db_index=True, db_constraint=False) - environment = FlexibleForeignKey("sentry.Environment", db_index=True, db_constraint=False) - first_seen = models.DateTimeField(default=timezone.now) - last_seen = models.DateTimeField(default=timezone.now, db_index=True) - - class Meta: - app_label = "sentry" - db_table = "sentry_environmentrelease" - unique_together = (("organization", "release", "environment"),) - - __repr__ = sane_repr("organization_id", "release_id", "environment_id") - - @classmethod - def get_cache_key(cls, organization_id, release_id, environment_id): - return f"releaseenv:2:{organization_id}:{release_id}:{environment_id}" - - @classmethod - def get_or_create(cls, project, release, environment, datetime, **kwargs): - with metrics.timer("models.releaseenvironment.get_or_create") as metric_tags: - return cls._get_or_create_impl(project, release, environment, datetime, metric_tags) - - @classmethod - def _get_or_create_impl(cls, project, release, environment, datetime, metric_tags): - cache_key = cls.get_cache_key(project.id, release.id, environment.id) - - instance = cache.get(cache_key) - if instance is None: - metric_tags["cache_hit"] = "false" - instance, created = cls.objects.get_or_create( - release_id=release.id, - organization_id=project.organization_id, - environment_id=environment.id, - defaults={"first_seen": datetime, "last_seen": datetime}, - ) - cache.set(cache_key, instance, 3600) - else: - metric_tags["cache_hit"] = "true" - created = False - - metric_tags["created"] = "true" if created else "false" - - # TODO(dcramer): this would be good to buffer, but until then we minimize - # updates to once a minute, and allow Postgres to optimistically skip - # it even if we can't - if not created and instance.last_seen < datetime - timedelta(seconds=60): - metric_tags["bumped"] = "true" - cls.objects.filter( - id=instance.id, last_seen__lt=datetime - timedelta(seconds=60) - ).update(last_seen=datetime) - instance.last_seen = datetime - cache.set(cache_key, instance, 3600) - else: - metric_tags["bumped"] = "false" - - return instance +from sentry.releases.models.releaseenvironment import * # noqa: F401,F403 diff --git a/src/sentry/models/releasefile.py b/src/sentry/models/releasefile.py index a2c19158983305..ab40aca4447409 100644 --- a/src/sentry/models/releasefile.py +++ b/src/sentry/models/releasefile.py @@ -1,404 +1,23 @@ -from __future__ import annotations - -import logging -import zipfile -from contextlib import contextmanager -from hashlib import sha1 -from io import BytesIO -from tempfile import TemporaryDirectory -from typing import IO, ClassVar, Self -from urllib.parse import urlunsplit - -import sentry_sdk -from django.db import models, router - -from sentry.backup.scopes import RelocationScope -from sentry.db.models import ( - BoundedBigIntegerField, - BoundedPositiveIntegerField, - FlexibleForeignKey, - Model, - region_silo_model, - sane_repr, +from sentry.releases.models.releasefile import ( + ARTIFACT_INDEX_FILENAME, + ARTIFACT_INDEX_TYPE, + ReleaseArchive, + ReleaseFile, + _ArtifactIndexData, + _ArtifactIndexGuard, + delete_from_artifact_index, + read_artifact_index, + update_artifact_index, ) -from sentry.db.models.manager.base import BaseManager -from sentry.models.distribution import Distribution -from sentry.models.files.file import File -from sentry.models.release import Release -from sentry.utils import json -from sentry.utils.db import atomic_transaction -from sentry.utils.hashlib import sha1_text -from sentry.utils.urls import urlsplit_best_effort -from sentry.utils.zip import safe_extract_zip - -logger = logging.getLogger(__name__) - - -ARTIFACT_INDEX_FILENAME = "artifact-index.json" -ARTIFACT_INDEX_TYPE = "release.artifact-index" - - -class PublicReleaseFileManager(models.Manager["ReleaseFile"]): - """Manager for all release files that are not internal. - - Internal release files include: - * Uploaded release archives - * Artifact index mapping URLs to release archives - - This manager has the overhead of always joining the File table in order - to filter release files. - - """ - - def get_queryset(self): - return super().get_queryset().select_related("file").filter(file__type="release.file") - - -@region_silo_model -class ReleaseFile(Model): - r""" - A ReleaseFile is an association between a Release and a File. - - The ident of the file should be sha1(name) or - sha1(name '\x00\x00' dist.name) and must be unique per release. - """ - - __relocation_scope__ = RelocationScope.Excluded - - organization_id = BoundedBigIntegerField(db_index=True) - # DEPRECATED - project_id = BoundedBigIntegerField(null=True) - release_id = BoundedBigIntegerField(db_index=True) - file = FlexibleForeignKey("sentry.File") - ident = models.CharField(max_length=40) - name = models.TextField() - dist_id = BoundedBigIntegerField(null=True, db_index=True) - - #: For classic file uploads, this field is 1. - #: For release archives, this field is 0. - #: For artifact indexes, this field is the number of artifacts contained - #: in the index. - artifact_count = BoundedPositiveIntegerField(null=True, default=1) - - __repr__ = sane_repr("release", "ident") - - objects: ClassVar[BaseManager[Self]] = BaseManager() # The default manager. - public_objects: ClassVar[PublicReleaseFileManager] = PublicReleaseFileManager() - - class Meta: - unique_together = (("release_id", "ident"),) - indexes = (models.Index(fields=("release_id", "name")),) - app_label = "sentry" - db_table = "sentry_releasefile" - - def save(self, *args, **kwargs): - from sentry.models.distribution import Distribution - - if not self.ident and self.name: - dist = None - if self.dist_id: - dist = Distribution.objects.get(pk=self.dist_id).name - self.ident = type(self).get_ident(self.name, dist) - return super().save(*args, **kwargs) - - def update(self, *args, **kwargs): - # If our name is changing, we must also change the ident - if "name" in kwargs and "ident" not in kwargs: - dist_name = None - dist_id = kwargs.get("dist_id") or self.dist_id - if dist_id: - dist_name = Distribution.objects.filter(pk=dist_id).values_list("name", flat=True)[ - 0 - ] - kwargs["ident"] = self.ident = type(self).get_ident(kwargs["name"], dist_name) - return super().update(*args, **kwargs) - - @classmethod - def get_ident(cls, name, dist=None): - if dist is not None: - return sha1_text(name + "\x00\x00" + dist).hexdigest() - return sha1_text(name).hexdigest() - - @classmethod - def normalize(cls, url): - """Transforms a full absolute url into 2 or 4 generalized options - - * the original url as input - * (optional) original url without querystring - * the full url, but stripped of scheme and netloc - * (optional) full url without scheme and netloc or querystring - """ - # Always ignore the fragment - scheme, netloc, path, query = urlsplit_best_effort(url) - - uri_without_fragment = (scheme, netloc, path, query, "") - uri_relative = ("", "", path, query, "") - uri_without_query = (scheme, netloc, path, "", "") - uri_relative_without_query = ("", "", path, "", "") - - urls = [urlunsplit(uri_without_fragment)] - if query: - urls.append(urlunsplit(uri_without_query)) - urls.append("~" + urlunsplit(uri_relative)) - if query: - urls.append("~" + urlunsplit(uri_relative_without_query)) - return urls - - -class ReleaseArchive: - """Read-only view of uploaded ZIP-archive of release files""" - - def __init__(self, fileobj: IO): - self._fileobj = fileobj - self._zip_file = zipfile.ZipFile(self._fileobj) - self.manifest = self._read_manifest() - self.artifact_count = len(self.manifest.get("files", {})) - files = self.manifest.get("files", {}) - self._entries_by_url = {entry["url"]: (path, entry) for path, entry in files.items()} - - def __enter__(self): - return self - - def __exit__(self, exc, value, tb): - self.close() - - def close(self): - self._zip_file.close() - self._fileobj.close() - - def info(self, filename: str) -> zipfile.ZipInfo: - return self._zip_file.getinfo(filename) - - def read(self, filename: str) -> bytes: - return self._zip_file.read(filename) - - def _read_manifest(self) -> dict: - manifest_bytes = self.read("manifest.json") - return json.loads(manifest_bytes.decode("utf-8")) - - def get_file_by_url(self, url: str) -> tuple[IO[bytes], dict]: - """Return file-like object and headers. - - The caller is responsible for closing the returned stream. - - May raise ``KeyError`` - """ - filename, entry = self._entries_by_url[url] - return self._zip_file.open(filename), entry.get("headers", {}) - - def extract(self) -> TemporaryDirectory: - """Extract contents to a temporary directory. - The caller is responsible for cleanup of the temporary files. - """ - temp_dir = TemporaryDirectory() - safe_extract_zip(self._fileobj, temp_dir.name) - - return temp_dir - - -class _ArtifactIndexData: - """Holds data of artifact index and keeps track of changes""" - - def __init__(self, data: dict, fresh=False): - self._data = data - self.changed = fresh - - @property - def data(self): - """Meant to be read-only""" - return self._data - - @property - def num_files(self): - return len(self._data.get("files", {})) - - def get(self, filename: str): - return self._data.get("files", {}).get(filename, None) - - def update_files(self, files: dict): - if files: - self._data.setdefault("files", {}).update(files) - self.changed = True - - def delete(self, filename: str) -> bool: - result = self._data.get("files", {}).pop(filename, None) - deleted = result is not None - if deleted: - self.changed = True - - return deleted - - -class _ArtifactIndexGuard: - """Ensures atomic write operations to the artifact index""" - - def __init__(self, release: Release, dist: Distribution | None, **filter_args): - self._release = release - self._dist = dist - self._ident = ReleaseFile.get_ident(ARTIFACT_INDEX_FILENAME, dist and dist.name) - self._filter_args = filter_args # Extra constraints on artifact index release file - - def readable_data(self) -> dict | None: - """Simple read, no synchronization necessary""" - try: - releasefile = self._releasefile_qs()[0] - except IndexError: - return None - else: - fp = releasefile.file.getfile() - with fp: - return json.load(fp) - - @contextmanager - def writable_data(self, create: bool, initial_artifact_count=None): - """Context manager for editable artifact index""" - with atomic_transaction( - using=( - router.db_for_write(ReleaseFile), - router.db_for_write(File), - ) - ): - created = False - if create: - releasefile, created = self._get_or_create_releasefile(initial_artifact_count) - else: - # Lock the row for editing: - # NOTE: Do not select_related('file') here, because we do not - # want to lock the File table - qs = self._releasefile_qs().select_for_update() - try: - releasefile = qs[0] - except IndexError: - releasefile = None - - if releasefile is None: - index_data = None - else: - if created: - index_data = _ArtifactIndexData({}, fresh=True) - else: - source_file = releasefile.file - if source_file.type != ARTIFACT_INDEX_TYPE: - raise RuntimeError("Unexpected file type for artifact index") - raw_data = json.load(source_file.getfile()) - index_data = _ArtifactIndexData(raw_data) - - yield index_data # editable reference to index - - if index_data is not None and index_data.changed: - if created: - target_file = releasefile.file - else: - target_file = File.objects.create( - name=ARTIFACT_INDEX_FILENAME, type=ARTIFACT_INDEX_TYPE - ) - - target_file.putfile(BytesIO(json.dumps(index_data.data).encode())) - - artifact_count = index_data.num_files - if not created: - # Update and clean existing - old_file = releasefile.file - releasefile.update(file=target_file, artifact_count=artifact_count) - old_file.delete() - - def _get_or_create_releasefile(self, initial_artifact_count): - """Make sure that the release file exists""" - return ReleaseFile.objects.select_for_update().get_or_create( - **self._key_fields(), - defaults={ - "artifact_count": initial_artifact_count, - "file": lambda: File.objects.create( - name=ARTIFACT_INDEX_FILENAME, - type=ARTIFACT_INDEX_TYPE, - ), - }, - ) - - def _releasefile_qs(self): - """QuerySet for selecting artifact index""" - return ReleaseFile.objects.filter(**self._key_fields(), **self._filter_args) - - def _key_fields(self): - """Columns needed to identify the artifact index in the db""" - return dict( - organization_id=self._release.organization_id, - release_id=self._release.id, - dist_id=self._dist.id if self._dist else self._dist, - name=ARTIFACT_INDEX_FILENAME, - ident=self._ident, - ) - - -@sentry_sdk.tracing.trace -def read_artifact_index(release: Release, dist: Distribution | None, **filter_args) -> dict | None: - """Get index data""" - guard = _ArtifactIndexGuard(release, dist, **filter_args) - return guard.readable_data() - - -def _compute_sha1(archive: ReleaseArchive, url: str) -> str: - data = archive.read(url) - return sha1(data).hexdigest() - - -@sentry_sdk.tracing.trace -def update_artifact_index( - release: Release, - dist: Distribution | None, - archive_file: File, - temp_file: IO | None = None, -): - """Add information from release archive to artifact index - - :returns: The created ReleaseFile instance - """ - releasefile = ReleaseFile.objects.create( - name=archive_file.name, - release_id=release.id, - organization_id=release.organization_id, - dist_id=dist.id if dist is not None else dist, - file=archive_file, - artifact_count=0, # Artifacts will be counted with artifact index - ) - - files_out = {} - with ReleaseArchive(temp_file or archive_file.getfile()) as archive: - manifest = archive.manifest - - files = manifest.get("files", {}) - if not files: - return - - for filename, info in files.items(): - info = info.copy() - url = info.pop("url") - info["filename"] = filename - info["archive_ident"] = releasefile.ident - info["date_created"] = archive_file.timestamp - info["sha1"] = _compute_sha1(archive, filename) - info["size"] = archive.info(filename).file_size - files_out[url] = info - - guard = _ArtifactIndexGuard(release, dist) - with guard.writable_data(create=True, initial_artifact_count=len(files_out)) as index_data: - index_data.update_files(files_out) - - return releasefile - - -@sentry_sdk.tracing.trace -def delete_from_artifact_index(release: Release, dist: Distribution | None, url: str) -> bool: - """Delete the file with the given url from the manifest. - - Does *not* delete the file from the zip archive. - - :returns: True if deleted - """ - guard = _ArtifactIndexGuard(release, dist) - with guard.writable_data(create=False) as index_data: - if index_data is not None: - return index_data.delete(url) - - return False +__all__ = ( + "ReleaseFile", + "read_artifact_index", + "_ArtifactIndexData", + "_ArtifactIndexGuard", + "update_artifact_index", + "ARTIFACT_INDEX_FILENAME", + "ARTIFACT_INDEX_TYPE", + "delete_from_artifact_index", + "ReleaseArchive", +) diff --git a/src/sentry/models/releaseheadcommit.py b/src/sentry/models/releaseheadcommit.py index b66e31a0f6534c..7c005bb5f40cd7 100644 --- a/src/sentry/models/releaseheadcommit.py +++ b/src/sentry/models/releaseheadcommit.py @@ -1,26 +1 @@ -from sentry.backup.scopes import RelocationScope -from sentry.db.models import ( - BoundedBigIntegerField, - BoundedPositiveIntegerField, - FlexibleForeignKey, - Model, - region_silo_model, - sane_repr, -) - - -@region_silo_model -class ReleaseHeadCommit(Model): - __relocation_scope__ = RelocationScope.Excluded - - organization_id = BoundedBigIntegerField(db_index=True) - repository_id = BoundedPositiveIntegerField() - release = FlexibleForeignKey("sentry.Release") - commit = FlexibleForeignKey("sentry.Commit") - - class Meta: - app_label = "sentry" - db_table = "sentry_releaseheadcommit" - unique_together = (("repository_id", "release"),) - - __repr__ = sane_repr("release_id", "commit_id", "repository_id") +from sentry.releases.models.releaseheadcommit import * # noqa: F401,F403 diff --git a/src/sentry/models/releaseprojectenvironment.py b/src/sentry/models/releaseprojectenvironment.py index c001d9ba65ec2c..860cc05e62d617 100644 --- a/src/sentry/models/releaseprojectenvironment.py +++ b/src/sentry/models/releaseprojectenvironment.py @@ -1,106 +1 @@ -from __future__ import annotations - -from datetime import timedelta -from enum import Enum - -from django.db import models -from django.utils import timezone - -from sentry.backup.scopes import RelocationScope -from sentry.db.models import ( - BoundedPositiveIntegerField, - FlexibleForeignKey, - Model, - region_silo_model, - sane_repr, -) -from sentry.utils import metrics -from sentry.utils.cache import cache - - -class ReleaseStages(str, Enum): - ADOPTED = "adopted" - LOW_ADOPTION = "low_adoption" - REPLACED = "replaced" - - -@region_silo_model -class ReleaseProjectEnvironment(Model): - __relocation_scope__ = RelocationScope.Excluded - - release = FlexibleForeignKey("sentry.Release") - project = FlexibleForeignKey("sentry.Project") - environment = FlexibleForeignKey("sentry.Environment") - new_issues_count = BoundedPositiveIntegerField(default=0) - first_seen = models.DateTimeField(default=timezone.now) - last_seen = models.DateTimeField(default=timezone.now, db_index=True) - last_deploy_id = BoundedPositiveIntegerField(null=True, db_index=True) - - adopted = models.DateTimeField(null=True, blank=True) - unadopted = models.DateTimeField(null=True, blank=True) - - class Meta: - app_label = "sentry" - db_table = "sentry_releaseprojectenvironment" - indexes = ( - models.Index(fields=("project", "adopted", "environment")), - models.Index(fields=("project", "unadopted", "environment")), - ) - unique_together = (("project", "release", "environment"),) - - __repr__ = sane_repr("project", "release", "environment") - - @classmethod - def get_cache_key(cls, release_id, project_id, environment_id): - return f"releaseprojectenv:{release_id}:{project_id}:{environment_id}" - - @classmethod - def get_or_create(cls, release, project, environment, datetime, **kwargs): - with metrics.timer("models.releaseprojectenvironment.get_or_create") as metrics_tags: - return cls._get_or_create_impl( - release, project, environment, datetime, metrics_tags, **kwargs - ) - - @classmethod - def _get_or_create_impl(cls, release, project, environment, datetime, metrics_tags, **kwargs): - cache_key = cls.get_cache_key(project.id, release.id, environment.id) - - instance = cache.get(cache_key) - if instance is None: - metrics_tags["cache_hit"] = "false" - instance, created = cls.objects.get_or_create( - release=release, - project=project, - environment=environment, - defaults={"first_seen": datetime, "last_seen": datetime}, - ) - cache.set(cache_key, instance, 3600) - else: - metrics_tags["cache_hit"] = "true" - created = False - - metrics_tags["created"] = "true" if created else "false" - - # Same as releaseenvironment model. Minimizes last_seen updates to once a minute - if not created and instance.last_seen < datetime - timedelta(seconds=60): - cls.objects.filter( - id=instance.id, last_seen__lt=datetime - timedelta(seconds=60) - ).update(last_seen=datetime) - instance.last_seen = datetime - cache.set(cache_key, instance, 3600) - metrics_tags["bumped"] = "true" - else: - metrics_tags["bumped"] = "false" - - return instance - - @property - def adoption_stages(self): - if self.adopted is not None and self.unadopted is None: - stage = ReleaseStages.ADOPTED - elif self.adopted is not None and self.unadopted is not None: - stage = ReleaseStages.REPLACED - else: - stage = ReleaseStages.LOW_ADOPTION - - return {"stage": stage, "adopted": self.adopted, "unadopted": self.unadopted} +from sentry.releases.models.releaseprojectenvironment import * # noqa: F401,F403 diff --git a/src/sentry/models/releases/constants.py b/src/sentry/models/releases/constants.py index 47ce332a21e834..1503ef67c4304f 100644 --- a/src/sentry/models/releases/constants.py +++ b/src/sentry/models/releases/constants.py @@ -1,5 +1 @@ -DB_VERSION_LENGTH = 250 - - -ERR_RELEASE_REFERENCED = "This release is referenced by active issues and cannot be removed." -ERR_RELEASE_HEALTH_DATA = "This release has health data and cannot be removed." +from sentry.releases.models.releases.constants import * # noqa: F401,F403 diff --git a/src/sentry/models/releases/exceptions.py b/src/sentry/models/releases/exceptions.py index d1d65b6f8d30b0..7fb924bd65cb3f 100644 --- a/src/sentry/models/releases/exceptions.py +++ b/src/sentry/models/releases/exceptions.py @@ -1,6 +1 @@ -class UnsafeReleaseDeletion(Exception): - pass - - -class ReleaseCommitError(Exception): - pass +from sentry.releases.models.releases.exceptions import * # noqa: F401,F403 diff --git a/src/sentry/models/releases/release_project.py b/src/sentry/models/releases/release_project.py index 889a5b080116d1..253b9e0bfc4cd3 100644 --- a/src/sentry/models/releases/release_project.py +++ b/src/sentry/models/releases/release_project.py @@ -1,65 +1 @@ -from __future__ import annotations - -import logging -from typing import ClassVar - -from django.db import models - -from sentry import features -from sentry.backup.scopes import RelocationScope -from sentry.db.models import ( - BoundedPositiveIntegerField, - FlexibleForeignKey, - Model, - region_silo_model, -) -from sentry.db.models.manager.base import BaseManager -from sentry.tasks.relay import schedule_invalidate_project_config - -logger = logging.getLogger(__name__) - - -class ReleaseProjectModelManager(BaseManager["ReleaseProject"]): - @staticmethod - def _on_post(project, trigger): - from sentry.dynamic_sampling import ProjectBoostedReleases - - project_boosted_releases = ProjectBoostedReleases(project.id) - # We want to invalidate the project config only if dynamic sampling is enabled and there exists boosted releases - # in the project. - if ( - features.has("organizations:dynamic-sampling", project.organization) - and project_boosted_releases.has_boosted_releases - ): - schedule_invalidate_project_config(project_id=project.id, trigger=trigger) - - def post_save(self, *, instance: ReleaseProject, created: bool, **kwargs: object) -> None: - self._on_post(project=instance.project, trigger="releaseproject.post_save") - - def post_delete(self, instance, **kwargs): - self._on_post(project=instance.project, trigger="releaseproject.post_delete") - - -@region_silo_model -class ReleaseProject(Model): - __relocation_scope__ = RelocationScope.Excluded - - project = FlexibleForeignKey("sentry.Project") - release = FlexibleForeignKey("sentry.Release") - new_groups = BoundedPositiveIntegerField(null=True, default=0) - - adopted = models.DateTimeField(null=True, blank=True) - unadopted = models.DateTimeField(null=True, blank=True) - first_seen_transaction = models.DateTimeField(null=True, blank=True) - - objects: ClassVar[ReleaseProjectModelManager] = ReleaseProjectModelManager() - - class Meta: - app_label = "sentry" - db_table = "sentry_release_project" - indexes = ( - models.Index(fields=("project", "adopted")), - models.Index(fields=("project", "unadopted")), - models.Index(fields=("project", "first_seen_transaction")), - ) - unique_together = (("project", "release"),) +from sentry.releases.models.releases.release_project import * # noqa: F401,F403 diff --git a/src/sentry/models/releases/set_commits.py b/src/sentry/models/releases/set_commits.py index 7ee958fcd9a274..e2fc7db2caeb1f 100644 --- a/src/sentry/models/releases/set_commits.py +++ b/src/sentry/models/releases/set_commits.py @@ -1,341 +1 @@ -from __future__ import annotations - -import itertools -import logging -import re -from typing import TypedDict - -from django.db import IntegrityError, router - -from sentry.constants import ObjectStatus -from sentry.db.postgres.transactions import in_test_hide_transaction_boundary -from sentry.locks import locks -from sentry.models.activity import Activity -from sentry.models.commitauthor import CommitAuthor -from sentry.models.commitfilechange import CommitFileChange -from sentry.models.grouphistory import GroupHistoryStatus, record_group_history -from sentry.models.groupinbox import GroupInbox, GroupInboxRemoveAction, remove_group_from_inbox -from sentry.models.release import Release -from sentry.models.releases.exceptions import ReleaseCommitError -from sentry.signals import issue_resolved -from sentry.users.services.user import RpcUser -from sentry.utils import metrics -from sentry.utils.db import atomic_transaction -from sentry.utils.retries import TimedRetryPolicy -from sentry.utils.strings import truncatechars - -logger = logging.getLogger(__name__) -from sentry.integrations.tasks.kick_off_status_syncs import kick_off_status_syncs -from sentry.models.commit import Commit -from sentry.models.group import Group, GroupStatus -from sentry.models.grouplink import GroupLink -from sentry.models.groupresolution import GroupResolution -from sentry.models.pullrequest import PullRequest -from sentry.models.releasecommit import ReleaseCommit -from sentry.models.releaseheadcommit import ReleaseHeadCommit -from sentry.models.repository import Repository -from sentry.plugins.providers.repository import RepositoryProvider - - -class _CommitDataKwargs(TypedDict, total=False): - author: CommitAuthor - message: str - date_added: str - - -def set_commits(release, commit_list): - commit_list.sort(key=lambda commit: commit.get("timestamp", 0), reverse=True) - - # todo(meredith): implement for IntegrationRepositoryProvider - commit_list = [ - c for c in commit_list if not RepositoryProvider.should_ignore_commit(c.get("message", "")) - ] - lock_key = Release.get_lock_key(release.organization_id, release.id) - # Acquire the lock for a maximum of 10 minutes - lock = locks.get(lock_key, duration=10 * 60, name="release_set_commits") - if lock.locked(): - # Signal failure to the consumer rapidly. This aims to prevent the number - # of timeouts and prevent web worker exhaustion when customers create - # the same release rapidly for different projects. - raise ReleaseCommitError - - with TimedRetryPolicy(10)(lock.acquire): - create_repositories(commit_list, release) - create_commit_authors(commit_list, release) - - with ( - atomic_transaction(using=router.db_for_write(type(release))), - in_test_hide_transaction_boundary(), - ): - - head_commit_by_repo, commit_author_by_commit = set_commits_on_release( - release, commit_list - ) - - fill_in_missing_release_head_commits(release, head_commit_by_repo) - update_group_resolutions(release, commit_author_by_commit) - - -@metrics.wraps("set_commits_on_release") -def set_commits_on_release(release, commit_list): - # TODO(dcramer): would be good to optimize the logic to avoid these - # deletes but not overly important - ReleaseCommit.objects.filter(release=release).delete() - - commit_author_by_commit = {} - head_commit_by_repo: dict[int, int] = {} - - latest_commit = None - for idx, data in enumerate(commit_list): - commit = set_commit(idx, data, release) - if idx == 0: - latest_commit = commit - - commit_author_by_commit[commit.id] = commit.author - head_commit_by_repo.setdefault(data["repo_model"].id, commit.id) - - release.update( - commit_count=len(commit_list), - authors=[ - str(a_id) - for a_id in ReleaseCommit.objects.filter( - release=release, commit__author_id__isnull=False - ) - .values_list("commit__author_id", flat=True) - .distinct() - ], - last_commit_id=latest_commit.id if latest_commit else None, - ) - return head_commit_by_repo, commit_author_by_commit - - -def set_commit(idx, data, release): - repo = data["repo_model"] - author = data["author_model"] - - commit_data: _CommitDataKwargs = {} - - # Update/set message and author if they are provided. - if author is not None: - commit_data["author"] = author - if "message" in data: - commit_data["message"] = data["message"] - if "timestamp" in data: - commit_data["date_added"] = data["timestamp"] - - commit, created = Commit.objects.get_or_create( - organization_id=release.organization_id, - repository_id=repo.id, - key=data["id"], - defaults=commit_data, - ) - if not created and any(getattr(commit, key) != value for key, value in commit_data.items()): - commit.update(**commit_data) - - if author is None: - author = commit.author - - # Guard against patch_set being None - patch_set = data.get("patch_set") or [] - if patch_set: - CommitFileChange.objects.bulk_create( - [ - CommitFileChange( - organization_id=release.organization.id, - commit=commit, - filename=patched_file["path"], - type=patched_file["type"], - ) - for patched_file in patch_set - ], - ignore_conflicts=True, - batch_size=100, - ) - - try: - with atomic_transaction(using=router.db_for_write(ReleaseCommit)): - ReleaseCommit.objects.create( - organization_id=release.organization_id, - release=release, - commit=commit, - order=idx, - ) - except IntegrityError: - pass - - return commit - - -def fill_in_missing_release_head_commits(release, head_commit_by_repo): - # fill any missing ReleaseHeadCommit entries - for repo_id, commit_id in head_commit_by_repo.items(): - try: - with atomic_transaction(using=router.db_for_write(ReleaseHeadCommit)): - ReleaseHeadCommit.objects.create( - organization_id=release.organization_id, - release_id=release.id, - repository_id=repo_id, - commit_id=commit_id, - ) - except IntegrityError: - pass - - -def update_group_resolutions(release, commit_author_by_commit): - release_commits = list( - ReleaseCommit.objects.filter(release=release) - .select_related("commit") - .values("commit_id", "commit__key") - ) - - commit_resolutions = list( - GroupLink.objects.filter( - linked_type=GroupLink.LinkedType.commit, - linked_id__in=[rc["commit_id"] for rc in release_commits], - ).values_list("group_id", "linked_id") - ) - - commit_group_authors = [ - (cr[0], commit_author_by_commit.get(cr[1])) for cr in commit_resolutions # group_id - ] - - pr_ids_by_merge_commit = list( - PullRequest.objects.filter( - merge_commit_sha__in=[rc["commit__key"] for rc in release_commits], - organization_id=release.organization_id, - ).values_list("id", flat=True) - ) - - pull_request_resolutions = list( - GroupLink.objects.filter( - relationship=GroupLink.Relationship.resolves, - linked_type=GroupLink.LinkedType.pull_request, - linked_id__in=pr_ids_by_merge_commit, - ).values_list("group_id", "linked_id") - ) - - pr_authors = list( - PullRequest.objects.filter( - id__in=[prr[1] for prr in pull_request_resolutions] - ).select_related("author") - ) - - pr_authors_dict = {pra.id: pra.author for pra in pr_authors} - - pull_request_group_authors = [ - (prr[0], pr_authors_dict.get(prr[1])) for prr in pull_request_resolutions - ] - - user_by_author: dict[CommitAuthor | None, RpcUser | None] = {None: None} - - commits_and_prs = list(itertools.chain(commit_group_authors, pull_request_group_authors)) - - group_project_lookup = dict( - Group.objects.filter(id__in=[group_id for group_id, _ in commits_and_prs]).values_list( - "id", "project_id" - ) - ) - - for group_id, author in commits_and_prs: - if author is not None and author not in user_by_author: - try: - user_by_author[author] = author.find_users()[0] - except IndexError: - user_by_author[author] = None - actor = user_by_author[author] - - with atomic_transaction( - using=( - router.db_for_write(GroupResolution), - router.db_for_write(Group), - # inside the remove_group_from_inbox - router.db_for_write(GroupInbox), - router.db_for_write(Activity), - ) - ): - GroupResolution.objects.create_or_update( - group_id=group_id, - values={ - "release": release, - "type": GroupResolution.Type.in_release, - "status": GroupResolution.Status.resolved, - "actor_id": actor.id if actor is not None else None, - }, - ) - group = Group.objects.get(id=group_id) - group.update(status=GroupStatus.RESOLVED, substatus=None) - remove_group_from_inbox(group, action=GroupInboxRemoveAction.RESOLVED, user=actor) - record_group_history(group, GroupHistoryStatus.RESOLVED, actor=actor) - - metrics.incr("group.resolved", instance="in_commit", skip_internal=True) - - issue_resolved.send_robust( - organization_id=release.organization_id, - user=actor, - group=group, - project=group.project, - resolution_type="with_commit", - sender=type(release), - ) - - kick_off_status_syncs.apply_async( - kwargs={"project_id": group_project_lookup[group_id], "group_id": group_id} - ) - - -def create_commit_authors(commit_list, release): - authors = {} - - for data in commit_list: - author_email = data.get("author_email") - if author_email is None and data.get("author_name"): - author_email = ( - re.sub(r"[^a-zA-Z0-9\-_\.]*", "", data["author_name"]).lower() + "@localhost" - ) - - author_email = truncatechars(author_email, 75) - - if not author_email: - author = None - elif author_email not in authors: - author_data = {"name": data.get("author_name")} - author, created = CommitAuthor.objects.get_or_create( - organization_id=release.organization_id, - email=author_email, - defaults=author_data, - ) - if author.name != author_data["name"]: - author.update(name=author_data["name"]) - authors[author_email] = author - else: - author = authors[author_email] - - data["author_model"] = author - - -def create_repositories(commit_list, release): - repos = {} - for data in commit_list: - repo_name = data.get("repository") or f"organization-{release.organization_id}" - if repo_name not in repos: - repo = ( - Repository.objects.filter( - organization_id=release.organization_id, - name=repo_name, - status=ObjectStatus.ACTIVE, - ) - .order_by("-pk") - .first() - ) - - if repo is None: - repo = Repository.objects.create( - organization_id=release.organization_id, - name=repo_name, - ) - - repos[repo_name] = repo - else: - repo = repos[repo_name] - - data["repo_model"] = repo +from sentry.releases.models.releases.set_commits import * # noqa: F401,F403 diff --git a/src/sentry/models/releases/util.py b/src/sentry/models/releases/util.py index 66fa5b2829f5fe..ea65236e456c92 100644 --- a/src/sentry/models/releases/util.py +++ b/src/sentry/models/releases/util.py @@ -1,256 +1 @@ -from __future__ import annotations - -import logging -from collections import namedtuple -from collections.abc import Sequence -from dataclasses import dataclass -from typing import TYPE_CHECKING, Self - -from django.db import models -from django.db.models import Case, F, Func, Q, Subquery, Value, When -from django.db.models.signals import pre_save -from sentry_relay.exceptions import RelayError -from sentry_relay.processing import parse_release - -from sentry.db.models.manager.base_query_set import BaseQuerySet -from sentry.exceptions import InvalidSearchQuery -from sentry.models.releases.release_project import ReleaseProject -from sentry.utils.numbers import validate_bigint - -if TYPE_CHECKING: - from sentry.models.release import Release # noqa: F401 - -logger = logging.getLogger(__name__) - - -class SemverVersion( - namedtuple("SemverVersion", "major minor patch revision prerelease_case prerelease") -): - pass - - -@dataclass -class SemverFilter: - operator: str - version_parts: Sequence[int | str] - package: str | Sequence[str] | None = None - negated: bool = False - - -class ReleaseQuerySet(BaseQuerySet["Release"]): - def annotate_prerelease_column(self): - """ - Adds a `prerelease_case` column to the queryset which is used to properly sort - by prerelease. We treat an empty (but not null) prerelease as higher than any - other value. - """ - return self.annotate( - prerelease_case=Case( - When(prerelease="", then=1), default=0, output_field=models.IntegerField() - ) - ) - - def filter_to_semver(self) -> Self: - """ - Filters the queryset to only include semver compatible rows - """ - return self.filter(major__isnull=False) - - def filter_by_semver_build( - self, - organization_id: int, - operator: str, - build: str, - project_ids: Sequence[int] | None = None, - negated: bool = False, - ) -> Self: - """ - Filters released by build. If the passed `build` is a numeric string, we'll filter on - `build_number` and make use of the passed operator. - If it is a non-numeric string, then we'll filter on `build_code` instead. We support a - wildcard only at the end of this string, so that we can filter efficiently via the index. - """ - qs = self.filter(organization_id=organization_id) - query_func = "exclude" if negated else "filter" - - if project_ids: - qs = qs.filter( - id__in=ReleaseProject.objects.filter(project_id__in=project_ids).values_list( - "release_id", flat=True - ) - ) - - if build.isdecimal() and validate_bigint(int(build)): - qs = getattr(qs, query_func)(**{f"build_number__{operator}": int(build)}) - else: - if not build or build.endswith("*"): - qs = getattr(qs, query_func)(build_code__startswith=build[:-1]) - else: - qs = getattr(qs, query_func)(build_code=build) - - return qs - - def filter_by_semver( - self, - organization_id: int, - semver_filter: SemverFilter, - project_ids: Sequence[int] | None = None, - ) -> Self: - """ - Filters releases based on a based `SemverFilter` instance. - `SemverFilter.version_parts` can contain up to 6 components, which should map - to the columns defined in `Release.SEMVER_COLS`. If fewer components are - included, then we will exclude later columns from the filter. - `SemverFilter.package` is optional, and if included we will filter the `package` - column using the provided value. - `SemverFilter.operator` should be a Django field filter. - - Typically we build a `SemverFilter` via `sentry.search.events.filter.parse_semver` - """ - qs = self.filter(organization_id=organization_id).annotate_prerelease_column() - query_func = "exclude" if semver_filter.negated else "filter" - - if semver_filter.package: - if isinstance(semver_filter.package, str): - qs = getattr(qs, query_func)(package=semver_filter.package) - else: - qs = getattr(qs, query_func)(package__in=semver_filter.package) - if project_ids: - qs = qs.filter( - id__in=ReleaseProject.objects.filter(project_id__in=project_ids).values_list( - "release_id", flat=True - ) - ) - - if semver_filter.version_parts: - filter_func = Func( - *( - Value(part) if isinstance(part, str) else part - for part in semver_filter.version_parts - ), - function="ROW", - ) - cols = self.model.SEMVER_COLS[: len(semver_filter.version_parts)] - qs = qs.annotate( - semver=Func( - *(F(col) for col in cols), function="ROW", output_field=models.JSONField() - ) - ) - qs = getattr(qs, query_func)(**{f"semver__{semver_filter.operator}": filter_func}) - return qs - - def filter_by_stage( - self, - organization_id: int, - operator: str, - value, - project_ids: Sequence[int] | None = None, - environments: list[str] | None = None, - ) -> Self: - from sentry.models.releaseprojectenvironment import ReleaseProjectEnvironment, ReleaseStages - from sentry.search.events.filter import to_list - - if not environments or len(environments) != 1: - raise InvalidSearchQuery("Choose a single environment to filter by release stage.") - - filters = { - ReleaseStages.ADOPTED: Q(adopted__isnull=False, unadopted__isnull=True), - ReleaseStages.REPLACED: Q(adopted__isnull=False, unadopted__isnull=False), - ReleaseStages.LOW_ADOPTION: Q(adopted__isnull=True, unadopted__isnull=True), - } - value = to_list(value) - operator_conversions = {"=": "IN", "!=": "NOT IN"} - operator = operator_conversions.get(operator, operator) - - for stage in value: - if stage not in filters: - raise InvalidSearchQuery("Unsupported release.stage value.") - - rpes = ReleaseProjectEnvironment.objects.filter( - release__organization_id=organization_id, - ).select_related("release") - - if project_ids: - rpes = rpes.filter(project_id__in=project_ids) - - query = Q() - if operator == "IN": - for stage in value: - query |= filters[stage] - elif operator == "NOT IN": - for stage in value: - query &= ~filters[stage] - - qs = self.filter(id__in=Subquery(rpes.filter(query).values_list("release_id", flat=True))) - return qs - - def order_by_recent(self) -> Self: - return self.order_by("-date_added", "-id") - - @staticmethod - def massage_semver_cols_into_release_object_data(kwargs): - """ - Helper function that takes kwargs as an argument and massages into it the release semver - columns (if possible) - Inputs: - * kwargs: data of the release that is about to be created - """ - if "version" in kwargs: - try: - version_info = parse_release(kwargs["version"]) - package = version_info.get("package") - version_parsed = version_info.get("version_parsed") - - if version_parsed is not None and all( - validate_bigint(version_parsed[field]) - for field in ("major", "minor", "patch", "revision") - ): - build_code = version_parsed.get("build_code") - build_number = ReleaseQuerySet._convert_build_code_to_build_number(build_code) - - kwargs.update( - { - "major": version_parsed.get("major"), - "minor": version_parsed.get("minor"), - "patch": version_parsed.get("patch"), - "revision": version_parsed.get("revision"), - "prerelease": version_parsed.get("pre") or "", - "build_code": build_code, - "build_number": build_number, - "package": package, - } - ) - except RelayError: - # This can happen on invalid legacy releases - pass - - @staticmethod - def _convert_build_code_to_build_number(build_code): - """ - Helper function that takes the build_code and checks if that build code can be parsed into - a 64 bit integer - Inputs: - * build_code: str - Returns: - * build_number - """ - build_number = None - if build_code is not None: - try: - build_code_as_int = int(build_code) - if validate_bigint(build_code_as_int): - build_number = build_code_as_int - except ValueError: - pass - return build_number - - -def parse_semver_pre_save(instance, **kwargs): - if instance.id: - return - ReleaseQuerySet.massage_semver_cols_into_release_object_data(instance.__dict__) - - -pre_save.connect( - parse_semver_pre_save, sender="sentry.Release", dispatch_uid="parse_semver_pre_save" -) +from sentry.releases.models.releases.util import * # noqa: F401,F403 diff --git a/src/sentry/releases/__init__.py b/src/sentry/releases/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/sentry/releases/models/__init__.py b/src/sentry/releases/models/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/sentry/releases/models/release.py b/src/sentry/releases/models/release.py new file mode 100644 index 00000000000000..0cb7e77a3a0a3a --- /dev/null +++ b/src/sentry/releases/models/release.py @@ -0,0 +1,820 @@ +# Sort commit list in reverse order +from __future__ import annotations + +import logging +from collections.abc import Mapping, Sequence +from typing import ClassVar, Literal, TypedDict + +import orjson +import sentry_sdk +from django.contrib.postgres.fields.array import ArrayField +from django.db import IntegrityError, models, router +from django.db.models import Case, Exists, F, Func, OuterRef, Sum, When +from django.utils import timezone +from django.utils.functional import cached_property +from django.utils.translation import gettext_lazy as _ +from sentry_relay.exceptions import RelayError +from sentry_relay.processing import parse_release + +from sentry.backup.scopes import RelocationScope +from sentry.constants import BAD_RELEASE_CHARS, COMMIT_RANGE_DELIMITER +from sentry.db.models import ( + BoundedBigIntegerField, + BoundedPositiveIntegerField, + FlexibleForeignKey, + JSONField, + Model, + region_silo_model, + sane_repr, +) +from sentry.db.models.fields.hybrid_cloud_foreign_key import HybridCloudForeignKey +from sentry.db.models.indexes import IndexWithPostgresNameLimits +from sentry.db.models.manager.base import BaseManager +from sentry.models.artifactbundle import ArtifactBundle +from sentry.models.commitauthor import CommitAuthor +from sentry.models.releases.constants import ( + DB_VERSION_LENGTH, + ERR_RELEASE_HEALTH_DATA, + ERR_RELEASE_REFERENCED, +) +from sentry.models.releases.exceptions import UnsafeReleaseDeletion +from sentry.models.releases.release_project import ReleaseProject +from sentry.models.releases.util import ReleaseQuerySet, SemverFilter, SemverVersion +from sentry.utils import metrics +from sentry.utils.cache import cache +from sentry.utils.db import atomic_transaction +from sentry.utils.hashlib import hash_values, md5_text +from sentry.utils.numbers import validate_bigint +from sentry.utils.sdk import set_span_attribute + +logger = logging.getLogger(__name__) + + +class _CommitDataKwargs(TypedDict, total=False): + author: CommitAuthor + message: str + date_added: str + + +class ReleaseStatus: + OPEN = 0 + ARCHIVED = 1 + + @classmethod + def from_string(cls, value): + if value == "open": + return cls.OPEN + elif value == "archived": + return cls.ARCHIVED + else: + raise ValueError(repr(value)) + + @classmethod + def to_string(cls, value): + # XXX(markus): Since the column is nullable we need to handle `null` here. + # However `null | undefined` in request payloads means "don't change + # status of release". This is why `from_string` does not consider + # `null` valid. + # + # We could remove `0` as valid state and only have `null` but I think + # that would make things worse. + # + # Eventually we should backfill releasestatus to 0 + if value is None or value == ReleaseStatus.OPEN: + return "open" + elif value == ReleaseStatus.ARCHIVED: + return "archived" + else: + raise ValueError(repr(value)) + + +def _get_cache_key(project_id: int, group_id: int, first: bool) -> str: + return f"g-r:{group_id}-{project_id}-{first}" + + +class ReleaseModelManager(BaseManager["Release"]): + def get_queryset(self) -> ReleaseQuerySet: + return ReleaseQuerySet(self.model, using=self._db) + + def annotate_prerelease_column(self): + return self.get_queryset().annotate_prerelease_column() + + def filter_to_semver(self) -> ReleaseQuerySet: + return self.get_queryset().filter_to_semver() + + def filter_by_semver_build( + self, + organization_id: int, + operator: str, + build: str, + project_ids: Sequence[int] | None = None, + negated: bool = False, + ) -> models.QuerySet: + return self.get_queryset().filter_by_semver_build( + organization_id, + operator, + build, + project_ids, + negated=negated, + ) + + def filter_by_semver( + self, + organization_id: int, + semver_filter: SemverFilter, + project_ids: Sequence[int] | None = None, + ) -> models.QuerySet: + return self.get_queryset().filter_by_semver(organization_id, semver_filter, project_ids) + + def filter_by_stage( + self, + organization_id: int, + operator: str, + value, + project_ids: Sequence[int] | None = None, + environments: list[str] | None = None, + ) -> models.QuerySet: + return self.get_queryset().filter_by_stage( + organization_id, operator, value, project_ids, environments + ) + + def order_by_recent(self): + return self.get_queryset().order_by_recent() + + def _get_group_release_version(self, group_id: int, orderby: str) -> str: + from sentry.models.grouprelease import GroupRelease + + # Using `id__in()` because there is no foreign key relationship. + return self.get( + id__in=GroupRelease.objects.filter(group_id=group_id) + .order_by(orderby) + .values("release_id")[:1] + ).version + + def get_group_release_version( + self, project_id: int, group_id: int, first: bool = True, use_cache: bool = True + ) -> str | None: + cache_key = _get_cache_key(project_id, group_id, first) + + release_version: Literal[False] | str | None = cache.get(cache_key) if use_cache else None + if release_version is False: + # We've cached the fact that no rows exist. + return None + + if release_version is None: + # Cache miss or not use_cache. + orderby = "first_seen" if first else "-last_seen" + try: + release_version = self._get_group_release_version(group_id, orderby) + except Release.DoesNotExist: + release_version = False + cache.set(cache_key, release_version, 3600) + + # Convert the False back into a None. + return release_version or None + + +@region_silo_model +class Release(Model): + """ + A release is generally created when a new version is pushed into a + production state. + + A commit is generally a git commit. See also releasecommit.py + """ + + __relocation_scope__ = RelocationScope.Excluded + + organization = FlexibleForeignKey("sentry.Organization") + projects = models.ManyToManyField( + "sentry.Project", related_name="releases", through=ReleaseProject + ) + status = BoundedPositiveIntegerField( + default=ReleaseStatus.OPEN, + null=True, + choices=( + (ReleaseStatus.OPEN, _("Open")), + (ReleaseStatus.ARCHIVED, _("Archived")), + ), + ) + + version = models.CharField(max_length=DB_VERSION_LENGTH) + # ref might be the branch name being released + ref = models.CharField(max_length=DB_VERSION_LENGTH, null=True, blank=True) + url = models.URLField(null=True, blank=True) + date_added = models.DateTimeField(default=timezone.now) + # DEPRECATED - not available in UI or editable from API + date_started = models.DateTimeField(null=True, blank=True) + date_released = models.DateTimeField(null=True, blank=True) + # arbitrary data recorded with the release + data = JSONField(default=dict) + # generally the release manager, or the person initiating the process + owner_id = HybridCloudForeignKey("sentry.User", on_delete="SET_NULL", null=True, blank=True) + + # materialized stats + commit_count = BoundedPositiveIntegerField(null=True, default=0) + last_commit_id = BoundedBigIntegerField(null=True) + authors = ArrayField(models.TextField(), default=list, null=True) + total_deploys = BoundedPositiveIntegerField(null=True, default=0) + last_deploy_id = BoundedPositiveIntegerField(null=True) + + # Denormalized semver columns. These will be filled if `version` matches at least + # part of our more permissive model of semver: + # `@...-+ + package = models.TextField(null=True) + major = models.BigIntegerField(null=True) + minor = models.BigIntegerField(null=True) + patch = models.BigIntegerField(null=True) + revision = models.BigIntegerField(null=True) + prerelease = models.TextField(null=True) + build_code = models.TextField(null=True) + # If `build_code` can be parsed as a 64 bit int we'll store it here as well for + # sorting/comparison purposes + build_number = models.BigIntegerField(null=True) + + # HACK HACK HACK + # As a transitional step we permit release rows to exist multiple times + # where they are "specialized" for a specific project. The goal is to + # later split up releases by project again. This is for instance used + # by the org release listing. + _for_project_id: int | None = None + # the user agent that set the release + user_agent = models.TextField(null=True) + + # Custom Model Manager required to override create method + objects: ClassVar[ReleaseModelManager] = ReleaseModelManager() + + class Meta: + app_label = "sentry" + db_table = "sentry_release" + unique_together = (("organization", "version"),) + indexes = [ + models.Index( + fields=["organization", "version"], + opclasses=["", "text_pattern_ops"], + name="sentry_release_version_btree", + ), + # We also use a functional index to order `prerelease` according to semver rules, + IndexWithPostgresNameLimits( + "organization", + "package", + F("major").desc(), + F("minor").desc(), + F("patch").desc(), + F("revision").desc(), + Case(When(prerelease="", then=1), default=0).desc(), + F("prerelease").desc(), + name="sentry_release_semver_by_package_idx", + ), + models.Index( + "organization", + F("major").desc(), + F("minor").desc(), + F("patch").desc(), + F("revision").desc(), + Case(When(prerelease="", then=1), default=0).desc(), + F("prerelease").desc(), + name="sentry_release_semver_idx", + ), + models.Index(fields=("organization", "build_code")), + models.Index(fields=("organization", "build_number")), + models.Index(fields=("organization", "date_added")), + models.Index(fields=("organization", "status")), + ] + + __repr__ = sane_repr("organization_id", "version") + + SEMVER_COLS = ["major", "minor", "patch", "revision", "prerelease_case", "prerelease"] + + def __eq__(self, other: object) -> bool: + """Make sure that specialized releases are only comparable to the same + other specialized release. This for instance lets us treat them + separately for serialization purposes. + """ + return ( + # don't treat `NotImplemented` as truthy + Model.__eq__(self, other) is True + and isinstance(other, Release) + and self._for_project_id == other._for_project_id + ) + + def __hash__(self): + # https://code.djangoproject.com/ticket/30333 + return super().__hash__() + + @staticmethod + def is_valid_version(value): + if value is None: + return False + + if any(c in value for c in BAD_RELEASE_CHARS): + return False + + value_stripped = str(value).strip() + return not ( + not value_stripped + or value_stripped in (".", "..") + or value_stripped.lower() == "latest" + ) + + @property + def is_semver_release(self): + return self.package is not None + + def get_previous_release(self, project): + """Get the release prior to this one. None if none exists""" + return ( + ReleaseProject.objects.filter(project=project, release__date_added__lt=self.date_added) + .order_by("-release__date_added") + .first() + ) + + @staticmethod + def is_semver_version(version): + """ + Method that checks if a version follows semantic versioning + """ + # If version is not a valid release version, or it has no package then we return False + if not Release.is_valid_version(version) or "@" not in version: + return False + + try: + version_info = parse_release(version, json_loads=orjson.loads) + version_parsed = version_info.get("version_parsed") + return version_parsed is not None and all( + validate_bigint(version_parsed[field]) + for field in ("major", "minor", "patch", "revision") + ) + except RelayError: + # This can happen on invalid legacy releases + return False + + @staticmethod + def is_release_newer_or_equal(org_id, release, other_release): + if release is None: + return False + + if other_release is None: + return True + + if release == other_release: + return True + + releases = { + release.version: float(release.date_added.timestamp()) + for release in Release.objects.filter( + organization_id=org_id, version__in=[release, other_release] + ) + } + release_date = releases.get(release) + other_release_date = releases.get(other_release) + + if release_date is not None and other_release_date is not None: + return release_date > other_release_date + + return False + + @property + def semver_tuple(self) -> SemverVersion: + return SemverVersion( + self.major, + self.minor, + self.patch, + self.revision, + 1 if self.prerelease == "" else 0, + self.prerelease, + ) + + @classmethod + def get_cache_key(cls, organization_id, version): + return f"release:3:{organization_id}:{md5_text(version).hexdigest()}" + + @classmethod + def get_lock_key(cls, organization_id, release_id): + return f"releasecommits:{organization_id}:{release_id}" + + @classmethod + def get(cls, project, version): + cache_key = cls.get_cache_key(project.organization_id, version) + + release = cache.get(cache_key) + if release is None: + try: + release = cls.objects.get( + organization_id=project.organization_id, projects=project, version=version + ) + except cls.DoesNotExist: + release = -1 + cache.set(cache_key, release, 300) + + if release == -1: + return + + return release + + @classmethod + def get_or_create(cls, project, version, date_added=None): + with metrics.timer("models.release.get_or_create") as metric_tags: + return cls._get_or_create_impl(project, version, date_added, metric_tags) + + @classmethod + def _get_or_create_impl(cls, project, version, date_added, metric_tags): + from sentry.models.project import Project + + if date_added is None: + date_added = timezone.now() + + cache_key = cls.get_cache_key(project.organization_id, version) + + release = cache.get(cache_key) + + if release in (None, -1): + # TODO(dcramer): if the cache result is -1 we could attempt a + # default create here instead of default get + project_version = (f"{project.slug}-{version}")[:DB_VERSION_LENGTH] + releases = list( + cls.objects.filter( + organization_id=project.organization_id, + version__in=[version, project_version], + projects=project, + ) + ) + + if releases: + try: + release = [r for r in releases if r.version == project_version][0] + except IndexError: + release = releases[0] + metric_tags["created"] = "false" + else: + try: + with atomic_transaction(using=router.db_for_write(cls)): + release = cls.objects.create( + organization_id=project.organization_id, + version=version, + date_added=date_added, + total_deploys=0, + ) + + metric_tags["created"] = "true" + except IntegrityError: + metric_tags["created"] = "false" + release = cls.objects.get( + organization_id=project.organization_id, version=version + ) + + # NOTE: `add_project` creates a ReleaseProject instance + release.add_project(project) + if not project.flags.has_releases: + project.flags.has_releases = True + project.update(flags=F("flags").bitor(Project.flags.has_releases)) + + # TODO(dcramer): upon creating a new release, check if it should be + # the new "latest release" for this project + cache.set(cache_key, release, 3600) + metric_tags["cache_hit"] = "false" + else: + metric_tags["cache_hit"] = "true" + + return release + + @cached_property + def version_info(self): + try: + return parse_release(self.version, json_loads=orjson.loads) + except RelayError: + # This can happen on invalid legacy releases + return None + + @classmethod + def merge(cls, to_release, from_releases): + # The following models reference release: + # ReleaseCommit.release + # ReleaseEnvironment.release_id + # ReleaseProject.release + # GroupRelease.release_id + # GroupResolution.release + # Group.first_release + # ReleaseFile.release + + from sentry.models.group import Group + from sentry.models.grouprelease import GroupRelease + from sentry.models.groupresolution import GroupResolution + from sentry.models.releasecommit import ReleaseCommit + from sentry.models.releaseenvironment import ReleaseEnvironment + from sentry.models.releasefile import ReleaseFile + from sentry.models.releaseprojectenvironment import ReleaseProjectEnvironment + from sentry.models.releases.release_project import ReleaseProject + + model_list = ( + ReleaseCommit, + ReleaseEnvironment, + ReleaseFile, + ReleaseProject, + ReleaseProjectEnvironment, + GroupRelease, + GroupResolution, + ) + for release in from_releases: + for model in model_list: + if hasattr(model, "release"): + update_kwargs = {"release": to_release} + else: + update_kwargs = {"release_id": to_release.id} + try: + with atomic_transaction(using=router.db_for_write(model)): + model.objects.filter(release_id=release.id).update(**update_kwargs) + except IntegrityError: + for item in model.objects.filter(release_id=release.id): + try: + with atomic_transaction(using=router.db_for_write(model)): + model.objects.filter(id=item.id).update(**update_kwargs) + except IntegrityError: + item.delete() + + Group.objects.filter(first_release=release).update(first_release=to_release) + + release.delete() + + def add_dist(self, name, date_added=None): + from sentry.models.distribution import Distribution + + if date_added is None: + date_added = timezone.now() + return Distribution.objects.get_or_create( + release=self, + name=name, + defaults={"date_added": date_added, "organization_id": self.organization_id}, + )[0] + + def add_project(self, project): + """ + Add a project to this release. + + Returns True if the project was added and did not already exist. + """ + from sentry.models.project import Project + + try: + with atomic_transaction(using=router.db_for_write(ReleaseProject)): + obj, created = ReleaseProject.objects.get_or_create(project=project, release=self) + if not project.flags.has_releases: + project.flags.has_releases = True + project.update(flags=F("flags").bitor(Project.flags.has_releases)) + except IntegrityError: + obj = None + created = False + + return obj, created + + def handle_commit_ranges(self, refs): + """ + Takes commit refs of the form: + [ + { + 'previousCommit': None, + 'commit': 'previous_commit..commit', + } + ] + Note: Overwrites 'previousCommit' and 'commit' + """ + for ref in refs: + if COMMIT_RANGE_DELIMITER in ref["commit"]: + ref["previousCommit"], ref["commit"] = ref["commit"].split(COMMIT_RANGE_DELIMITER) + + def set_refs(self, refs, user_id, fetch=False): + with sentry_sdk.start_span(op="set_refs"): + from sentry.api.exceptions import InvalidRepository + from sentry.models.commit import Commit + from sentry.models.releaseheadcommit import ReleaseHeadCommit + from sentry.models.repository import Repository + from sentry.tasks.commits import fetch_commits + + names = {r["repository"] for r in refs} + repos = list( + Repository.objects.filter(organization_id=self.organization_id, name__in=names) + ) + repos_by_name = {r.name: r for r in repos} + invalid_repos = names - set(repos_by_name.keys()) + if invalid_repos: + raise InvalidRepository(f"Invalid repository names: {','.join(invalid_repos)}") + + self.handle_commit_ranges(refs) + + for ref in refs: + repo = repos_by_name[ref["repository"]] + + commit = Commit.objects.get_or_create( + organization_id=self.organization_id, repository_id=repo.id, key=ref["commit"] + )[0] + # update head commit for repo/release if exists + ReleaseHeadCommit.objects.create_or_update( + organization_id=self.organization_id, + repository_id=repo.id, + release=self, + values={"commit": commit}, + ) + if fetch: + prev_release = get_previous_release(self) + fetch_commits.apply_async( + kwargs={ + "release_id": self.id, + "user_id": user_id, + "refs": refs, + "prev_release_id": prev_release and prev_release.id, + } + ) + + @sentry_sdk.trace + def set_commits(self, commit_list): + """ + Bind a list of commits to this release. + + This will clear any existing commit log and replace it with the given + commits. + """ + set_span_attribute("release.set_commits", len(commit_list)) + + from sentry.models.releases.set_commits import set_commits + + set_commits(self, commit_list) + + def safe_delete(self): + """Deletes a release if possible or raises a `UnsafeReleaseDeletion` + exception. + """ + from sentry import release_health + from sentry.models.group import Group + from sentry.models.releasefile import ReleaseFile + + # we don't want to remove the first_release metadata on the Group, and + # while people might want to kill a release (maybe to remove files), + # removing the release is prevented + if Group.objects.filter(first_release=self).exists(): + raise UnsafeReleaseDeletion(ERR_RELEASE_REFERENCED) + + # We do not allow releases with health data to be deleted because + # the upserting from snuba data would create the release again. + # We would need to be able to delete this data from snuba which we + # can't do yet. + project_ids = list(self.projects.values_list("id").all()) + if release_health.backend.check_has_health_data( + [(p[0], self.version) for p in project_ids] + ): + raise UnsafeReleaseDeletion(ERR_RELEASE_HEALTH_DATA) + + # TODO(dcramer): this needs to happen in the queue as it could be a long + # and expensive operation + file_list = ReleaseFile.objects.filter(release_id=self.id).select_related("file") + for releasefile in file_list: + releasefile.file.delete() + releasefile.delete() + self.delete() + + def count_artifacts(self): + """Sum the artifact_counts of all release files. + + An artifact count of NULL is interpreted as 1. + """ + counts = get_artifact_counts([self.id]) + return counts.get(self.id, 0) + + def count_artifacts_in_artifact_bundles(self, project_ids: Sequence[int]): + """ + Counts the number of artifacts in the artifact bundles associated with this release and a set of projects. + """ + qs = ( + ArtifactBundle.objects.filter( + organization_id=self.organization.id, + releaseartifactbundle__release_name=self.version, + projectartifactbundle__project_id__in=project_ids, + ) + .annotate(count=Sum(Func(F("artifact_count"), 1, function="COALESCE"))) + .values_list("releaseartifactbundle__release_name", "count") + ) + + qs.query.group_by = ["releaseartifactbundle__release_name"] + + if len(qs) == 0: + return None + + return qs[0] + + def clear_commits(self): + """ + Delete all release-specific commit data associated to this release. We will not delete the Commit model values because other releases may use these commits. + """ + with sentry_sdk.start_span(op="clear_commits"): + from sentry.models.releasecommit import ReleaseCommit + from sentry.models.releaseheadcommit import ReleaseHeadCommit + + ReleaseHeadCommit.objects.get( + organization_id=self.organization_id, release=self + ).delete() + ReleaseCommit.objects.filter( + organization_id=self.organization_id, release=self + ).delete() + + self.authors = [] + self.commit_count = 0 + self.last_commit_id = None + self.save() + + +def get_artifact_counts(release_ids: list[int]) -> Mapping[int, int]: + """Get artifact count grouped by IDs""" + from sentry.models.releasefile import ReleaseFile + + qs = ( + ReleaseFile.objects.filter(release_id__in=release_ids) + .annotate(count=Sum(Func(F("artifact_count"), 1, function="COALESCE"))) + .values_list("release_id", "count") + ) + qs.query.group_by = ["release_id"] + return dict(qs) + + +def follows_semver_versioning_scheme(org_id, project_id, release_version=None): + """ + Checks if we should follow semantic versioning scheme for ordering based on + 1. Latest ten releases of the project_id passed in all follow semver + 2. provided release version argument is a valid semver version + + Inputs: + * org_id + * project_id + * release_version + Returns: + Boolean that indicates if we should follow semantic version or not + """ + # TODO(ahmed): Move this function else where to be easily accessible for re-use + # TODO: this method could be moved to the Release model manager + cache_key = "follows_semver:1:%s" % hash_values([org_id, project_id]) + follows_semver = cache.get(cache_key) + + if follows_semver is None: + # Check if the latest ten releases are semver compliant + releases_list = list( + Release.objects.filter( + organization_id=org_id, projects__id__in=[project_id], status=ReleaseStatus.OPEN + ) + .using_replica() + .order_by("-date_added")[:10] + ) + + if not releases_list: + cache.set(cache_key, False, 3600) + return False + + # TODO(ahmed): re-visit/replace these conditions once we enable project wide `semver` setting + # A project is said to be following semver versioning schemes if it satisfies the following + # conditions:- + # 1: At least one semver compliant in the most recent 3 releases + # 2: At least 3 semver compliant releases in the most recent 10 releases + if len(releases_list) <= 2: + # Most recent release is considered to decide if project follows semver + follows_semver = releases_list[0].is_semver_release + elif len(releases_list) < 10: + # We forego condition 2 and it is enough if condition 1 is satisfied to consider this + # project to have semver compliant releases + follows_semver = any(release.is_semver_release for release in releases_list[0:3]) + else: + # Count number of semver releases in the last ten + semver_matches = sum(map(lambda release: release.is_semver_release, releases_list)) + + at_least_three_in_last_ten = semver_matches >= 3 + at_least_one_in_last_three = any( + release.is_semver_release for release in releases_list[0:3] + ) + + follows_semver = at_least_one_in_last_three and at_least_three_in_last_ten + cache.set(cache_key, follows_semver, 3600) + + # Check release_version that is passed is semver compliant + if release_version: + follows_semver = follows_semver and Release.is_semver_version(release_version) + return follows_semver + + +def get_previous_release(release: Release) -> Release | None: + # NOTE: Keeping the below todo. Just optimizing the query. + # + # TODO: this does the wrong thing unless you are on the most + # recent release. Add a timestamp compare? + return ( + Release.objects.filter(organization_id=release.organization_id) + .filter( + Exists( + ReleaseProject.objects.filter( + release=OuterRef("pk"), + project_id__in=ReleaseProject.objects.filter(release=release).values_list( + "project_id", flat=True + ), + ) + ) + ) + .extra(select={"sort": "COALESCE(date_released, date_added)"}) + .exclude(version=release.version) + .order_by("-sort") + .first() + ) diff --git a/src/sentry/releases/models/release_threshold/__init__.py b/src/sentry/releases/models/release_threshold/__init__.py new file mode 100644 index 00000000000000..ab449eab63b6ad --- /dev/null +++ b/src/sentry/releases/models/release_threshold/__init__.py @@ -0,0 +1,3 @@ +from .release_threshold import ReleaseThreshold + +__all__ = ("ReleaseThreshold",) diff --git a/src/sentry/releases/models/release_threshold/constants.py b/src/sentry/releases/models/release_threshold/constants.py new file mode 100644 index 00000000000000..e4cd871db21494 --- /dev/null +++ b/src/sentry/releases/models/release_threshold/constants.py @@ -0,0 +1,93 @@ +class ReleaseThresholdType: + TOTAL_ERROR_COUNT = 0 + NEW_ISSUE_COUNT = 1 + UNHANDLED_ISSUE_COUNT = 2 + REGRESSED_ISSUE_COUNT = 3 + FAILURE_RATE = 4 + CRASH_FREE_SESSION_RATE = 5 + CRASH_FREE_USER_RATE = 6 + + TOTAL_ERROR_COUNT_STR = "total_error_count" + NEW_ISSUE_COUNT_STR = "new_issue_count" + UNHANDLED_ISSUE_COUNT_STR = "unhandled_issue_count" + REGRESSED_ISSUE_COUNT_STR = "regressed_issue_count" + FAILURE_RATE_STR = "failure_rate" + CRASH_FREE_SESSION_RATE_STR = "crash_free_session_rate" + CRASH_FREE_USER_RATE_STR = "crash_free_user_rate" + + @classmethod + def as_choices(cls): + return ( + (cls.TOTAL_ERROR_COUNT_STR, cls.TOTAL_ERROR_COUNT), + (cls.NEW_ISSUE_COUNT_STR, cls.NEW_ISSUE_COUNT), + (cls.UNHANDLED_ISSUE_COUNT_STR, cls.UNHANDLED_ISSUE_COUNT), + (cls.REGRESSED_ISSUE_COUNT_STR, cls.REGRESSED_ISSUE_COUNT), + (cls.FAILURE_RATE_STR, cls.FAILURE_RATE), + (cls.CRASH_FREE_SESSION_RATE_STR, cls.CRASH_FREE_SESSION_RATE), + (cls.CRASH_FREE_USER_RATE_STR, cls.CRASH_FREE_USER_RATE), + ) + + @classmethod + def as_str_choices(cls): + return ( + (cls.TOTAL_ERROR_COUNT_STR, cls.TOTAL_ERROR_COUNT_STR), + (cls.NEW_ISSUE_COUNT_STR, cls.NEW_ISSUE_COUNT_STR), + (cls.UNHANDLED_ISSUE_COUNT_STR, cls.UNHANDLED_ISSUE_COUNT_STR), + (cls.REGRESSED_ISSUE_COUNT_STR, cls.REGRESSED_ISSUE_COUNT_STR), + (cls.FAILURE_RATE_STR, cls.FAILURE_RATE_STR), + (cls.CRASH_FREE_SESSION_RATE_STR, cls.CRASH_FREE_SESSION_RATE_STR), + (cls.CRASH_FREE_USER_RATE_STR, cls.CRASH_FREE_USER_RATE_STR), + ) + + +class TriggerType: + OVER = 0 + UNDER = 1 + + OVER_STR = "over" + UNDER_STR = "under" + + @classmethod + def as_choices(cls): # choices for model column + return ( + (cls.OVER_STR, cls.OVER), + (cls.UNDER_STR, cls.UNDER), + ) + + @classmethod + def as_str_choices(cls): # choices for serializer + return ( + (cls.OVER_STR, cls.OVER_STR), + (cls.UNDER_STR, cls.UNDER_STR), + ) + + +THRESHOLD_TYPE_INT_TO_STR = { + ReleaseThresholdType.TOTAL_ERROR_COUNT: ReleaseThresholdType.TOTAL_ERROR_COUNT_STR, + ReleaseThresholdType.NEW_ISSUE_COUNT: ReleaseThresholdType.NEW_ISSUE_COUNT_STR, + ReleaseThresholdType.UNHANDLED_ISSUE_COUNT: ReleaseThresholdType.UNHANDLED_ISSUE_COUNT_STR, + ReleaseThresholdType.REGRESSED_ISSUE_COUNT: ReleaseThresholdType.REGRESSED_ISSUE_COUNT_STR, + ReleaseThresholdType.FAILURE_RATE: ReleaseThresholdType.FAILURE_RATE_STR, + ReleaseThresholdType.CRASH_FREE_SESSION_RATE: ReleaseThresholdType.CRASH_FREE_SESSION_RATE_STR, + ReleaseThresholdType.CRASH_FREE_USER_RATE: ReleaseThresholdType.CRASH_FREE_USER_RATE_STR, +} + +THRESHOLD_TYPE_STR_TO_INT = { + ReleaseThresholdType.TOTAL_ERROR_COUNT_STR: ReleaseThresholdType.TOTAL_ERROR_COUNT, + ReleaseThresholdType.NEW_ISSUE_COUNT_STR: ReleaseThresholdType.NEW_ISSUE_COUNT, + ReleaseThresholdType.UNHANDLED_ISSUE_COUNT_STR: ReleaseThresholdType.UNHANDLED_ISSUE_COUNT, + ReleaseThresholdType.REGRESSED_ISSUE_COUNT_STR: ReleaseThresholdType.REGRESSED_ISSUE_COUNT, + ReleaseThresholdType.FAILURE_RATE_STR: ReleaseThresholdType.FAILURE_RATE, + ReleaseThresholdType.CRASH_FREE_SESSION_RATE_STR: ReleaseThresholdType.CRASH_FREE_SESSION_RATE, + ReleaseThresholdType.CRASH_FREE_USER_RATE_STR: ReleaseThresholdType.CRASH_FREE_USER_RATE, +} + +TRIGGER_TYPE_INT_TO_STR = { + TriggerType.OVER: TriggerType.OVER_STR, + TriggerType.UNDER: TriggerType.UNDER_STR, +} + +TRIGGER_TYPE_STRING_TO_INT = { + TriggerType.OVER_STR: TriggerType.OVER, + TriggerType.UNDER_STR: TriggerType.UNDER, +} diff --git a/src/sentry/releases/models/release_threshold/release_threshold.py b/src/sentry/releases/models/release_threshold/release_threshold.py new file mode 100644 index 00000000000000..a9acb5f2d19ac6 --- /dev/null +++ b/src/sentry/releases/models/release_threshold/release_threshold.py @@ -0,0 +1,34 @@ +from django.db import models +from django.utils import timezone + +from sentry.backup.scopes import RelocationScope +from sentry.db.models import FlexibleForeignKey, Model, region_silo_model +from sentry.db.models.fields.bounded import BoundedPositiveIntegerField +from sentry.models.release_threshold.constants import ReleaseThresholdType +from sentry.models.release_threshold.constants import TriggerType as ReleaseThresholdTriggerType + + +@region_silo_model +class ReleaseThreshold(Model): + """ + NOTE: + To transition to utilizing AlertRules, there are some duplicated attrs we'll want to dedup. + AlertRule model should house metadata on the AlertRule itself (eg. type of alert rule) + AlertRuleTrigger model should house the trigger requirements (eg. value, over/under trigger type) + - TODO: Will need to determine how this translates to release_threshold evaluation + QuerySubscription model subscribes the AlertRule to specific query in Snuba + SnubaQuery model represents the actual query run in Snuba + - TODO: replace query constructed in release_thresholds api with activated SnubaQuery / determine whether we're constructing the same query or not + """ + + __relocation_scope__ = RelocationScope.Excluded + + threshold_type = BoundedPositiveIntegerField(choices=ReleaseThresholdType.as_choices()) + trigger_type = BoundedPositiveIntegerField(choices=ReleaseThresholdTriggerType.as_choices()) + + value = models.IntegerField() + window_in_seconds = models.PositiveIntegerField() + + project = FlexibleForeignKey("sentry.Project", db_index=True, related_name="release_thresholds") + environment = FlexibleForeignKey("sentry.Environment", null=True, db_index=True) + date_added = models.DateTimeField(default=timezone.now) diff --git a/src/sentry/releases/models/releaseactivity.py b/src/sentry/releases/models/releaseactivity.py new file mode 100644 index 00000000000000..12b91f0c558a2c --- /dev/null +++ b/src/sentry/releases/models/releaseactivity.py @@ -0,0 +1,25 @@ +from django.db import models +from django.utils import timezone + +from sentry.backup.scopes import RelocationScope +from sentry.db.models import ( + BoundedPositiveIntegerField, + FlexibleForeignKey, + Model, + region_silo_model, +) +from sentry.types.releaseactivity import CHOICES + + +@region_silo_model +class ReleaseActivity(Model): + __relocation_scope__ = RelocationScope.Excluded + + release = FlexibleForeignKey("sentry.Release", db_index=True) + type = BoundedPositiveIntegerField(null=False, choices=CHOICES) + data = models.JSONField(default=dict) + date_added = models.DateTimeField(default=timezone.now) + + class Meta: + app_label = "sentry" + db_table = "sentry_releaseactivity" diff --git a/src/sentry/releases/models/releasecommit.py b/src/sentry/releases/models/releasecommit.py new file mode 100644 index 00000000000000..b6c42c36d0240b --- /dev/null +++ b/src/sentry/releases/models/releasecommit.py @@ -0,0 +1,28 @@ +from sentry.backup.scopes import RelocationScope +from sentry.db.models import ( + BoundedBigIntegerField, + BoundedPositiveIntegerField, + FlexibleForeignKey, + Model, + region_silo_model, + sane_repr, +) + + +@region_silo_model +class ReleaseCommit(Model): + __relocation_scope__ = RelocationScope.Excluded + + organization_id = BoundedBigIntegerField(db_index=True) + # DEPRECATED + project_id = BoundedBigIntegerField(null=True) + release = FlexibleForeignKey("sentry.Release") + commit = FlexibleForeignKey("sentry.Commit") + order = BoundedPositiveIntegerField() + + class Meta: + app_label = "sentry" + db_table = "sentry_releasecommit" + unique_together = (("release", "commit"), ("release", "order")) + + __repr__ = sane_repr("release_id", "commit_id", "order") diff --git a/src/sentry/releases/models/releaseenvironment.py b/src/sentry/releases/models/releaseenvironment.py new file mode 100644 index 00000000000000..e88dd90727d70a --- /dev/null +++ b/src/sentry/releases/models/releaseenvironment.py @@ -0,0 +1,79 @@ +from datetime import timedelta + +from django.db import models +from django.utils import timezone + +from sentry.backup.scopes import RelocationScope +from sentry.db.models import ( + BoundedBigIntegerField, + FlexibleForeignKey, + Model, + region_silo_model, + sane_repr, +) +from sentry.utils import metrics +from sentry.utils.cache import cache + + +@region_silo_model +class ReleaseEnvironment(Model): + __relocation_scope__ = RelocationScope.Excluded + + organization = FlexibleForeignKey("sentry.Organization", db_index=True, db_constraint=False) + # DEPRECATED + project_id = BoundedBigIntegerField(null=True) + release = FlexibleForeignKey("sentry.Release", db_index=True, db_constraint=False) + environment = FlexibleForeignKey("sentry.Environment", db_index=True, db_constraint=False) + first_seen = models.DateTimeField(default=timezone.now) + last_seen = models.DateTimeField(default=timezone.now, db_index=True) + + class Meta: + app_label = "sentry" + db_table = "sentry_environmentrelease" + unique_together = (("organization", "release", "environment"),) + + __repr__ = sane_repr("organization_id", "release_id", "environment_id") + + @classmethod + def get_cache_key(cls, organization_id, release_id, environment_id): + return f"releaseenv:2:{organization_id}:{release_id}:{environment_id}" + + @classmethod + def get_or_create(cls, project, release, environment, datetime, **kwargs): + with metrics.timer("models.releaseenvironment.get_or_create") as metric_tags: + return cls._get_or_create_impl(project, release, environment, datetime, metric_tags) + + @classmethod + def _get_or_create_impl(cls, project, release, environment, datetime, metric_tags): + cache_key = cls.get_cache_key(project.id, release.id, environment.id) + + instance = cache.get(cache_key) + if instance is None: + metric_tags["cache_hit"] = "false" + instance, created = cls.objects.get_or_create( + release_id=release.id, + organization_id=project.organization_id, + environment_id=environment.id, + defaults={"first_seen": datetime, "last_seen": datetime}, + ) + cache.set(cache_key, instance, 3600) + else: + metric_tags["cache_hit"] = "true" + created = False + + metric_tags["created"] = "true" if created else "false" + + # TODO(dcramer): this would be good to buffer, but until then we minimize + # updates to once a minute, and allow Postgres to optimistically skip + # it even if we can't + if not created and instance.last_seen < datetime - timedelta(seconds=60): + metric_tags["bumped"] = "true" + cls.objects.filter( + id=instance.id, last_seen__lt=datetime - timedelta(seconds=60) + ).update(last_seen=datetime) + instance.last_seen = datetime + cache.set(cache_key, instance, 3600) + else: + metric_tags["bumped"] = "false" + + return instance diff --git a/src/sentry/releases/models/releasefile.py b/src/sentry/releases/models/releasefile.py new file mode 100644 index 00000000000000..a2c19158983305 --- /dev/null +++ b/src/sentry/releases/models/releasefile.py @@ -0,0 +1,404 @@ +from __future__ import annotations + +import logging +import zipfile +from contextlib import contextmanager +from hashlib import sha1 +from io import BytesIO +from tempfile import TemporaryDirectory +from typing import IO, ClassVar, Self +from urllib.parse import urlunsplit + +import sentry_sdk +from django.db import models, router + +from sentry.backup.scopes import RelocationScope +from sentry.db.models import ( + BoundedBigIntegerField, + BoundedPositiveIntegerField, + FlexibleForeignKey, + Model, + region_silo_model, + sane_repr, +) +from sentry.db.models.manager.base import BaseManager +from sentry.models.distribution import Distribution +from sentry.models.files.file import File +from sentry.models.release import Release +from sentry.utils import json +from sentry.utils.db import atomic_transaction +from sentry.utils.hashlib import sha1_text +from sentry.utils.urls import urlsplit_best_effort +from sentry.utils.zip import safe_extract_zip + +logger = logging.getLogger(__name__) + + +ARTIFACT_INDEX_FILENAME = "artifact-index.json" +ARTIFACT_INDEX_TYPE = "release.artifact-index" + + +class PublicReleaseFileManager(models.Manager["ReleaseFile"]): + """Manager for all release files that are not internal. + + Internal release files include: + * Uploaded release archives + * Artifact index mapping URLs to release archives + + This manager has the overhead of always joining the File table in order + to filter release files. + + """ + + def get_queryset(self): + return super().get_queryset().select_related("file").filter(file__type="release.file") + + +@region_silo_model +class ReleaseFile(Model): + r""" + A ReleaseFile is an association between a Release and a File. + + The ident of the file should be sha1(name) or + sha1(name '\x00\x00' dist.name) and must be unique per release. + """ + + __relocation_scope__ = RelocationScope.Excluded + + organization_id = BoundedBigIntegerField(db_index=True) + # DEPRECATED + project_id = BoundedBigIntegerField(null=True) + release_id = BoundedBigIntegerField(db_index=True) + file = FlexibleForeignKey("sentry.File") + ident = models.CharField(max_length=40) + name = models.TextField() + dist_id = BoundedBigIntegerField(null=True, db_index=True) + + #: For classic file uploads, this field is 1. + #: For release archives, this field is 0. + #: For artifact indexes, this field is the number of artifacts contained + #: in the index. + artifact_count = BoundedPositiveIntegerField(null=True, default=1) + + __repr__ = sane_repr("release", "ident") + + objects: ClassVar[BaseManager[Self]] = BaseManager() # The default manager. + public_objects: ClassVar[PublicReleaseFileManager] = PublicReleaseFileManager() + + class Meta: + unique_together = (("release_id", "ident"),) + indexes = (models.Index(fields=("release_id", "name")),) + app_label = "sentry" + db_table = "sentry_releasefile" + + def save(self, *args, **kwargs): + from sentry.models.distribution import Distribution + + if not self.ident and self.name: + dist = None + if self.dist_id: + dist = Distribution.objects.get(pk=self.dist_id).name + self.ident = type(self).get_ident(self.name, dist) + return super().save(*args, **kwargs) + + def update(self, *args, **kwargs): + # If our name is changing, we must also change the ident + if "name" in kwargs and "ident" not in kwargs: + dist_name = None + dist_id = kwargs.get("dist_id") or self.dist_id + if dist_id: + dist_name = Distribution.objects.filter(pk=dist_id).values_list("name", flat=True)[ + 0 + ] + kwargs["ident"] = self.ident = type(self).get_ident(kwargs["name"], dist_name) + return super().update(*args, **kwargs) + + @classmethod + def get_ident(cls, name, dist=None): + if dist is not None: + return sha1_text(name + "\x00\x00" + dist).hexdigest() + return sha1_text(name).hexdigest() + + @classmethod + def normalize(cls, url): + """Transforms a full absolute url into 2 or 4 generalized options + + * the original url as input + * (optional) original url without querystring + * the full url, but stripped of scheme and netloc + * (optional) full url without scheme and netloc or querystring + """ + # Always ignore the fragment + scheme, netloc, path, query = urlsplit_best_effort(url) + + uri_without_fragment = (scheme, netloc, path, query, "") + uri_relative = ("", "", path, query, "") + uri_without_query = (scheme, netloc, path, "", "") + uri_relative_without_query = ("", "", path, "", "") + + urls = [urlunsplit(uri_without_fragment)] + if query: + urls.append(urlunsplit(uri_without_query)) + urls.append("~" + urlunsplit(uri_relative)) + if query: + urls.append("~" + urlunsplit(uri_relative_without_query)) + return urls + + +class ReleaseArchive: + """Read-only view of uploaded ZIP-archive of release files""" + + def __init__(self, fileobj: IO): + self._fileobj = fileobj + self._zip_file = zipfile.ZipFile(self._fileobj) + self.manifest = self._read_manifest() + self.artifact_count = len(self.manifest.get("files", {})) + files = self.manifest.get("files", {}) + self._entries_by_url = {entry["url"]: (path, entry) for path, entry in files.items()} + + def __enter__(self): + return self + + def __exit__(self, exc, value, tb): + self.close() + + def close(self): + self._zip_file.close() + self._fileobj.close() + + def info(self, filename: str) -> zipfile.ZipInfo: + return self._zip_file.getinfo(filename) + + def read(self, filename: str) -> bytes: + return self._zip_file.read(filename) + + def _read_manifest(self) -> dict: + manifest_bytes = self.read("manifest.json") + return json.loads(manifest_bytes.decode("utf-8")) + + def get_file_by_url(self, url: str) -> tuple[IO[bytes], dict]: + """Return file-like object and headers. + + The caller is responsible for closing the returned stream. + + May raise ``KeyError`` + """ + filename, entry = self._entries_by_url[url] + return self._zip_file.open(filename), entry.get("headers", {}) + + def extract(self) -> TemporaryDirectory: + """Extract contents to a temporary directory. + + The caller is responsible for cleanup of the temporary files. + """ + temp_dir = TemporaryDirectory() + safe_extract_zip(self._fileobj, temp_dir.name) + + return temp_dir + + +class _ArtifactIndexData: + """Holds data of artifact index and keeps track of changes""" + + def __init__(self, data: dict, fresh=False): + self._data = data + self.changed = fresh + + @property + def data(self): + """Meant to be read-only""" + return self._data + + @property + def num_files(self): + return len(self._data.get("files", {})) + + def get(self, filename: str): + return self._data.get("files", {}).get(filename, None) + + def update_files(self, files: dict): + if files: + self._data.setdefault("files", {}).update(files) + self.changed = True + + def delete(self, filename: str) -> bool: + result = self._data.get("files", {}).pop(filename, None) + deleted = result is not None + if deleted: + self.changed = True + + return deleted + + +class _ArtifactIndexGuard: + """Ensures atomic write operations to the artifact index""" + + def __init__(self, release: Release, dist: Distribution | None, **filter_args): + self._release = release + self._dist = dist + self._ident = ReleaseFile.get_ident(ARTIFACT_INDEX_FILENAME, dist and dist.name) + self._filter_args = filter_args # Extra constraints on artifact index release file + + def readable_data(self) -> dict | None: + """Simple read, no synchronization necessary""" + try: + releasefile = self._releasefile_qs()[0] + except IndexError: + return None + else: + fp = releasefile.file.getfile() + with fp: + return json.load(fp) + + @contextmanager + def writable_data(self, create: bool, initial_artifact_count=None): + """Context manager for editable artifact index""" + with atomic_transaction( + using=( + router.db_for_write(ReleaseFile), + router.db_for_write(File), + ) + ): + created = False + if create: + releasefile, created = self._get_or_create_releasefile(initial_artifact_count) + else: + # Lock the row for editing: + # NOTE: Do not select_related('file') here, because we do not + # want to lock the File table + qs = self._releasefile_qs().select_for_update() + try: + releasefile = qs[0] + except IndexError: + releasefile = None + + if releasefile is None: + index_data = None + else: + if created: + index_data = _ArtifactIndexData({}, fresh=True) + else: + source_file = releasefile.file + if source_file.type != ARTIFACT_INDEX_TYPE: + raise RuntimeError("Unexpected file type for artifact index") + raw_data = json.load(source_file.getfile()) + index_data = _ArtifactIndexData(raw_data) + + yield index_data # editable reference to index + + if index_data is not None and index_data.changed: + if created: + target_file = releasefile.file + else: + target_file = File.objects.create( + name=ARTIFACT_INDEX_FILENAME, type=ARTIFACT_INDEX_TYPE + ) + + target_file.putfile(BytesIO(json.dumps(index_data.data).encode())) + + artifact_count = index_data.num_files + if not created: + # Update and clean existing + old_file = releasefile.file + releasefile.update(file=target_file, artifact_count=artifact_count) + old_file.delete() + + def _get_or_create_releasefile(self, initial_artifact_count): + """Make sure that the release file exists""" + return ReleaseFile.objects.select_for_update().get_or_create( + **self._key_fields(), + defaults={ + "artifact_count": initial_artifact_count, + "file": lambda: File.objects.create( + name=ARTIFACT_INDEX_FILENAME, + type=ARTIFACT_INDEX_TYPE, + ), + }, + ) + + def _releasefile_qs(self): + """QuerySet for selecting artifact index""" + return ReleaseFile.objects.filter(**self._key_fields(), **self._filter_args) + + def _key_fields(self): + """Columns needed to identify the artifact index in the db""" + return dict( + organization_id=self._release.organization_id, + release_id=self._release.id, + dist_id=self._dist.id if self._dist else self._dist, + name=ARTIFACT_INDEX_FILENAME, + ident=self._ident, + ) + + +@sentry_sdk.tracing.trace +def read_artifact_index(release: Release, dist: Distribution | None, **filter_args) -> dict | None: + """Get index data""" + guard = _ArtifactIndexGuard(release, dist, **filter_args) + return guard.readable_data() + + +def _compute_sha1(archive: ReleaseArchive, url: str) -> str: + data = archive.read(url) + return sha1(data).hexdigest() + + +@sentry_sdk.tracing.trace +def update_artifact_index( + release: Release, + dist: Distribution | None, + archive_file: File, + temp_file: IO | None = None, +): + """Add information from release archive to artifact index + + :returns: The created ReleaseFile instance + """ + releasefile = ReleaseFile.objects.create( + name=archive_file.name, + release_id=release.id, + organization_id=release.organization_id, + dist_id=dist.id if dist is not None else dist, + file=archive_file, + artifact_count=0, # Artifacts will be counted with artifact index + ) + + files_out = {} + with ReleaseArchive(temp_file or archive_file.getfile()) as archive: + manifest = archive.manifest + + files = manifest.get("files", {}) + if not files: + return + + for filename, info in files.items(): + info = info.copy() + url = info.pop("url") + info["filename"] = filename + info["archive_ident"] = releasefile.ident + info["date_created"] = archive_file.timestamp + info["sha1"] = _compute_sha1(archive, filename) + info["size"] = archive.info(filename).file_size + files_out[url] = info + + guard = _ArtifactIndexGuard(release, dist) + with guard.writable_data(create=True, initial_artifact_count=len(files_out)) as index_data: + index_data.update_files(files_out) + + return releasefile + + +@sentry_sdk.tracing.trace +def delete_from_artifact_index(release: Release, dist: Distribution | None, url: str) -> bool: + """Delete the file with the given url from the manifest. + + Does *not* delete the file from the zip archive. + + :returns: True if deleted + """ + guard = _ArtifactIndexGuard(release, dist) + with guard.writable_data(create=False) as index_data: + if index_data is not None: + return index_data.delete(url) + + return False diff --git a/src/sentry/releases/models/releaseheadcommit.py b/src/sentry/releases/models/releaseheadcommit.py new file mode 100644 index 00000000000000..b66e31a0f6534c --- /dev/null +++ b/src/sentry/releases/models/releaseheadcommit.py @@ -0,0 +1,26 @@ +from sentry.backup.scopes import RelocationScope +from sentry.db.models import ( + BoundedBigIntegerField, + BoundedPositiveIntegerField, + FlexibleForeignKey, + Model, + region_silo_model, + sane_repr, +) + + +@region_silo_model +class ReleaseHeadCommit(Model): + __relocation_scope__ = RelocationScope.Excluded + + organization_id = BoundedBigIntegerField(db_index=True) + repository_id = BoundedPositiveIntegerField() + release = FlexibleForeignKey("sentry.Release") + commit = FlexibleForeignKey("sentry.Commit") + + class Meta: + app_label = "sentry" + db_table = "sentry_releaseheadcommit" + unique_together = (("repository_id", "release"),) + + __repr__ = sane_repr("release_id", "commit_id", "repository_id") diff --git a/src/sentry/releases/models/releaseprojectenvironment.py b/src/sentry/releases/models/releaseprojectenvironment.py new file mode 100644 index 00000000000000..c001d9ba65ec2c --- /dev/null +++ b/src/sentry/releases/models/releaseprojectenvironment.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from datetime import timedelta +from enum import Enum + +from django.db import models +from django.utils import timezone + +from sentry.backup.scopes import RelocationScope +from sentry.db.models import ( + BoundedPositiveIntegerField, + FlexibleForeignKey, + Model, + region_silo_model, + sane_repr, +) +from sentry.utils import metrics +from sentry.utils.cache import cache + + +class ReleaseStages(str, Enum): + ADOPTED = "adopted" + LOW_ADOPTION = "low_adoption" + REPLACED = "replaced" + + +@region_silo_model +class ReleaseProjectEnvironment(Model): + __relocation_scope__ = RelocationScope.Excluded + + release = FlexibleForeignKey("sentry.Release") + project = FlexibleForeignKey("sentry.Project") + environment = FlexibleForeignKey("sentry.Environment") + new_issues_count = BoundedPositiveIntegerField(default=0) + first_seen = models.DateTimeField(default=timezone.now) + last_seen = models.DateTimeField(default=timezone.now, db_index=True) + last_deploy_id = BoundedPositiveIntegerField(null=True, db_index=True) + + adopted = models.DateTimeField(null=True, blank=True) + unadopted = models.DateTimeField(null=True, blank=True) + + class Meta: + app_label = "sentry" + db_table = "sentry_releaseprojectenvironment" + indexes = ( + models.Index(fields=("project", "adopted", "environment")), + models.Index(fields=("project", "unadopted", "environment")), + ) + unique_together = (("project", "release", "environment"),) + + __repr__ = sane_repr("project", "release", "environment") + + @classmethod + def get_cache_key(cls, release_id, project_id, environment_id): + return f"releaseprojectenv:{release_id}:{project_id}:{environment_id}" + + @classmethod + def get_or_create(cls, release, project, environment, datetime, **kwargs): + with metrics.timer("models.releaseprojectenvironment.get_or_create") as metrics_tags: + return cls._get_or_create_impl( + release, project, environment, datetime, metrics_tags, **kwargs + ) + + @classmethod + def _get_or_create_impl(cls, release, project, environment, datetime, metrics_tags, **kwargs): + cache_key = cls.get_cache_key(project.id, release.id, environment.id) + + instance = cache.get(cache_key) + if instance is None: + metrics_tags["cache_hit"] = "false" + instance, created = cls.objects.get_or_create( + release=release, + project=project, + environment=environment, + defaults={"first_seen": datetime, "last_seen": datetime}, + ) + cache.set(cache_key, instance, 3600) + else: + metrics_tags["cache_hit"] = "true" + created = False + + metrics_tags["created"] = "true" if created else "false" + + # Same as releaseenvironment model. Minimizes last_seen updates to once a minute + if not created and instance.last_seen < datetime - timedelta(seconds=60): + cls.objects.filter( + id=instance.id, last_seen__lt=datetime - timedelta(seconds=60) + ).update(last_seen=datetime) + instance.last_seen = datetime + cache.set(cache_key, instance, 3600) + metrics_tags["bumped"] = "true" + else: + metrics_tags["bumped"] = "false" + + return instance + + @property + def adoption_stages(self): + if self.adopted is not None and self.unadopted is None: + stage = ReleaseStages.ADOPTED + elif self.adopted is not None and self.unadopted is not None: + stage = ReleaseStages.REPLACED + else: + stage = ReleaseStages.LOW_ADOPTION + + return {"stage": stage, "adopted": self.adopted, "unadopted": self.unadopted} diff --git a/src/sentry/releases/models/releases/__init__.py b/src/sentry/releases/models/releases/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/sentry/releases/models/releases/constants.py b/src/sentry/releases/models/releases/constants.py new file mode 100644 index 00000000000000..47ce332a21e834 --- /dev/null +++ b/src/sentry/releases/models/releases/constants.py @@ -0,0 +1,5 @@ +DB_VERSION_LENGTH = 250 + + +ERR_RELEASE_REFERENCED = "This release is referenced by active issues and cannot be removed." +ERR_RELEASE_HEALTH_DATA = "This release has health data and cannot be removed." diff --git a/src/sentry/releases/models/releases/exceptions.py b/src/sentry/releases/models/releases/exceptions.py new file mode 100644 index 00000000000000..d1d65b6f8d30b0 --- /dev/null +++ b/src/sentry/releases/models/releases/exceptions.py @@ -0,0 +1,6 @@ +class UnsafeReleaseDeletion(Exception): + pass + + +class ReleaseCommitError(Exception): + pass diff --git a/src/sentry/releases/models/releases/release_project.py b/src/sentry/releases/models/releases/release_project.py new file mode 100644 index 00000000000000..889a5b080116d1 --- /dev/null +++ b/src/sentry/releases/models/releases/release_project.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import logging +from typing import ClassVar + +from django.db import models + +from sentry import features +from sentry.backup.scopes import RelocationScope +from sentry.db.models import ( + BoundedPositiveIntegerField, + FlexibleForeignKey, + Model, + region_silo_model, +) +from sentry.db.models.manager.base import BaseManager +from sentry.tasks.relay import schedule_invalidate_project_config + +logger = logging.getLogger(__name__) + + +class ReleaseProjectModelManager(BaseManager["ReleaseProject"]): + @staticmethod + def _on_post(project, trigger): + from sentry.dynamic_sampling import ProjectBoostedReleases + + project_boosted_releases = ProjectBoostedReleases(project.id) + # We want to invalidate the project config only if dynamic sampling is enabled and there exists boosted releases + # in the project. + if ( + features.has("organizations:dynamic-sampling", project.organization) + and project_boosted_releases.has_boosted_releases + ): + schedule_invalidate_project_config(project_id=project.id, trigger=trigger) + + def post_save(self, *, instance: ReleaseProject, created: bool, **kwargs: object) -> None: + self._on_post(project=instance.project, trigger="releaseproject.post_save") + + def post_delete(self, instance, **kwargs): + self._on_post(project=instance.project, trigger="releaseproject.post_delete") + + +@region_silo_model +class ReleaseProject(Model): + __relocation_scope__ = RelocationScope.Excluded + + project = FlexibleForeignKey("sentry.Project") + release = FlexibleForeignKey("sentry.Release") + new_groups = BoundedPositiveIntegerField(null=True, default=0) + + adopted = models.DateTimeField(null=True, blank=True) + unadopted = models.DateTimeField(null=True, blank=True) + first_seen_transaction = models.DateTimeField(null=True, blank=True) + + objects: ClassVar[ReleaseProjectModelManager] = ReleaseProjectModelManager() + + class Meta: + app_label = "sentry" + db_table = "sentry_release_project" + indexes = ( + models.Index(fields=("project", "adopted")), + models.Index(fields=("project", "unadopted")), + models.Index(fields=("project", "first_seen_transaction")), + ) + unique_together = (("project", "release"),) diff --git a/src/sentry/releases/models/releases/set_commits.py b/src/sentry/releases/models/releases/set_commits.py new file mode 100644 index 00000000000000..7ee958fcd9a274 --- /dev/null +++ b/src/sentry/releases/models/releases/set_commits.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import itertools +import logging +import re +from typing import TypedDict + +from django.db import IntegrityError, router + +from sentry.constants import ObjectStatus +from sentry.db.postgres.transactions import in_test_hide_transaction_boundary +from sentry.locks import locks +from sentry.models.activity import Activity +from sentry.models.commitauthor import CommitAuthor +from sentry.models.commitfilechange import CommitFileChange +from sentry.models.grouphistory import GroupHistoryStatus, record_group_history +from sentry.models.groupinbox import GroupInbox, GroupInboxRemoveAction, remove_group_from_inbox +from sentry.models.release import Release +from sentry.models.releases.exceptions import ReleaseCommitError +from sentry.signals import issue_resolved +from sentry.users.services.user import RpcUser +from sentry.utils import metrics +from sentry.utils.db import atomic_transaction +from sentry.utils.retries import TimedRetryPolicy +from sentry.utils.strings import truncatechars + +logger = logging.getLogger(__name__) +from sentry.integrations.tasks.kick_off_status_syncs import kick_off_status_syncs +from sentry.models.commit import Commit +from sentry.models.group import Group, GroupStatus +from sentry.models.grouplink import GroupLink +from sentry.models.groupresolution import GroupResolution +from sentry.models.pullrequest import PullRequest +from sentry.models.releasecommit import ReleaseCommit +from sentry.models.releaseheadcommit import ReleaseHeadCommit +from sentry.models.repository import Repository +from sentry.plugins.providers.repository import RepositoryProvider + + +class _CommitDataKwargs(TypedDict, total=False): + author: CommitAuthor + message: str + date_added: str + + +def set_commits(release, commit_list): + commit_list.sort(key=lambda commit: commit.get("timestamp", 0), reverse=True) + + # todo(meredith): implement for IntegrationRepositoryProvider + commit_list = [ + c for c in commit_list if not RepositoryProvider.should_ignore_commit(c.get("message", "")) + ] + lock_key = Release.get_lock_key(release.organization_id, release.id) + # Acquire the lock for a maximum of 10 minutes + lock = locks.get(lock_key, duration=10 * 60, name="release_set_commits") + if lock.locked(): + # Signal failure to the consumer rapidly. This aims to prevent the number + # of timeouts and prevent web worker exhaustion when customers create + # the same release rapidly for different projects. + raise ReleaseCommitError + + with TimedRetryPolicy(10)(lock.acquire): + create_repositories(commit_list, release) + create_commit_authors(commit_list, release) + + with ( + atomic_transaction(using=router.db_for_write(type(release))), + in_test_hide_transaction_boundary(), + ): + + head_commit_by_repo, commit_author_by_commit = set_commits_on_release( + release, commit_list + ) + + fill_in_missing_release_head_commits(release, head_commit_by_repo) + update_group_resolutions(release, commit_author_by_commit) + + +@metrics.wraps("set_commits_on_release") +def set_commits_on_release(release, commit_list): + # TODO(dcramer): would be good to optimize the logic to avoid these + # deletes but not overly important + ReleaseCommit.objects.filter(release=release).delete() + + commit_author_by_commit = {} + head_commit_by_repo: dict[int, int] = {} + + latest_commit = None + for idx, data in enumerate(commit_list): + commit = set_commit(idx, data, release) + if idx == 0: + latest_commit = commit + + commit_author_by_commit[commit.id] = commit.author + head_commit_by_repo.setdefault(data["repo_model"].id, commit.id) + + release.update( + commit_count=len(commit_list), + authors=[ + str(a_id) + for a_id in ReleaseCommit.objects.filter( + release=release, commit__author_id__isnull=False + ) + .values_list("commit__author_id", flat=True) + .distinct() + ], + last_commit_id=latest_commit.id if latest_commit else None, + ) + return head_commit_by_repo, commit_author_by_commit + + +def set_commit(idx, data, release): + repo = data["repo_model"] + author = data["author_model"] + + commit_data: _CommitDataKwargs = {} + + # Update/set message and author if they are provided. + if author is not None: + commit_data["author"] = author + if "message" in data: + commit_data["message"] = data["message"] + if "timestamp" in data: + commit_data["date_added"] = data["timestamp"] + + commit, created = Commit.objects.get_or_create( + organization_id=release.organization_id, + repository_id=repo.id, + key=data["id"], + defaults=commit_data, + ) + if not created and any(getattr(commit, key) != value for key, value in commit_data.items()): + commit.update(**commit_data) + + if author is None: + author = commit.author + + # Guard against patch_set being None + patch_set = data.get("patch_set") or [] + if patch_set: + CommitFileChange.objects.bulk_create( + [ + CommitFileChange( + organization_id=release.organization.id, + commit=commit, + filename=patched_file["path"], + type=patched_file["type"], + ) + for patched_file in patch_set + ], + ignore_conflicts=True, + batch_size=100, + ) + + try: + with atomic_transaction(using=router.db_for_write(ReleaseCommit)): + ReleaseCommit.objects.create( + organization_id=release.organization_id, + release=release, + commit=commit, + order=idx, + ) + except IntegrityError: + pass + + return commit + + +def fill_in_missing_release_head_commits(release, head_commit_by_repo): + # fill any missing ReleaseHeadCommit entries + for repo_id, commit_id in head_commit_by_repo.items(): + try: + with atomic_transaction(using=router.db_for_write(ReleaseHeadCommit)): + ReleaseHeadCommit.objects.create( + organization_id=release.organization_id, + release_id=release.id, + repository_id=repo_id, + commit_id=commit_id, + ) + except IntegrityError: + pass + + +def update_group_resolutions(release, commit_author_by_commit): + release_commits = list( + ReleaseCommit.objects.filter(release=release) + .select_related("commit") + .values("commit_id", "commit__key") + ) + + commit_resolutions = list( + GroupLink.objects.filter( + linked_type=GroupLink.LinkedType.commit, + linked_id__in=[rc["commit_id"] for rc in release_commits], + ).values_list("group_id", "linked_id") + ) + + commit_group_authors = [ + (cr[0], commit_author_by_commit.get(cr[1])) for cr in commit_resolutions # group_id + ] + + pr_ids_by_merge_commit = list( + PullRequest.objects.filter( + merge_commit_sha__in=[rc["commit__key"] for rc in release_commits], + organization_id=release.organization_id, + ).values_list("id", flat=True) + ) + + pull_request_resolutions = list( + GroupLink.objects.filter( + relationship=GroupLink.Relationship.resolves, + linked_type=GroupLink.LinkedType.pull_request, + linked_id__in=pr_ids_by_merge_commit, + ).values_list("group_id", "linked_id") + ) + + pr_authors = list( + PullRequest.objects.filter( + id__in=[prr[1] for prr in pull_request_resolutions] + ).select_related("author") + ) + + pr_authors_dict = {pra.id: pra.author for pra in pr_authors} + + pull_request_group_authors = [ + (prr[0], pr_authors_dict.get(prr[1])) for prr in pull_request_resolutions + ] + + user_by_author: dict[CommitAuthor | None, RpcUser | None] = {None: None} + + commits_and_prs = list(itertools.chain(commit_group_authors, pull_request_group_authors)) + + group_project_lookup = dict( + Group.objects.filter(id__in=[group_id for group_id, _ in commits_and_prs]).values_list( + "id", "project_id" + ) + ) + + for group_id, author in commits_and_prs: + if author is not None and author not in user_by_author: + try: + user_by_author[author] = author.find_users()[0] + except IndexError: + user_by_author[author] = None + actor = user_by_author[author] + + with atomic_transaction( + using=( + router.db_for_write(GroupResolution), + router.db_for_write(Group), + # inside the remove_group_from_inbox + router.db_for_write(GroupInbox), + router.db_for_write(Activity), + ) + ): + GroupResolution.objects.create_or_update( + group_id=group_id, + values={ + "release": release, + "type": GroupResolution.Type.in_release, + "status": GroupResolution.Status.resolved, + "actor_id": actor.id if actor is not None else None, + }, + ) + group = Group.objects.get(id=group_id) + group.update(status=GroupStatus.RESOLVED, substatus=None) + remove_group_from_inbox(group, action=GroupInboxRemoveAction.RESOLVED, user=actor) + record_group_history(group, GroupHistoryStatus.RESOLVED, actor=actor) + + metrics.incr("group.resolved", instance="in_commit", skip_internal=True) + + issue_resolved.send_robust( + organization_id=release.organization_id, + user=actor, + group=group, + project=group.project, + resolution_type="with_commit", + sender=type(release), + ) + + kick_off_status_syncs.apply_async( + kwargs={"project_id": group_project_lookup[group_id], "group_id": group_id} + ) + + +def create_commit_authors(commit_list, release): + authors = {} + + for data in commit_list: + author_email = data.get("author_email") + if author_email is None and data.get("author_name"): + author_email = ( + re.sub(r"[^a-zA-Z0-9\-_\.]*", "", data["author_name"]).lower() + "@localhost" + ) + + author_email = truncatechars(author_email, 75) + + if not author_email: + author = None + elif author_email not in authors: + author_data = {"name": data.get("author_name")} + author, created = CommitAuthor.objects.get_or_create( + organization_id=release.organization_id, + email=author_email, + defaults=author_data, + ) + if author.name != author_data["name"]: + author.update(name=author_data["name"]) + authors[author_email] = author + else: + author = authors[author_email] + + data["author_model"] = author + + +def create_repositories(commit_list, release): + repos = {} + for data in commit_list: + repo_name = data.get("repository") or f"organization-{release.organization_id}" + if repo_name not in repos: + repo = ( + Repository.objects.filter( + organization_id=release.organization_id, + name=repo_name, + status=ObjectStatus.ACTIVE, + ) + .order_by("-pk") + .first() + ) + + if repo is None: + repo = Repository.objects.create( + organization_id=release.organization_id, + name=repo_name, + ) + + repos[repo_name] = repo + else: + repo = repos[repo_name] + + data["repo_model"] = repo diff --git a/src/sentry/releases/models/releases/util.py b/src/sentry/releases/models/releases/util.py new file mode 100644 index 00000000000000..66fa5b2829f5fe --- /dev/null +++ b/src/sentry/releases/models/releases/util.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import logging +from collections import namedtuple +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Self + +from django.db import models +from django.db.models import Case, F, Func, Q, Subquery, Value, When +from django.db.models.signals import pre_save +from sentry_relay.exceptions import RelayError +from sentry_relay.processing import parse_release + +from sentry.db.models.manager.base_query_set import BaseQuerySet +from sentry.exceptions import InvalidSearchQuery +from sentry.models.releases.release_project import ReleaseProject +from sentry.utils.numbers import validate_bigint + +if TYPE_CHECKING: + from sentry.models.release import Release # noqa: F401 + +logger = logging.getLogger(__name__) + + +class SemverVersion( + namedtuple("SemverVersion", "major minor patch revision prerelease_case prerelease") +): + pass + + +@dataclass +class SemverFilter: + operator: str + version_parts: Sequence[int | str] + package: str | Sequence[str] | None = None + negated: bool = False + + +class ReleaseQuerySet(BaseQuerySet["Release"]): + def annotate_prerelease_column(self): + """ + Adds a `prerelease_case` column to the queryset which is used to properly sort + by prerelease. We treat an empty (but not null) prerelease as higher than any + other value. + """ + return self.annotate( + prerelease_case=Case( + When(prerelease="", then=1), default=0, output_field=models.IntegerField() + ) + ) + + def filter_to_semver(self) -> Self: + """ + Filters the queryset to only include semver compatible rows + """ + return self.filter(major__isnull=False) + + def filter_by_semver_build( + self, + organization_id: int, + operator: str, + build: str, + project_ids: Sequence[int] | None = None, + negated: bool = False, + ) -> Self: + """ + Filters released by build. If the passed `build` is a numeric string, we'll filter on + `build_number` and make use of the passed operator. + If it is a non-numeric string, then we'll filter on `build_code` instead. We support a + wildcard only at the end of this string, so that we can filter efficiently via the index. + """ + qs = self.filter(organization_id=organization_id) + query_func = "exclude" if negated else "filter" + + if project_ids: + qs = qs.filter( + id__in=ReleaseProject.objects.filter(project_id__in=project_ids).values_list( + "release_id", flat=True + ) + ) + + if build.isdecimal() and validate_bigint(int(build)): + qs = getattr(qs, query_func)(**{f"build_number__{operator}": int(build)}) + else: + if not build or build.endswith("*"): + qs = getattr(qs, query_func)(build_code__startswith=build[:-1]) + else: + qs = getattr(qs, query_func)(build_code=build) + + return qs + + def filter_by_semver( + self, + organization_id: int, + semver_filter: SemverFilter, + project_ids: Sequence[int] | None = None, + ) -> Self: + """ + Filters releases based on a based `SemverFilter` instance. + `SemverFilter.version_parts` can contain up to 6 components, which should map + to the columns defined in `Release.SEMVER_COLS`. If fewer components are + included, then we will exclude later columns from the filter. + `SemverFilter.package` is optional, and if included we will filter the `package` + column using the provided value. + `SemverFilter.operator` should be a Django field filter. + + Typically we build a `SemverFilter` via `sentry.search.events.filter.parse_semver` + """ + qs = self.filter(organization_id=organization_id).annotate_prerelease_column() + query_func = "exclude" if semver_filter.negated else "filter" + + if semver_filter.package: + if isinstance(semver_filter.package, str): + qs = getattr(qs, query_func)(package=semver_filter.package) + else: + qs = getattr(qs, query_func)(package__in=semver_filter.package) + if project_ids: + qs = qs.filter( + id__in=ReleaseProject.objects.filter(project_id__in=project_ids).values_list( + "release_id", flat=True + ) + ) + + if semver_filter.version_parts: + filter_func = Func( + *( + Value(part) if isinstance(part, str) else part + for part in semver_filter.version_parts + ), + function="ROW", + ) + cols = self.model.SEMVER_COLS[: len(semver_filter.version_parts)] + qs = qs.annotate( + semver=Func( + *(F(col) for col in cols), function="ROW", output_field=models.JSONField() + ) + ) + qs = getattr(qs, query_func)(**{f"semver__{semver_filter.operator}": filter_func}) + return qs + + def filter_by_stage( + self, + organization_id: int, + operator: str, + value, + project_ids: Sequence[int] | None = None, + environments: list[str] | None = None, + ) -> Self: + from sentry.models.releaseprojectenvironment import ReleaseProjectEnvironment, ReleaseStages + from sentry.search.events.filter import to_list + + if not environments or len(environments) != 1: + raise InvalidSearchQuery("Choose a single environment to filter by release stage.") + + filters = { + ReleaseStages.ADOPTED: Q(adopted__isnull=False, unadopted__isnull=True), + ReleaseStages.REPLACED: Q(adopted__isnull=False, unadopted__isnull=False), + ReleaseStages.LOW_ADOPTION: Q(adopted__isnull=True, unadopted__isnull=True), + } + value = to_list(value) + operator_conversions = {"=": "IN", "!=": "NOT IN"} + operator = operator_conversions.get(operator, operator) + + for stage in value: + if stage not in filters: + raise InvalidSearchQuery("Unsupported release.stage value.") + + rpes = ReleaseProjectEnvironment.objects.filter( + release__organization_id=organization_id, + ).select_related("release") + + if project_ids: + rpes = rpes.filter(project_id__in=project_ids) + + query = Q() + if operator == "IN": + for stage in value: + query |= filters[stage] + elif operator == "NOT IN": + for stage in value: + query &= ~filters[stage] + + qs = self.filter(id__in=Subquery(rpes.filter(query).values_list("release_id", flat=True))) + return qs + + def order_by_recent(self) -> Self: + return self.order_by("-date_added", "-id") + + @staticmethod + def massage_semver_cols_into_release_object_data(kwargs): + """ + Helper function that takes kwargs as an argument and massages into it the release semver + columns (if possible) + Inputs: + * kwargs: data of the release that is about to be created + """ + if "version" in kwargs: + try: + version_info = parse_release(kwargs["version"]) + package = version_info.get("package") + version_parsed = version_info.get("version_parsed") + + if version_parsed is not None and all( + validate_bigint(version_parsed[field]) + for field in ("major", "minor", "patch", "revision") + ): + build_code = version_parsed.get("build_code") + build_number = ReleaseQuerySet._convert_build_code_to_build_number(build_code) + + kwargs.update( + { + "major": version_parsed.get("major"), + "minor": version_parsed.get("minor"), + "patch": version_parsed.get("patch"), + "revision": version_parsed.get("revision"), + "prerelease": version_parsed.get("pre") or "", + "build_code": build_code, + "build_number": build_number, + "package": package, + } + ) + except RelayError: + # This can happen on invalid legacy releases + pass + + @staticmethod + def _convert_build_code_to_build_number(build_code): + """ + Helper function that takes the build_code and checks if that build code can be parsed into + a 64 bit integer + Inputs: + * build_code: str + Returns: + * build_number + """ + build_number = None + if build_code is not None: + try: + build_code_as_int = int(build_code) + if validate_bigint(build_code_as_int): + build_number = build_code_as_int + except ValueError: + pass + return build_number + + +def parse_semver_pre_save(instance, **kwargs): + if instance.id: + return + ReleaseQuerySet.massage_semver_cols_into_release_object_data(instance.__dict__) + + +pre_save.connect( + parse_semver_pre_save, sender="sentry.Release", dispatch_uid="parse_semver_pre_save" +)