Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 1 addition & 15 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,6 @@ jobs:
- core
- filestorage
- __flaky__
storage:
- default
- legacy_storage
continue-on-error: true
steps:
- name: Checkout repo
Expand All @@ -68,17 +65,6 @@ jobs:

cat <<EOF >> .env
STORAGES='{
"legacy_storage": {
"BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage",
"OPTIONS": {
"access_key": "minioadmin",
"secret_key": "minioadmin",
"bucket_name": "qfieldcloud-local-legacy",
"region_name": "",
"endpoint_url": "http://172.17.0.1:8009"
},
"QFC_IS_LEGACY": true
},
"webdav": {
"BACKEND": "qfieldcloud.filestorage.backend.QfcWebDavStorage",
"OPTIONS": {
Expand All @@ -102,7 +88,7 @@ jobs:
}'
EOF

echo "STORAGES_PROJECT_DEFAULT_STORAGE=${{ matrix.storage }}" >> .env
echo "STORAGES_PROJECT_DEFAULT_STORAGE=default" >> .env

- name: Pull docker containers
run: docker compose pull
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# Generated by Django 3.2.25 on 2024-06-18 01:02

import django.core.validators
from django.conf import settings
from django.db import migrations, models

import qfieldcloud.core.models
Expand All @@ -10,7 +9,7 @@


def get_file_storage_name():
return settings.LEGACY_STORAGE_NAME or "default"
return "default"


class Migration(migrations.Migration):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Generated by Django 4.2.28 on 2026-02-11 07:31

from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("core", "0094_alter_project_are_attachments_versioned_and_more"),
]

operations = [
migrations.RemoveField(
model_name="project",
name="legacy_thumbnail_uri",
),
migrations.RemoveField(
model_name="useraccount",
name="legacy_avatar_uri",
),
]
111 changes: 12 additions & 99 deletions docker-app/qfieldcloud/core/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from typing import TYPE_CHECKING, Any, cast
from uuid import uuid4

from deprecated import deprecated
from django.conf import settings
from django.contrib.auth.models import AbstractUser
from django.contrib.auth.models import UserManager as DjangoUserManager
Expand Down Expand Up @@ -39,9 +38,8 @@
)
from timezone_field import TimeZoneField

from qfieldcloud.core import utils, validators
from qfieldcloud.core import validators
from qfieldcloud.core.fields import DynamicStorageFileField, QfcImageField, QfcImageFile
from qfieldcloud.core.utils2 import storage
from qfieldcloud.subscription.exceptions import ReachedMaxOrganizationMembersError

if TYPE_CHECKING:
Expand Down Expand Up @@ -467,13 +465,6 @@ class UserAccount(models.Model):
twitter = models.CharField(max_length=255, default="", blank=True)
is_email_public = models.BooleanField(default=False)

# TODO Delete with QF-4963 Drop support for legacy storage
legacy_avatar_uri = models.CharField(
_("Legacy Profile Picture URI"),
max_length=255,
blank=True,
)

avatar = QfcImageField(
_("Avatar Picture"),
upload_to=get_user_account_avatar_upload_to,
Expand Down Expand Up @@ -518,23 +509,12 @@ def storage_used_bytes(self) -> float:
project_files_used_quota = (
FileVersion.objects.filter(
file__file_type=File.FileType.PROJECT_FILE,
file__project__in=self.user.projects.exclude(
file_storage=settings.LEGACY_STORAGE_NAME
),
file__project__in=self.user.projects,
).aggregate(sum_bytes=Sum("size"))["sum_bytes"]
or 0
)

# TODO: Delete with QF-4963 Drop support for legacy storage
legacy_used_quota = (
self.user.projects.filter(
file_storage=settings.LEGACY_STORAGE_NAME
).aggregate(sum_bytes=Sum("file_storage_bytes"))["sum_bytes"]
# if there are no projects, the value will be `None`
or 0
)

used_quota = project_files_used_quota + legacy_used_quota
used_quota = project_files_used_quota

return used_quota

Expand Down Expand Up @@ -1211,11 +1191,6 @@ class Meta:
),
)

# TODO: Delete with QF-4963 Drop support for legacy storage
legacy_thumbnail_uri = models.CharField(
_("Legacy Thumbnail Picture URI"), max_length=255, blank=True
)

thumbnail = DynamicStorageFileField(
_("Thumbnail Picture"),
upload_to=get_project_thumbnail_upload_to,
Expand Down Expand Up @@ -1491,17 +1466,9 @@ def owner_aware_storage_keep_versions(self) -> int:

@property
def thumbnail_url(self) -> StrOrPromise:
"""Returns the url to the project's thumbnail or empty string if no URL provided.

Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
if self.uses_legacy_storage:
if not self.legacy_thumbnail_uri:
return ""
else:
if not self.thumbnail:
return ""
"""Returns the url to the project's thumbnail or empty string if no URL provided."""
if not self.thumbnail:
return ""

return reverse_lazy(
"filestorage_project_thumbnails",
Expand Down Expand Up @@ -1575,45 +1542,14 @@ def private(self) -> bool:
# still used in the project serializer
return not self.is_public

@property
def uses_legacy_storage(self) -> bool:
"""Whether the storage of the project is legacy.

Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
return self.file_storage == settings.LEGACY_STORAGE_NAME

@cached_property
@deprecated
def legacy_files(self) -> list[utils.S3ObjectWithVersions]:
"""Gets all the files from S3 storage. This is potentially slow. Results are cached on the instance.

Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
if self.uses_legacy_storage:
return list(utils.get_project_files_with_versions(str(self.id)))
else:
raise NotImplementedError(
"The `Project.legacy_files` method is not implemented for projects stored in non-legacy storage"
)

@property
def project_files(self) -> "FileQueryset":
"""Returns the files of type PROJECT related to the project."""
return self.all_files.with_type_project()

@property
def project_files_count(self) -> int:
"""
Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
if self.uses_legacy_storage:
return len(self.legacy_files)
else:
return self.project_files.count()
return self.project_files.count()

@property
def users(self):
Expand Down Expand Up @@ -1852,14 +1788,7 @@ def direct_collaborators(self) -> ProjectCollaboratorQueryset:
)

def delete(self, *args, **kwargs):
"""Deletes the project and the thumbnail for the legacy storage.

Todo:
* Delete with QF-4963 Drop support for legacy storage
"""
if self.uses_legacy_storage:
storage.delete_project_thumbnail(self)

"""Deletes the project and the thumbnail for the legacy storage."""
return super().delete(*args, **kwargs)

@property
Expand All @@ -1878,15 +1807,9 @@ def save(self, recompute_storage=False, *args, **kwargs):
additional_update_fields = set()

if recompute_storage:
# TODO Delete with QF-4963 Drop support for legacy storage
if self.uses_legacy_storage:
self.file_storage_bytes = storage.get_project_file_storage_in_bytes(
self
)
else:
self.file_storage_bytes = self.project_files.aggregate(
file_storage_bytes=Sum("versions__size", default=0)
)["file_storage_bytes"]
self.file_storage_bytes = self.project_files.aggregate(
file_storage_bytes=Sum("versions__size", default=0)
)["file_storage_bytes"]

additional_update_fields.add("file_storage_bytes")

Expand All @@ -1912,11 +1835,6 @@ def save(self, recompute_storage=False, *args, **kwargs):
def get_file(self, filename: str) -> File:
return self.project_files.get_by_name(filename) # type: ignore

def legacy_get_file(self, filename: str) -> utils.S3ObjectWithVersions:
files = filter(lambda f: f.latest.name == filename, self.legacy_files)

return next(files)


class ProjectCollaboratorQueryset(models.QuerySet):
def validated(self, skip_invalid=False):
Expand Down Expand Up @@ -2710,10 +2628,5 @@ class Meta:
)

def _get_file_storage_name(self) -> str:
# TODO Delete with QF-4963 Drop support for legacy storage
# Legacy storage - use default storage
if self.project.uses_legacy_storage:
return "default"

# Non-legacy storage - use same storage as project
# Use same storage as project
return self.project.file_storage
24 changes: 8 additions & 16 deletions docker-app/qfieldcloud/core/tests/test_commands.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import csv
import io
import os

from django.core.files.base import ContentFile
Expand All @@ -8,7 +7,6 @@

from qfieldcloud.core.models import Person, Project
from qfieldcloud.core.tests.utils import set_subscription, setup_subscription_plans
from qfieldcloud.core.utils2 import storage
from qfieldcloud.filestorage.models import File, FileVersion


Expand All @@ -23,20 +21,14 @@ def setUpTestData(cls):
# Project
cls.p1 = Project.objects.create(name="test_project", owner=user)

# TODO Delete with QF-4963 Drop support for legacy storage
if cls.p1.uses_legacy_storage:
storage.upload_project_file(
cls.p1, io.BytesIO(b"Hello world!"), "project.qgs"
)
else:
FileVersion.objects.add_version(
project=cls.p1,
filename="file.name",
# NOTE the dummy name is required when running tests on GitHub CI, but not locally. Spent few hours before I isolated this...
content=ContentFile(b"Hello world!", "dummy.name"),
file_type=File.FileType.PROJECT_FILE,
uploaded_by=user,
)
FileVersion.objects.add_version(
project=cls.p1,
filename="file.name",
# NOTE the dummy name is required when running tests on GitHub CI, but not locally. Spent few hours before I isolated this...
content=ContentFile(b"Hello world!", "dummy.name"),
file_type=File.FileType.PROJECT_FILE,
uploaded_by=user,
)

def test_extracts3data_output_to_file(self):
output_file = "extracted.csv"
Expand Down
Loading
Loading