Issue 8101 fix 6266 after review

specifyweb/backend/businessrules/migration_utils.py

@@ -1,4 +1,4 @@
-from typing import Tuple, List
+from typing import List
 
 from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule
 
@@ -47,6 +47,7 @@ def catnum_rule_uneditable(apps, schema_editor=None):
         model_rules = UniquenessRule.objects.filter(modelName="Collectionobject", discipline_id=discipline.id, isDatabaseConstraint=False)
 
         has_catalognumber_rule = False
+        matching_rule_ids: List[int] = []
         for rule in model_rules: 
             rule_fields = rule.uniquenessrulefield_set.all()
 
@@ -59,8 +60,11 @@ def catnum_rule_uneditable(apps, schema_editor=None):
             # exception if more than one result is returned
             if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"):
                 has_catalognumber_rule = True
+                matching_rule_ids.append(rule.id)
 
-        if not has_catalognumber_rule:
+        if has_catalognumber_rule:
+            UniquenessRule.objects.filter(id__in=matching_rule_ids).update(isDatabaseConstraint=True)
+        else:
             create_uniqueness_rule(
                 "Collectionobject",
                 discipline=discipline,

specifyweb/backend/businessrules/uniqueness_rules.py

@@ -362,12 +362,19 @@ def create_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f
                                                     isDatabaseConstraint=is_database_constraint,
                                                     discipline=discipline)
 
+    expected_fields = set(fields)
+    expected_scopes = set(scopes)
+
     for rule in candidate_rules:
         all_fields = rule.uniquenessrulefield_set.all()
-        matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False)
-        matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True)
+        existing_fields = set(
+            all_fields.filter(isScope=False).values_list("fieldPath", flat=True)
+        )
+        existing_scopes = set(
+            all_fields.filter(isScope=True).values_list("fieldPath", flat=True)
+        )
         # If the rule already exists, skip creating the rule
-        if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): 
+        if existing_fields == expected_fields and existing_scopes == expected_scopes: 
             return
 
     logger.info(f"Creating uniqueness rule on {model_name} with fields {fields} and scopes {scopes} for the discipline {discipline.name if discipline else 'Global'}")
@@ -393,13 +400,20 @@ def remove_uniqueness_rule(model_name, raw_discipline, is_database_constraint, f
     candidate_rules = UniquenessRule.objects.filter(
         modelName=model_name, isDatabaseConstraint=is_database_constraint, discipline=discipline)
 
+    expected_fields = set(fields)
+    expected_scopes = set(scopes)
+
     rule_ids = []
     for rule in candidate_rules: 
         all_fields = rule.uniquenessrulefield_set.all()
-        matching_fields = all_fields.filter(fieldPath__in=fields, isScope=False)
-        matching_scopes = all_fields.filter(fieldPath__in=scopes, isScope=True)
+        existing_fields = set(
+            all_fields.filter(isScope=False).values_list("fieldPath", flat=True)
+        )
+        existing_scopes = set(
+            all_fields.filter(isScope=True).values_list("fieldPath", flat=True)
+        )
         # If the rule exists, add it to the list of rules to be deleted
-        if len(matching_fields) == len(fields) and len(matching_scopes) == len(scopes): 
+        if existing_fields == expected_fields and existing_scopes == expected_scopes: 
             rule_ids.append(rule.id)
 
     UniquenessRuleField.objects.filter(
@@ -421,24 +435,18 @@ def fix_global_default_rules(registry=None):
     UniquenessRule = registry.get_model('businessrules', 'UniquenessRule') \
         if registry \
         else models.UniquenessRule
-    UniquenessRuleField = registry.get_model('businessrules', 'UniquenessRuleField') \
-        if registry \
-        else models.UniquenessRuleField
-
-    global_rule_fields = UniquenessRuleField.objects.filter(
-        uniquenessrule__discipline__isnull=True
-    ).values(
-        "uniquenessrule__modelName",
-        "uniquenessrule__isDatabaseConstraint",
-        "fieldPath",
-        "isScope",
-    )
-
-    global_rule_exists = UniquenessRule.objects.filter(
-        discipline__isnull=True,
-        modelName=OuterRef("modelName"),
-        isDatabaseConstraint=OuterRef("isDatabaseConstraint"),
-    )
+    global_rule_signatures = {
+        (
+            rule.modelName,
+            rule.isDatabaseConstraint,
+            frozenset(
+                rule.uniquenessrulefield_set.values_list("fieldPath", "isScope")
+            ),
+        )
+        for rule in UniquenessRule.objects.filter(
+            discipline__isnull=True
+        ).prefetch_related("uniquenessrulefield_set")
+    }
 
     discipline_ids = (
         UniquenessRule.objects.exclude(discipline__isnull=True)
@@ -448,28 +456,16 @@ def fix_global_default_rules(registry=None):
 
     for discipline_id in discipline_ids:
         with transaction.atomic():
-            # Delete matching fields for this discipline
-            matching_fields_qs = UniquenessRuleField.objects.filter(
-                uniquenessrule__discipline_id=discipline_id
-            ).filter(
-                Exists(
-                    global_rule_fields.filter(
-                        **{
-                            "uniquenessrule__modelName": OuterRef("uniquenessrule__modelName"),
-                            "uniquenessrule__isDatabaseConstraint": OuterRef("uniquenessrule__isDatabaseConstraint"),
-                            "fieldPath": OuterRef("fieldPath"),
-                            "isScope": OuterRef("isScope"),
-                        }
-                    )
+            for rule in UniquenessRule.objects.filter(
+                discipline_id=discipline_id
+            ).prefetch_related("uniquenessrulefield_set"):
+                signature = (
+                    rule.modelName,
+                    rule.isDatabaseConstraint,
+                    frozenset(
+                        rule.uniquenessrulefield_set.values_list("fieldPath", "isScope")
+                    ),
                 )
-            )
-            matching_fields_qs.delete()
-
-            # Delete UniquenessRule rows for this discipline that are now empty
-            empty_rules_qs = (
-                UniquenessRule.objects.filter(discipline_id=discipline_id)
-                .annotate(field_count=Count("uniquenessrulefield"))
-                .filter(field_count=0)  # now empty after field deletions
-                .filter(Exists(global_rule_exists))
-            )
-            empty_rules_qs.delete()
+                if signature in global_rule_signatures:
+                        rule.uniquenessrulefield_set.all().delete()
+                        rule.delete()

specifyweb/backend/patches/migration_utils.py

@@ -10,27 +10,29 @@ def apply_migrations(app_registry, schema_editor=None):
     update_coordinates(app_registry, schema_editor)
 
 def update_is_accepted(app_registry, schema_editor=None):
+    db_alias = schema_editor.connection.alias if schema_editor is not None else "default"
     for tree in SPECIFY_TREES:
         tree_filters = {
             "isaccepted": False,
             "accepted" + tree.lower() + "__isnull": True
         }
 
         tree_model = app_registry.get_model("specify", tree)
-        tree_model.objects.filter(**tree_filters).update(isaccepted=True)
+        tree_model._base_manager.using(db_alias).filter(**tree_filters).update(isaccepted=True)
 
 
 def update_coordinates(app_registry, schema_editor=None):
+    db_alias = schema_editor.connection.alias if schema_editor is not None else "default"
     Locality = app_registry.get_model("specify", "Locality")
 
-    Locality.objects.filter(lat1text__isnull=True, latitude1__isnull=False) \
+    Locality._base_manager.using(db_alias).filter(lat1text__isnull=True, latitude1__isnull=False) \
         .update(lat1text=F("latitude1"))
 
-    Locality.objects.filter(long1text__isnull=True, longitude1__isnull=False) \
+    Locality._base_manager.using(db_alias).filter(long1text__isnull=True, longitude1__isnull=False) \
         .update(long1text=F("longitude1"))
 
-    Locality.objects.filter(lat2text__isnull=True, latitude2__isnull=False) \
+    Locality._base_manager.using(db_alias).filter(lat2text__isnull=True, latitude2__isnull=False) \
         .update(lat2text=F("latitude2"))
 
-    Locality.objects.filter(long2text__isnull=True, longitude2__isnull=False) \
+    Locality._base_manager.using(db_alias).filter(long2text__isnull=True, longitude2__isnull=False) \
         .update(long2text=F("longitude2"))

specifyweb/backend/permissions/initialize.py

@@ -53,13 +53,14 @@ def create_admins(apps=apps) -> None:
     UserPolicy = apps.get_model('permissions', 'UserPolicy')
     Specifyuser = apps.get_model('specify', 'Specifyuser')
 
-    if UserPolicy.objects.filter(collection__isnull=True, resource='%', action='%').exists():
-        # don't do anything if there is already any admin.
-        return
-
     users = Specifyuser.objects.all()
     for user in users:
-        if is_sp6_user_permissions_migrated(user, apps):
+        if UserPolicy.objects.filter(
+            collection__isnull=True,
+            specifyuser_id=user.id,
+            resource="%",
+            action="%",
+        ).exists():
             continue
         if is_legacy_admin(user):
             UserPolicy.objects.get_or_create(
@@ -112,17 +113,12 @@ def assign_users_to_roles(apps=apps) -> None:
             JOIN spprincipal p ON p.SpPrincipalID = up.SpPrincipalID
             JOIN collection c ON c.UserGroupScopeId = p.userGroupScopeID
             WHERE p.groupType IS NULL
-            AND u.SpecifyUserID NOT IN (
-                SELECT ur.specifyuser_id
+            AND NOT EXISTS (
+                SELECT 1
                 FROM spuserrole ur 
                 JOIN sprole r ON r.id = ur.role_id 
-                WHERE r.collection_id = p.usergroupscopeid
-            )
-            AND c.UserGroupScopeId NOT IN (
-                SELECT DISTINCT r.collection_id
-                FROM spuserrole ur 
-                JOIN sprole r ON r.id = ur.role_id
-                JOIN collection c ON c.UserGroupScopeId = r.collection_id
+                WHERE r.collection_id = c.UserGroupScopeId
+                AND ur.specifyuser_id = u.SpecifyUserID
             );
         """)
 

specifyweb/backend/stored_queries/execution.py

@@ -878,7 +878,8 @@ def execute(
 
 
 
-        log_sqlalchemy_query(query) # Debugging
+        if settings.DEBUG:
+            log_sqlalchemy_query(query)
         return {"results": apply_special_post_query_processing(query, tableid, field_specs, collection, user)}
 
 def build_query(
@@ -1097,7 +1098,8 @@ def series_post_query(query, limit=40, offset=0, sort_type=0, co_id_cat_num_pair
     and adding a co_id colum and formatted catnum range column.
     Sort the results by the first catnum in the range."""
 
-    log_sqlalchemy_query(query)  # Debugging
+    if settings.DEBUG:
+            log_sqlalchemy_query(query)
 
     def parse_catalog_for_comparing(s):
         def check_for_decimal(s):

specifyweb/backend/stored_queries/tests/tests_legacy.py

@@ -841,9 +841,6 @@ def test_sqlalchemy_model_errors(self):
     ]
   },
   "CollectionObject": {
-    "not_found": [
-      "projects"
-    ],
     "incorrect_direction": {
       "cojo": [
         "onetomany",

specifyweb/specify/api/utils.py

@@ -1,5 +1,6 @@
 import logging
 
+from django.conf import settings
 from specifyweb.specify import models as spmodels
 from specifyweb.backend.businessrules.exceptions import BusinessRuleException
 
@@ -19,13 +20,16 @@ def get_spmodel_class(model_name: str):
     raise AttributeError(f"Model '{model_name}' not found in models module.")
 
 def log_sqlalchemy_query(query):
-    # Call this function to debug the raw SQL query generated by SQLAlchemy
+    if not settings.DEBUG:
+        return
+
     from sqlalchemy.dialects import mysql
-    compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True})
+    compiled_query = query.statement.compile(dialect=mysql.dialect())
     raw_sql = str(compiled_query).replace('\n', ' ') + ';'
-    logger.debug('='.join(['' for _ in range(80)]))
-    logger.debug(raw_sql)
-    logger.debug('='.join(['' for _ in range(80)]))
+    logger.debug("%s", "=" * 80)
+    logger.debug("SQL: %s", raw_sql)
+    logger.debug("Params: %s", compiled_query.params)
+    logger.debug("%s", "=" * 80)
     # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546
     # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query)
 
@@ -90,4 +94,4 @@ def get_picklists(collection: spmodels.Collection, tablename: str, fieldname: st
         if len(collection_picklists) > 0:
             picklists = collection_picklists
 
-    return picklists, schemaitem
+    return picklists, schemaitem

specifyweb/specify/management/commands/run_key_migration_functions.py

@@ -214,7 +214,7 @@ def add_arguments(self, parser):
             nargs="*",
             type=str,
             choices=tuple(self.funcs.keys()),
-            help=f"Optional: specify one or more functions to run",
+            help="Optional: specify one or more functions to run",
         )
         parser.add_argument(
             "--verbose",
@@ -247,6 +247,6 @@ def handle(self, *args, **options):
                         self.stdout.write(self.style.SUCCESS(f"Applying {func_name}..."))
                         func(self.stdout.write if verbose else None)
                         self.stdout.write(self.style.SUCCESS(f"Applied {func_name}"))
-        except Exception as e:
-            logger.error(f"An error occurred: {e}")
+        except Exception:
+            logger.exception("An error occurred while running key migrations")
             raise

specifyweb/specify/migration_utils/default_cots.py

@@ -136,22 +136,25 @@ def fix_tectonic_unit_treedef_discipline_links(apps):
     Discipline = apps.get_model('specify', 'Discipline')
     Tectonicunittreedef = apps.get_model('specify', 'Tectonicunittreedef')
 
-    empty_tectonic_unit_treedefs = Tectonicunittreedef.objects.filter(discipline__isnull=True)
-    empty_disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True)
-    for empty_discipline in empty_disciplines:
-        if not empty_tectonic_unit_treedefs.exists():
-            new_tectonic_unit_treedef = Tectonicunittreedef.objects.create(
-                name=f'{empty_discipline.name} Tectonic Unit Tree',
-                discipline=empty_discipline
-            )
-        else:
-            empty_discipline.tectonicunittreedef = empty_tectonic_unit_treedefs.first()
-            empty_discipline.save()
-
-    for empty_tectonic_unit_treedef in empty_tectonic_unit_treedefs:
-        if empty_disciplines.exists():
-            empty_tectonic_unit_treedef.discipline = empty_disciplines.first()
-            empty_tectonic_unit_treedef.save()
-        else:
-            empty_tectonic_unit_treedef.discipline = empty_disciplines.last()
-            empty_tectonic_unit_treedef.save()
+    empty_tectonic_unit_treedefs = list(
+        Tectonicunittreedef.objects.filter(discipline__isnull=True).order_by('id')
+    )
+    empty_disciplines = list(
+        Discipline.objects.filter(tectonicunittreedef__isnull=True).order_by('id')
+    )
+
+    for discipline, tectonic_unit_treedef in zip(
+        empty_disciplines, empty_tectonic_unit_treedefs
+    ):
+        tectonic_unit_treedef.discipline = discipline
+        tectonic_unit_treedef.save()
+        discipline.tectonicunittreedef = tectonic_unit_treedef
+        discipline.save()
+
+    for discipline in empty_disciplines[len(empty_tectonic_unit_treedefs):]:
+        tectonic_unit_treedef = Tectonicunittreedef.objects.create(
+            name=f'{discipline.name} Tectonic Unit Tree',
+            discipline=discipline
+        )
+        discipline.tectonicunittreedef = tectonic_unit_treedef
+        discipline.save()

specifyweb/specify/migration_utils/sp7_schemaconfig.py

@@ -148,7 +148,7 @@
     'RelativeAge': ['number2', 'yesno2', 'relativeAgeId', 'relativeAgePeriod', 'text1', 'agent1', 'collectionDate', 'text2', 'agent2', 'date1', 'date2', 'collectionObject', 'relativeAgeCitations', 'number1', 'yesno1'],
     'CollectionObject': ['collectionObjectType', 'relativeAges', 'absoluteAges', 'cojo'],
     'AbsoluteAgeCitation': ['collectionMember', 'absoluteAgeCitationId'],
-    'RelativeAgeCitation': ['absoluteAgeCitationId', 'collectionMember'],
+    'RelativeAgeCitation': ['relativeAgeCitationId', 'collectionMember'],
     'TectonicUnit': ['collectionMember', 'nodeNumber', 'yesno1', 'tectonicUnitId', 'number1', 'yesno2', 'number2', 'rankId', 'text1'],
     'TectonicUnitTreeDefItem': ['children', 'rankId', 'parent', 'treeDef', 'treeEntries', 'tectonicUnitTreeDefItemId'],
     'TectonicUnitTreeDef': ['discipline', 'treeEntries', 'tectonicUnitTreeDefId']