Synthetic SearchIndex classes for custom objects (#137)

bctiemann · web-flow · commit 2b9f2b7be878 · 2025-08-15T13:28:09.000-07:00
diff --git a/netbox_custom_objects/__init__.py b/netbox_custom_objects/__init__.py
@@ -18,6 +18,36 @@ def is_running_migration():
     return False
 
 
+def is_in_clear_cache():
+    """
+    Check if the code is currently being called from Django's clear_cache() method.
+
+    TODO: This is fairly ugly, but in models.CustomObjectType.get_model() we call
+    meta = type() which calls clear_cache on the model which causes a call to
+    get_models() which in-turn calls get_model and therefore recurses.
+
+    This catches the specific case of a recursive call to get_models() from
+    clear_cache() which is the only case we care about, so should be relatively
+    safe.  An alternative should be found for this.
+    """
+    import inspect
+
+    frame = inspect.currentframe()
+    try:
+        # Walk up the call stack to see if we're being called from clear_cache
+        while frame:
+            if (
+                frame.f_code.co_name == "clear_cache"
+                and "django/apps/registry.py" in frame.f_code.co_filename
+            ):
+                return True
+            frame = frame.f_back
+        return False
+    finally:
+        # Clean up the frame reference
+        del frame
+
+
 def check_custom_object_type_table_exists():
     """
     Check if the CustomObjectType table exists in the database.
@@ -49,6 +79,7 @@ class CustomObjectsPluginConfig(PluginConfig):
     default_settings = {}
     required_settings = []
     template_extensions = "template_content.template_extensions"
+    _in_get_models = False  # Recursion guard
 
     def get_model(self, model_name, require_ready=True):
         try:
@@ -84,33 +115,45 @@ def get_model(self, model_name, require_ready=True):
 
     def get_models(self, include_auto_created=False, include_swapped=False):
         """Return all models for this plugin, including custom object type models."""
+
         # Get the regular Django models first
         for model in super().get_models(include_auto_created, include_swapped):
             yield model
 
-        # Suppress warnings about database calls during model loading
-        with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore", category=RuntimeWarning, message=".*database.*"
-            )
-            warnings.filterwarnings(
-                "ignore", category=UserWarning, message=".*database.*"
-            )
+        # Prevent recursion
+        if self._in_get_models and is_in_clear_cache():
+            # Skip dynamic model creation if we're in a recursive get_models call
+            return
 
-            # Skip custom object type model loading if running during migration
-            if is_running_migration() or not check_custom_object_type_table_exists():
-                return
-
-            # Add custom object type models
-            from .models import CustomObjectType
-
-            custom_object_types = CustomObjectType.objects.all()
-            for custom_type in custom_object_types:
-                # Only yield already cached models during discovery
-                if CustomObjectType.is_model_cached(custom_type.id):
-                    model = CustomObjectType.get_cached_model(custom_type.id)
+        self._in_get_models = True
+        try:
+            # Suppress warnings about database calls during model loading
+            with warnings.catch_warnings():
+                warnings.filterwarnings(
+                    "ignore", category=RuntimeWarning, message=".*database.*"
+                )
+                warnings.filterwarnings(
+                    "ignore", category=UserWarning, message=".*database.*"
+                )
+
+                # Skip custom object type model loading if running during migration
+                if (
+                    is_running_migration()
+                    or not check_custom_object_type_table_exists()
+                ):
+                    return
+
+                # Add custom object type models
+                from .models import CustomObjectType
+
+                custom_object_types = CustomObjectType.objects.all()
+                for custom_type in custom_object_types:
+                    model = custom_type.get_model()
                     if model:
                         yield model
+        finally:
+            # Clean up the recursion guard
+            self._in_get_models = False
 
 
 config = CustomObjectsPluginConfig
diff --git a/netbox_custom_objects/choices.py b/netbox_custom_objects/choices.py
@@ -18,3 +18,21 @@ class MappingFieldTypeChoices(ChoiceSet):
         (DATETIME, _("DateTime"), "blue"),
         (OBJECT, _("Object"), "orange"),
     )
+
+
+#
+# Search
+#
+
+class SearchWeightChoices(ChoiceSet):
+    WEIGHT_NONE = 0
+    WEIGHT_LOW = 1000
+    WEIGHT_MEDIUM = 500
+    WEIGHT_HIGH = 100
+
+    CHOICES = (
+        (WEIGHT_HIGH, _('High (100)')),
+        (WEIGHT_MEDIUM, _('Medium (500)')),
+        (WEIGHT_LOW, _('Low (1000)')),
+        (WEIGHT_NONE, _('Not searchable')),
+    )
diff --git a/netbox_custom_objects/forms.py b/netbox_custom_objects/forms.py
@@ -9,6 +9,7 @@
 from utilities.forms.rendering import FieldSet
 from utilities.object_types import object_type_name
 
+from netbox_custom_objects.choices import SearchWeightChoices
 from netbox_custom_objects.constants import APP_LABEL
 from netbox_custom_objects.models import (CustomObjectObjectType,
                                           CustomObjectType,
@@ -104,13 +105,21 @@ class CustomObjectTypeFieldForm(CustomFieldForm):
         queryset=CustomObjectObjectType.objects.public(),
         help_text=_("Type of the related object (for object/multi-object fields only)"),
     )
+    search_weight = forms.ChoiceField(
+        choices=SearchWeightChoices,
+        required=False,
+        help_text=_(
+            "Weighting for search. Lower values are considered more important. Fields with a search weight of 0 "
+            "will be ignored."
+        ),
+    )
 
     fieldsets = (
         FieldSet(
             "custom_object_type",
-            "primary",
             "name",
             "label",
+            "primary",
             "group_name",
             "description",
             "type",
diff --git a/netbox_custom_objects/migrations/0006_alter_customobjecttypefield_name_and_more.py b/netbox_custom_objects/migrations/0006_alter_customobjecttypefield_name_and_more.py
@@ -0,0 +1,40 @@
+# Generated by Django 5.2.4 on 2025-08-12 17:38
+
+import django.core.validators
+import re
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("netbox_custom_objects", "0005_alter_customobjecttype_description"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="customobjecttypefield",
+            name="name",
+            field=models.CharField(
+                max_length=50,
+                validators=[
+                    django.core.validators.RegexValidator(
+                        flags=re.RegexFlag["IGNORECASE"],
+                        message="Only alphanumeric characters and underscores are allowed.",
+                        regex="^[a-z0-9_]+$",
+                    ),
+                    django.core.validators.RegexValidator(
+                        flags=re.RegexFlag["IGNORECASE"],
+                        inverse_match=True,
+                        message="Double underscores are not permitted in custom object field names.",
+                        regex="__",
+                    ),
+                ],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="customobjecttypefield",
+            name="search_weight",
+            field=models.PositiveSmallIntegerField(default=500),
+        ),
+    ]
diff --git a/netbox_custom_objects/models.py b/netbox_custom_objects/models.py
@@ -41,6 +41,7 @@
     get_model_features,
 )
 from netbox.registry import registry
+from netbox.search import SearchIndex
 from utilities import filters
 from utilities.datetime import datetime_from_timestamp
 from utilities.object_types import object_type_name
@@ -291,6 +292,7 @@ def get_or_create_content_type(self):
     def _fetch_and_generate_field_attrs(
         self,
         fields,
+        skip_object_fields=False,
     ):
         field_attrs = {
             "_primary_field_id": -1,
@@ -307,6 +309,10 @@ def _fetch_and_generate_field_attrs(
 
         for field in fields:
             field_type = FIELD_TYPE_CLASS[field.type]()
+            if skip_object_fields:
+                if field.type in [CustomFieldTypeChoices.TYPE_OBJECT, CustomFieldTypeChoices.TYPE_MULTIOBJECT]:
+                    continue
+
             field_name = field.name
 
             field_attrs["_field_objects"][field.id] = {
@@ -356,11 +362,32 @@ def get_content_type_label(custom_object_type_id):
         custom_object_type = CustomObjectType.objects.get(pk=custom_object_type_id)
         return f"Custom Objects > {custom_object_type.name}"
 
+    def register_custom_object_search_index(self, model):
+        # model must be an instance of this CustomObjectType's get_model() generated class
+        fields = []
+        for field in self.fields.filter(search_weight__gt=0):
+            fields.append((field.name, field.search_weight))
+
+        attrs = {
+            "model": model,
+            "fields": tuple(fields),
+            "display_attrs": tuple(),
+        }
+        search_index = type(
+            f"{self.name}SearchIndex",
+            (SearchIndex,),
+            attrs,
+        )
+        label = f"{APP_LABEL}.{self.get_table_model_name(self.id).lower()}"
+        registry["search"][label] = search_index
+
     def get_model(
         self,
         fields=None,
         manytomany_models=None,
         app_label=None,
+        skip_object_fields=False,
+        no_cache=False,
     ):
         """
         Generates a temporary Django model based on available fields that belong to
@@ -376,12 +403,16 @@ def get_model(
             have the same app_label. If passed along in this parameter, then the
             generated model will use that one instead of generating a unique one.
         :type app_label: Optional[String]
+        :param skip_object_fields: Don't add object or multiobject fields to the model
+        :type skip_object_fields: bool
+        :param no_cache: Don't cache the generated model or attempt to pull from cache
+        :type no_cache: bool
         :return: The generated model.
         :rtype: Model
         """
 
         # Check if we have a cached model for this CustomObjectType
-        if self.is_model_cached(self.id):
+        if self.is_model_cached(self.id) and not no_cache:
             model = self.get_cached_model(self.id)
             # Ensure the serializer is registered even for cached models
             from netbox_custom_objects.api.serializers import get_serializer_class
@@ -422,7 +453,7 @@ def get_model(
             "custom_object_type_id": self.id,
         }
 
-        field_attrs = self._fetch_and_generate_field_attrs(fields)
+        field_attrs = self._fetch_and_generate_field_attrs(fields, skip_object_fields=skip_object_fields)
 
         attrs.update(**field_attrs)
 
@@ -471,14 +502,18 @@ def wrapped_post_through_setup(self, cls):
             self._after_model_generation(attrs, model)
 
         # Cache the generated model
-        self._model_cache[self.id] = model
+        if not no_cache:
+            self._model_cache[self.id] = model
 
         # Register the serializer for this model
         if not manytomany_models:
             from netbox_custom_objects.api.serializers import get_serializer_class
 
             get_serializer_class(model)
 
+        # Register the global SearchIndex for this model
+        self.register_custom_object_search_index(model)
+
         return model
 
     def create_model(self):
@@ -487,7 +522,6 @@ def create_model(self):
 
         # Ensure the ContentType exists and is immediately available
         ct = self.get_or_create_content_type()
-        model = self.get_model()
         features = get_model_features(model)
         ct.public = True
         ct.features = features
@@ -496,6 +530,8 @@ def create_model(self):
         with connection.schema_editor() as schema_editor:
             schema_editor.create_model(model)
 
+        self.register_custom_object_search_index(model)
+
     def save(self, *args, **kwargs):
         needs_db_create = self._state.adding
         super().save(*args, **kwargs)
@@ -532,9 +568,10 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode
         max_length=50,
         choices=CustomFieldTypeChoices,
         default=CustomFieldTypeChoices.TYPE_TEXT,
-        help_text=_("The type of data this custom field holds"),
+        help_text=_("The type of data this custom object field holds"),
     )
     primary = models.BooleanField(
+        verbose_name=_("primary name field"),
         default=False,
         help_text=_(
             "Indicates that this field's value will be used as the object's displayed name"
@@ -550,7 +587,7 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode
     name = models.CharField(
         verbose_name=_("name"),
         max_length=50,
-        help_text=_("Internal field name"),
+        help_text=_("Internal field name, e.g. \"vendor_label\""),
         validators=(
             RegexValidator(
                 regex=r"^[a-z0-9_]+$",
@@ -560,7 +597,7 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode
             RegexValidator(
                 regex=r"__",
                 message=_(
-                    "Double underscores are not permitted in custom field names."
+                    "Double underscores are not permitted in custom object field names."
                 ),
                 flags=re.IGNORECASE,
                 inverse_match=True,
@@ -579,7 +616,7 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode
         verbose_name=_("group name"),
         max_length=50,
         blank=True,
-        help_text=_("Custom fields within the same group will be displayed together"),
+        help_text=_("Custom object fields within the same group will be displayed together"),
     )
     description = models.CharField(
         verbose_name=_("description"), max_length=200, blank=True
@@ -598,9 +635,9 @@ class CustomObjectTypeField(CloningMixin, ExportTemplatesMixin, ChangeLoggedMode
     )
     search_weight = models.PositiveSmallIntegerField(
         verbose_name=_("search weight"),
-        default=1000,
+        default=500,
         help_text=_(
-            "Weighting for search. Lower values are considered more important. Fields with a search weight of zero "
+            "Weighting for search. Lower values are considered more important. Fields with a search weight of 0 "
             "will be ignored."
         ),
     )
@@ -1274,6 +1311,9 @@ def save(self, *args, **kwargs):
 
         super().save(*args, **kwargs)
 
+        # Reregister SearchIndex with new set of searchable fields
+        self.custom_object_type.register_custom_object_search_index(model)
+
     def delete(self, *args, **kwargs):
         field_type = FIELD_TYPE_CLASS[self.type]()
         model_field = field_type.get_model_field(self)
@@ -1292,6 +1332,9 @@ def delete(self, *args, **kwargs):
 
         super().delete(*args, **kwargs)
 
+        # Reregister SearchIndex with new set of searchable fields
+        self.custom_object_type.register_custom_object_search_index(model)
+
 
 class CustomObjectObjectTypeManager(ObjectTypeManager):
 
diff --git a/netbox_custom_objects/search.py b/netbox_custom_objects/search.py
