Search: Add rank modifier to search index

TYPE: Feature
LINK: None

Author: Tschuppi81

src/onegov/file/models/file.py

@@ -111,6 +111,7 @@ class SearchableFile(ORMSearchable):
     """
 
     fts_public = True
+    fts_rank_modifier = 0.1
     fts_title_property = 'name'
     fts_properties = {
         'name': {'type': 'text', 'weight': 'A'},

src/onegov/org/models/meeting.py

@@ -38,6 +38,7 @@ class Meeting(
 
     fts_type_title = _('Meetings')
     fts_public = True
+    fts_rank_modifier = 0.1
     fts_title_property = 'display_name'
     fts_properties = {
         'title_text': {'type': 'text', 'weight': 'A'},

src/onegov/org/models/meeting_item.py

@@ -23,6 +23,7 @@ class MeetingItem(Base, ORMSearchable):
 
     fts_type_title = _('Agenda')
     fts_public = True
+    fts_rank_modifier = 0.1
     fts_title_property = 'title'
     fts_properties = {
         'title': {'type': 'text', 'weight': 'A'},

src/onegov/org/models/search.py

@@ -11,7 +11,7 @@
 from markupsafe import Markup
 from operator import itemgetter
 from sedate import align_date_to_day, as_datetime, replace_timezone, utcnow
-from sqlalchemy import case, func, inspect, type_coerce
+from sqlalchemy import func, inspect, type_coerce
 from sqlalchemy_utils import escape_like
 
 
@@ -329,22 +329,8 @@ def generic_search(self) -> Query[SearchIndex]:
                         math.log(1e-6)
                     )
                 )
-                # HACK: We may want to add some fts_rank_modifier property
-                #       to searchable models instead and add a column to
-                #       the index for that, that would allow us to weigh
-                #       results per type more effectively. Currently files
-                #       are he most egregious offender though, so we just
-                #       hardcode this into the query.
-                * case(
-                    (SearchIndex.owner_tablename == 'files', 0.1),
-                    # NOTE: Tickets may be excluded entirely in the
-                    #       future but for now we'll de-prioritize them
-                    (SearchIndex.owner_tablename == 'tickets', 0.2),
-                    # de-priorize RIS meeting documents
-                    (SearchIndex.owner_tablename == 'par_meetings', 0.1),
-                    (SearchIndex.owner_tablename == 'par_meeting_items', 0.1),
-                    else_=1.0
-                )
+                # per-type rank weight (Searchable::fts_rank_modifier)
+                * SearchIndex.rank_modifier
             ).desc().label('rank')
         )
         return self.apply_common_filters(query)

src/onegov/search/indexer.py

@@ -37,6 +37,7 @@ class IndexTask(TypedDict):
         public: bool
         suggestion: list[str]
         tags: list[str]
+        rank_modifier: float
         last_change: datetime | None
         publication_start: datetime | None
         publication_end: datetime | None
@@ -175,6 +176,7 @@ def index(
                 '_public': task['public'],
                 '_access': task.get('access', 'public'),
                 '_last_change': task['last_change'],
+                '_rank_modifier': task['rank_modifier'],
                 '_tags': _tags,
                 '_suggestion': task['suggestion'],
                 '_publication_start':
@@ -259,6 +261,7 @@ def index(
                 SearchIndex.public: bindparam('_public'),
                 SearchIndex.access: bindparam('_access'),
                 SearchIndex.last_change: bindparam('_last_change'),
+                SearchIndex.rank_modifier: bindparam('_rank_modifier'),
                 SearchIndex._tags:
                     bindparam('_tags', type_=ARRAY(Text)),
                 SearchIndex.suggestion: bindparam('_suggestion'),
@@ -281,6 +284,7 @@ def index(
                 'public': stmt.excluded.public,
                 'access': stmt.excluded.access,
                 'last_change': stmt.excluded.last_change,
+                'rank_modifier': stmt.excluded.rank_modifier,
                 'tags': stmt.excluded.tags,
                 'suggestion': stmt.excluded.suggestion,
                 'title_vector': stmt.excluded.title_vector,
@@ -631,6 +635,7 @@ def index_task(
                 'public': obj.fts_public,
                 'suggestion': [],
                 'tags': obj.fts_tags or [],
+                'rank_modifier': obj.fts_rank_modifier,
                 'last_change': obj.fts_last_change,
                 'publication_start': obj.fts_publication_start,
                 'publication_end': obj.fts_publication_end,

src/onegov/search/mixins.py

@@ -164,6 +164,12 @@ def fts_tags(self) -> list[str] | None:
         """ A list of tags associated with this content. """
         return None
 
+    @property
+    def fts_rank_modifier(self) -> float:
+        """ Multiplies this entry's search rank. 1.0 is neutral; a lower
+        value de-prioritizes this type in mixed search results. """
+        return 1.0
+
 
 class ORMSearchable(Searchable):
     """ Extends the default :class:`Searchable` class with sensible defaults

src/onegov/search/search_index.py

@@ -65,6 +65,9 @@ class SearchIndex(Base, UTCPublicationMixin):
     #: Suggestions for search functionality (Searchable::fts_suggestion)
     suggestion: Mapped[list[str] | None] = mapped_column(ARRAY(String))
 
+    #: Rank multiplier for this entry (Searchable::fts_rank_modifier)
+    rank_modifier: Mapped[float] = mapped_column(default=1.0)
+
     #: Postgres full-text search (fts) index (title)
     title_vector: Mapped[str] = mapped_column(TSVECTOR)
 

src/onegov/search/upgrade.py

@@ -7,7 +7,7 @@
 
 from onegov.core.upgrade import upgrade_task, UpgradeContext
 from onegov.search.utils import searchable_sqlalchemy_models
-from sqlalchemy import inspect, text, Column, String
+from sqlalchemy import inspect, text, Column, Float, String
 from sqlalchemy.dialects.postgresql import ARRAY, TSVECTOR
 
 
@@ -98,3 +98,18 @@ def split_title_and_data_tsvector_columns(context: UpgradeContext) -> None:
             columns=['data_vector'],
             postgresql_using='gin'
         )
+
+
+@upgrade_task('Add rank_modifier to search_index')
+def add_search_index_rank_modifier(context: UpgradeContext) -> None:
+    if not context.has_table('search_index'):
+        return
+    if context.has_column('search_index', 'rank_modifier'):
+        return
+    context.operations.add_column(
+        'search_index',
+        Column(
+            'rank_modifier', Float, nullable=False, server_default='1.0'
+        )
+    )
+    # existing files/tickets rows correct themselves on next reindex

src/onegov/ticket/models/ticket.py

@@ -182,6 +182,7 @@ class Ticket(Base, TimestampMixin, ORMSearchable):
     #       so we just manually specify it for now.
     fts_type_title = TranslationString('Tickets', domain='onegov.org')
     fts_public = False
+    fts_rank_modifier = 0.2
     fts_title_property = 'number'
     fts_properties = {
         'number': {'type': 'text', 'weight': 'A'},

tests/onegov/search/test_indexer.py

@@ -107,6 +107,7 @@ def fts_tags(self) -> list[str]:
         'public': True,
         'suggestion': ['About'],
         'tags': ['aboutus', 'company'],
+        'rank_modifier': 1.0,
         'publication_start': None,
         'publication_end': None,
         'last_change': creation_date,
@@ -148,6 +149,7 @@ def fts_tags(self) -> list[str]:
         'public': True,
         'suggestion': ['About'],
         'tags': ['aboutus', 'company'],
+        'rank_modifier': 1.0,
         'publication_start': None,
         'publication_end': None,
         'last_change': creation_date,
@@ -292,6 +294,7 @@ def test_indexer_process(
         'publication_end': None,
         'suggestion': [],
         'tags': [],
+        'rank_modifier': 1.0,
         'language': 'en',
         'title': 'Go ahead and jump',
         'properties': {'title': 'Go ahead and jump'},
@@ -339,6 +342,7 @@ def test_indexer_process_mid_transaction(
         'language': 'en',
         'suggestion': ['John Doe'],
         'tags': [],
+        'rank_modifier': 1.0,
         'access': 'public',
         'public': True,
         'publication_start': None,
@@ -358,6 +362,7 @@ def test_indexer_process_mid_transaction(
         'language': 'en',
         'suggestion': ['Jane Doe'],
         'tags': [],
+        'rank_modifier': 1.0,
         'access': 'public',
         'public': True,
         'publication_start': None,
@@ -379,6 +384,7 @@ def test_indexer_process_mid_transaction(
         'language': 'en',
         'suggestion': ['Paul Atishon', 'Atishon Paul'],
         'tags': [],
+        'rank_modifier': 1.0,
         'access': 'public',
         'public': True,
         'publication_start': None,
@@ -418,6 +424,7 @@ def test_tags(
         'language': 'en',
         'suggestion': [],
         'tags': ['foo', 'BAR', 'baz'],
+        'rank_modifier': 1.0,
         'access': 'public',
         'public': True,
         'publication_start': None,