Add Xtransformer to backend

.github/workflows/cicd.yml

@@ -81,17 +81,17 @@ jobs:
         # Selectively install the optional dependencies for some Python versions
         # For Python 3.10:
         if [[ ${{ matrix.python-version }} == '3.10' ]]; then
-          poetry install -E "nn omikuji yake voikko stwfsa";
+          poetry install -E "nn omikuji yake voikko stwfsa pecos";
         fi
         # For Python 3.11:
         if [[ ${{ matrix.python-version }} == '3.11' ]]; then
-          poetry install -E "fasttext spacy estnltk";
+          poetry install -E "fasttext spacy estnltk pecos";
           # download the small English pretrained spaCy model needed by spacy analyzer
           poetry run python -m spacy download en_core_web_sm --upgrade-strategy only-if-needed
         fi
         # For Python 3.12:
         if [[ ${{ matrix.python-version }} == '3.12' ]]; then
-          poetry install -E "nn fasttext yake stwfsa voikko spacy";
+          poetry install -E "nn fasttext yake stwfsa voikko spacy pecos";
           # download the small English pretrained spaCy model needed by spacy analyzer
           poetry run python -m spacy download en_core_web_sm --upgrade-strategy only-if-needed
         fi

Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.12-slim-bookworm
 LABEL org.opencontainers.image.authors="grp-natlibfi-annif@helsinki.fi"
 SHELL ["/bin/bash", "-c"]
 
-ARG optional_dependencies="voikko fasttext nn omikuji yake spacy stwfsa"
+ARG optional_dependencies="voikko fasttext nn omikuji yake spacy stwfsa pecos"
 ARG POETRY_VIRTUALENVS_CREATE=false
 
 # Install system dependencies needed at runtime:
@@ -36,6 +36,10 @@ RUN if [[ $optional_dependencies =~ "spacy" ]]; then \
 			python -m spacy download $model; \
 		done; \
 	fi
+RUN if [[ $optional_dependencies =~ "pecos" ]]; then \
+	mkdir /.cache -m a=rwx; \
+	fi
+
 
 # Second round of installation with the actual code:
 COPY annif /Annif/annif

annif/backend/__init__.py

@@ -89,6 +89,17 @@ def _tfidf() -> Type[AnnifBackend]:
     return tfidf.TFIDFBackend
 
 
+def _xtransformer() -> Type[AnnifBackend]:
+    try:
+        from . import xtransformer
+
+        return xtransformer.XTransformerBackend
+    except ImportError:
+        raise ValueError(
+            "XTransformer not available, not enabling XTransformer backend"
+        )
+
+
 def _yake() -> Type[AnnifBackend]:
     try:
         from . import yake
@@ -111,6 +122,7 @@ def _yake() -> Type[AnnifBackend]:
     "stwfsa": _stwfsa,
     "svc": _svc,
     "tfidf": _tfidf,
+    "xtransformer": _xtransformer,
     "yake": _yake,
 }
 

annif/backend/fasttext.py

@@ -117,11 +117,7 @@ def _create_model(self, params: dict[str, Any], jobs: int) -> None:
         self.info("creating fastText model")
         trainpath = os.path.join(self.datadir, self.TRAIN_FILE)
         modelpath = os.path.join(self.datadir, self.MODEL_FILE)
-        params = {
-            param: self.FASTTEXT_PARAMS[param](val)
-            for param, val in params.items()
-            if param in self.FASTTEXT_PARAMS
-        }
+        params = annif.util.apply_param_parse_config(self.FASTTEXT_PARAMS, params)
         if jobs != 0:  # jobs set by user to non-default value
             params["thread"] = jobs
         self.debug("Model parameters: {}".format(params))

annif/backend/mixins.py

@@ -7,6 +7,8 @@
 from typing import TYPE_CHECKING, Any
 
 import joblib
+import numpy as np
+from pecos.utils.featurization.text.vectorizers import Vectorizer
 from sklearn.feature_extraction.text import TfidfVectorizer
 
 import annif.util
@@ -57,7 +59,6 @@ def _suggest(
             return []
         return self._suggest_chunks(chunktexts, params)
 
-
 class TfidfVectorizerMixin:
     """Annif backend mixin that implements TfidfVectorizer functionality"""
 
@@ -92,3 +93,57 @@ def create_vectorizer(
             self.vectorizer, self.datadir, self.VECTORIZER_FILE, method=joblib.dump
         )
         return veccorpus
+
+class PecosTfidfVectorizerMixin:
+    """Annif backend mixin that implements TfidfVectorizer functionality from Pecos"""
+
+    VECTORIZER_FILE = "vectorizer"
+
+    vectorizer = None
+
+    def initialize_vectorizer(self) -> None:
+        if self.vectorizer is None:
+            path = os.path.join(self.datadir, self.VECTORIZER_FILE)
+            if os.path.exists(path):
+                self.debug("loading vectorizer from {}".format(path))
+
+                self.vectorizer = Vectorizer.load(path)
+            else:
+                raise NotInitializedException(
+                    "vectorizer file '{}' not found".format(path),
+                    backend_id=self.backend_id,
+                )
+
+    def vectorizer_dict(self, params: dict[str, Any]) -> dict[str, Any]:
+        """Create a vectorizer configuration dictionary from the given parameters."""
+
+        config = {
+            "base_vect_configs": [
+                {
+                    "ngram_range": params.get("ngram_range", [1, 1]),
+                    "max_df_ratio": 0.98,
+                    "analyzer": "word",
+                    "min_df_cnt": params.get("min_df", 1),
+                }
+            ]
+        }
+        return {"type": "tfidf", "kwargs": {**config}} 
+
+
+    def create_vectorizer(
+        self, input: Iterable[str], params: dict[str, Any] = None
+    ) -> csr_matrix:
+
+        self.info("creating Pecos vectorizer")
+        if params is None:
+            params = {}
+        data = list(input)
+        vectorizer_config = self.vectorizer_dict(params)
+        self.vectorizer = Vectorizer.train(data, vectorizer_config, np.float32)
+        self.vectorizer.save(os.path.join(self.datadir, self.VECTORIZER_FILE))
+        veccorpus = self.vectorizer.predict(
+            data,
+            threads=params.get("threads", -1)
+        )
+
+        return veccorpus

annif/backend/omikuji.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import os.path
-import shutil
 from typing import TYPE_CHECKING, Any
 
 import omikuji
@@ -103,9 +102,7 @@ def _create_model(self, params: dict[str, Any], jobs: int) -> None:
         hyper_param.collapse_every_n_layers = int(params["collapse_every_n_layers"])
 
         self._model = omikuji.Model.train_on_data(train_path, hyper_param, jobs or None)
-        if os.path.exists(model_path):
-            shutil.rmtree(model_path)
-        self._model.save(os.path.join(self.datadir, self.MODEL_FILE))
+        annif.util.atomic_save_folder(self._model, model_path)
 
     def _train(
         self,

annif/backend/stwfsa.py

@@ -7,7 +7,7 @@
 
 from annif.exception import NotInitializedException, NotSupportedException
 from annif.suggestion import SubjectSuggestion
-from annif.util import atomic_save, boolean
+from annif.util import apply_param_parse_config, atomic_save, boolean
 
 from . import backend
 
@@ -106,11 +106,7 @@ def _train(
         jobs: int = 0,
     ) -> None:
         X, y = self._load_data(corpus)
-        new_params = {
-            key: self.STWFSA_PARAMETERS[key](val)
-            for key, val in params.items()
-            if key in self.STWFSA_PARAMETERS
-        }
+        new_params = apply_param_parse_config(self.STWFSA_PARAMETERS, params)
         p = StwfsapyPredictor(
             graph=self.project.vocab.as_graph(),
             langs=frozenset([params["language"]]),