Take analysis settings out of models

The approach where each model can define custom analysers did not match
Elasticsearch's structure well, and created more complexity than it was
worth.

Author: Treora

annotator/annotation.py

@@ -35,13 +35,10 @@
     }
 }
 
-ANALYSIS = {}
-
 class Annotation(es.Model):
 
     __type__ = TYPE
     __mapping__ = MAPPING
-    __analysis__ = ANALYSIS
 
     def save(self, *args, **kwargs):
         _add_default_permissions(self)

annotator/elasticsearch.py

@@ -72,9 +72,8 @@ def drop_all(self):
             self.conn.indices.close(self.index)
             self.conn.indices.delete(self.index)
 
-    def create_models(self, models):
+    def create_models(self, models, analysis_settings):
         mappings = _compile_mappings(models)
-        analysis = _compile_analysis(models)
 
         # Test for index existence while also checking if connection works
         try:
@@ -88,17 +87,17 @@ def create_models(self, models):
             # If index does not yet exist, simply create the index
             self.conn.indices.create(self.index, body={
                 'mappings': mappings,
-                'settings': {'analysis': analysis},
+                'settings': {'analysis': analysis_settings},
             })
         else:
             # Otherwise, update its settings and mappings
-            self._update_analysis(analysis)
+            self._update_analysis(analysis_settings)
             self._update_mappings(mappings)
 
     def _update_analysis(self, analysis):
         """Update analyzers and filters"""
-        settings = self.conn.indices.get_settings(index=self.index)
-        existing = settings[self.index]['settings']['index']['analysis']
+        settings = self.conn.indices.get_settings(index=self.index).values()[0]
+        existing = settings['settings']['index'].get('analysis', {})
         # Only bother if new settings would differ from existing settings
         if not self._analysis_up_to_date(existing, analysis):
             try:
@@ -122,7 +121,7 @@ def _update_mappings(self, mappings):
 
     @staticmethod
     def _analysis_up_to_date(existing, analysis):
-        """Tell whether existing settings are up to date"""
+        """Tell whether existing analysis settings are up to date"""
         new_analysis = existing.copy()
         for section, items in analysis.items():
             new_analysis.setdefault(section,{}).update(items)
@@ -264,21 +263,6 @@ def _compile_mappings(models):
     return mappings
 
 
-def _compile_analysis(models):
-    """Merge the custom analyzers and such from the models"""
-    analysis = {}
-    for model in models:
-        for section, items in model.get_analysis().items():
-            existing_items = analysis.setdefault(section, {})
-            for name in items:
-                if name in existing_items:
-                    fmt = "Duplicate definition of 'index.analysis.{}.{}'."
-                    msg = fmt.format(section, name)
-                    raise RuntimeError(msg)
-            existing_items.update(items)
-    return analysis
-
-
 def _csv_split(s, delimiter=','):
     return [r for r in csv.reader([s], delimiter=delimiter)][0]
 

annotator/elasticsearch_analyzers.py

@@ -0,0 +1,9 @@
+"""Custom Elasticsearch analyzers that can be used for indexing fields in
+   models (Annotation, Document).
+"""
+
+ANALYSIS = {
+    'analyzer': {},
+    'filter': {},
+    'tokenizer': {},
+}

annotator/reindexer.py

@@ -4,11 +4,13 @@
 
 from .annotation import Annotation
 from .document import Document
+from .elasticsearch_analyzers import ANALYSIS
 
 
 class Reindexer(object):
 
     es_models = Annotation, Document
+    analysis_settings = ANALYSIS
 
     def __init__(self, conn, interactive=False):
         self.conn = conn
@@ -60,7 +62,9 @@ def alias(self, index, alias):
 
     def get_index_config(self):
         # Configure index mappings
-        index_config = {'mappings': {}}
+        index_config = {'mappings': {},
+                        'settings': {'analysis': self.analysis_settings},
+                        }
         for model in self.es_models:
             index_config['mappings'].update(model.get_mapping())
         return index_config

run.py

@@ -20,7 +20,8 @@
 
 from flask import Flask, g, current_app
 import elasticsearch
-from annotator import es, annotation, auth, authz, document, store
+from annotator import es, annotation, auth, authz, document, \
+                      elasticsearch_analyzers, store
 from tests.helpers import MockUser, MockConsumer, MockAuthenticator
 from tests.helpers import mock_authorizer
 
@@ -61,7 +62,8 @@ def main():
         es.authorization_enabled = app.config['AUTHZ_ON']
 
     try:
-        es.create_models([annotation.Annotation, document.Document])
+        es.create_models(models=[annotation.Annotation, document.Document],
+                         analysis_settings=elasticsearch_analyzers.ANALYSIS)
     except elasticsearch.exceptions.RequestError as e:
         if e.error.startswith('MergeMappingException'):
             date = time.strftime('%Y-%m-%d')

tests/__init__.py

@@ -1,7 +1,8 @@
 import os
 from flask import Flask, g, request
 
-from annotator import es, auth, authz, annotation, store, document
+from annotator import es, auth, authz, annotation, document, \
+                      elasticsearch_analyzers, store
 
 from .helpers import MockUser, MockConsumer
 
@@ -33,7 +34,8 @@ def setup_class(cls):
         es.drop_all()
 
     def setup(self):
-        es.create_models([annotation.Annotation, document.Document])
+        es.create_models(models=[annotation.Annotation, document.Document],
+                         analysis_settings=elasticsearch_analyzers.ANALYSIS)
         es.conn.cluster.health(wait_for_status='yellow')
         self.cli = self.app.test_client()