Merge pull request #1034 from openzim/larger_MAX_INDEXABLE_TITLE_WORD_SIZE

Changed xapian index compaction to safer FULL mode and increased MAX_INDEXABLE_TITLE_WORD_SIZE

Author: kelson42

.github/workflows/ci.yml

@@ -128,7 +128,7 @@ jobs:
         run: |
           set PKG_CONFIG_PATH=%cd%\BUILD_win-amd64\INSTALL\lib\pkgconfig
           dir %PKG_CONFIG_PATH%
-          meson.exe setup . build -Dwith_xapian_fuller=false -Dwerror=false --buildtype=release
+          meson.exe setup . build -Dwerror=false --buildtype=release
           cd build
           ninja.exe
 

meson.build

@@ -30,7 +30,6 @@ else
     private_conf.set('ENABLE_USE_MMAP', get_option('USE_MMAP'))
 endif
 private_conf.set('ENABLE_USE_BUFFER_HEADER', get_option('USE_BUFFER_HEADER'))
-private_conf.set('ENABLE_XAPIAN_FULLER', get_option('with_xapian_fuller'))
 
 static_linkage = get_option('static-linkage')
 static_linkage = static_linkage or get_option('default_library')=='static'

meson_options.txt

@@ -22,7 +22,5 @@ option('tests', type : 'boolean', value : true,
   description : 'Build the tests.')
 option('with_xapian', type : 'boolean', value: true,
   description: 'Build libzim with xapian support')
-option('with_xapian_fuller', type: 'boolean', value: true,
-  description: 'Create xapian archive using "FULLER" compaction.\nThis is a workaround for a compilation issue on Windows. This will be removed soon')
 option('test_data_dir', type : 'string', value: '',
   description: 'Where the test data are. If not set, meson will use a internal directory in  the build dir. If you want to download the data in the specified directory you can use `meson download_test_data`. As a special value, you can pass `none` to deactivate test using external test data.')

src/config.h.in

@@ -19,8 +19,6 @@
 
 #mesondefine ENABLE_XAPIAN
 
-#mesondefine ENABLE_XAPIAN_FULLER
-
 #mesondefine ENABLE_USE_MMAP
 
 #mesondefine ENABLE_USE_BUFFER_HEADER

src/constants.h

@@ -22,9 +22,7 @@
 #define DEFAULT_CLUSTER_SIZE 2*1024*1024
 
 // The size in bytes of the longest word that is indexable in a title.
-// Xapian's default value is 64 while the hard limit is 245, however crashes
-// have been observed with values as low as 150 (demonstrated by the unit test
-// Suggestion.handlingOfTooLongWords in test/suggestion.cpp).
+// Xapian's default value is 64 while the hard limit is 245.
 // Note that a similar limit applies to full-text indexing but we don't
 // provide a way to control it (so it is at Xapian's default value of 64)
-#define MAX_INDEXABLE_TITLE_WORD_SIZE 64
+#define MAX_INDEXABLE_TITLE_WORD_SIZE 240

src/writer/xapianIndexer.cpp

@@ -202,12 +202,7 @@ void XapianIndexer::indexTitle(const std::string& path, const std::string& title
 void XapianIndexer::indexingPostlude()
 {
   this->writableDatabase.commit();
-#if defined ENABLE_XAPIAN_FULLER
-  auto flags = Xapian::DBCOMPACT_SINGLE_FILE|Xapian::Compactor::FULLER;
-#else
-  auto flags = Xapian::DBCOMPACT_SINGLE_FILE;
-#endif
-  this->writableDatabase.compact(indexPath, flags);
+  this->writableDatabase.compact(indexPath, Xapian::DBCOMPACT_SINGLE_FILE|Xapian::Compactor::FULL);
   this->writableDatabase.close();
 }