First stab at tantivy's codec

Convert SegmentReader, InvertedIndexReader and postinglists to traits.
Add special functions to pushdown certain performance methods to keep
them strictly typed.

We rely on a ObjectSafeCodec contraption to avoid the proliferation of generics.
That object's point is to make sure we can build TermScorer with a concrete
codec specific type before reboxing it. (same thing for PhraseScorer).

fix performance regression: fix incorrect scorer cast for buffered union
bock wand

Author: fulmicoton-dd

benches/str_search_and_get.rs

@@ -45,7 +45,7 @@ fn build_shared_indices(num_docs: usize, distribution: &str) -> BenchIndex {
         match distribution {
             "dense_random" => {
                 for _doc_id in 0..num_docs {
-                    let suffix = rng.gen_range(0u64..1000u64);
+                    let suffix = rng.random_range(0u64..1000u64);
                     let str_val = format!("str_{:03}", suffix);
 
                     writer
@@ -71,7 +71,7 @@ fn build_shared_indices(num_docs: usize, distribution: &str) -> BenchIndex {
             }
             "sparse_random" => {
                 for _doc_id in 0..num_docs {
-                    let suffix = rng.gen_range(0u64..1000000u64);
+                    let suffix = rng.random_range(0u64..1000000u64);
                     let str_val = format!("str_{:07}", suffix);
 
                     writer

common/src/bitset.rs

@@ -178,13 +178,11 @@ impl TinySet {
 #[derive(Clone)]
 pub struct BitSet {
     tinysets: Box<[TinySet]>,
-    len: u64,
     max_value: u32,
 }
 impl std::fmt::Debug for BitSet {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("BitSet")
-            .field("len", &self.len)
             .field("max_value", &self.max_value)
             .finish()
     }
@@ -212,7 +210,6 @@ impl BitSet {
         let tinybitsets = vec![TinySet::empty(); num_buckets as usize].into_boxed_slice();
         BitSet {
             tinysets: tinybitsets,
-            len: 0,
             max_value,
         }
     }
@@ -230,7 +227,6 @@ impl BitSet {
         }
         BitSet {
             tinysets: tinybitsets,
-            len: max_value as u64,
             max_value,
         }
     }
@@ -249,17 +245,19 @@ impl BitSet {
 
     /// Intersect with tinysets
     fn intersect_update_with_iter(&mut self, other: impl Iterator<Item = TinySet>) {
-        self.len = 0;
         for (left, right) in self.tinysets.iter_mut().zip(other) {
             *left = left.intersect(right);
-            self.len += left.len() as u64;
         }
     }
 
     /// Returns the number of elements in the `BitSet`.
     #[inline]
     pub fn len(&self) -> usize {
-        self.len as usize
+        self.tinysets
+            .iter()
+            .copied()
+            .map(|tinyset| tinyset.len())
+            .sum::<u32>() as usize
     }
 
     /// Inserts an element in the `BitSet`
@@ -268,7 +266,7 @@ impl BitSet {
         // we do not check saturated els.
         let higher = el / 64u32;
         let lower = el % 64u32;
-        self.len += u64::from(self.tinysets[higher as usize].insert_mut(lower));
+        self.tinysets[higher as usize].insert_mut(lower);
     }
 
     /// Inserts an element in the `BitSet`
@@ -277,7 +275,7 @@ impl BitSet {
         // we do not check saturated els.
         let higher = el / 64u32;
         let lower = el % 64u32;
-        self.len -= u64::from(self.tinysets[higher as usize].remove_mut(lower));
+        self.tinysets[higher as usize].remove_mut(lower);
     }
 
     /// Returns true iff the elements is in the `BitSet`.
@@ -299,6 +297,9 @@ impl BitSet {
             .map(|delta_bucket| bucket + delta_bucket as u32)
     }
 
+    /// Returns the maximum number of elements in the bitset.
+    ///
+    /// Warning: The largest element the bitset can contain is `max_value - 1`.
     #[inline]
     pub fn max_value(&self) -> u32 {
         self.max_value

examples/custom_collector.rs

@@ -70,7 +70,7 @@ impl Collector for StatsCollector {
     fn for_segment(
         &self,
         _segment_local_id: u32,
-        segment_reader: &SegmentReader,
+        segment_reader: &dyn SegmentReader,
     ) -> tantivy::Result<StatsSegmentCollector> {
         let fast_field_reader = segment_reader.fast_fields().u64(&self.field)?;
         Ok(StatsSegmentCollector {

examples/faceted_search_with_tweaked_score.rs

@@ -65,7 +65,7 @@ fn main() -> tantivy::Result<()> {
         );
         let top_docs_by_custom_score =
             // Call TopDocs with a custom tweak score
-            TopDocs::with_limit(2).tweak_score(move |segment_reader: &SegmentReader| {
+            TopDocs::with_limit(2).tweak_score(move |segment_reader: &dyn SegmentReader| {
                 let ingredient_reader = segment_reader.facet_reader("ingredient").unwrap();
                 let facet_dict = ingredient_reader.facet_dict();
 

examples/iterating_docs_and_positions.rs

@@ -91,46 +91,10 @@ fn main() -> tantivy::Result<()> {
         }
     }
 
-    // A `Term` is a text token associated with a field.
-    // Let's go through all docs containing the term `title:the` and access their position
-    let term_the = Term::from_field_text(title, "the");
-
-    // Some other powerful operations (especially `.skip_to`) may be useful to consume these
+    // Some other powerful operations (especially `.seek`) may be useful to consume these
     // posting lists rapidly.
     // You can check for them in the [`DocSet`](https://docs.rs/tantivy/~0/tantivy/trait.DocSet.html) trait
     // and the [`Postings`](https://docs.rs/tantivy/~0/tantivy/trait.Postings.html) trait
 
-    // Also, for some VERY specific high performance use case like an OLAP analysis of logs,
-    // you can get better performance by accessing directly the blocks of doc ids.
-    for segment_reader in searcher.segment_readers() {
-        // A segment contains different data structure.
-        // Inverted index stands for the combination of
-        // - the term dictionary
-        // - the inverted lists associated with each terms and their positions
-        let inverted_index = segment_reader.inverted_index(title)?;
-
-        // This segment posting object is like a cursor over the documents matching the term.
-        // The `IndexRecordOption` arguments tells tantivy we will be interested in both term
-        // frequencies and positions.
-        //
-        // If you don't need all this information, you may get better performance by decompressing
-        // less information.
-        if let Some(mut block_segment_postings) =
-            inverted_index.read_block_postings(&term_the, IndexRecordOption::Basic)?
-        {
-            loop {
-                let docs = block_segment_postings.docs();
-                if docs.is_empty() {
-                    break;
-                }
-                // Once again these docs MAY contains deleted documents as well.
-                let docs = block_segment_postings.docs();
-                // Prints `Docs [0, 2].`
-                println!("Docs {docs:?}");
-                block_segment_postings.advance();
-            }
-        }
-    }
-
     Ok(())
 }

examples/warmer.rs

@@ -43,7 +43,7 @@ impl DynamicPriceColumn {
         }
     }
 
-    pub fn price_for_segment(&self, segment_reader: &SegmentReader) -> Option<Arc<Vec<Price>>> {
+    pub fn price_for_segment(&self, segment_reader: &dyn SegmentReader) -> Option<Arc<Vec<Price>>> {
         let segment_key = (segment_reader.segment_id(), segment_reader.delete_opstamp());
         self.price_cache.read().unwrap().get(&segment_key).cloned()
     }
@@ -157,7 +157,7 @@ fn main() -> tantivy::Result<()> {
     let query = query_parser.parse_query("cooking")?;
 
     let searcher = reader.searcher();
-    let score_by_price = move |segment_reader: &SegmentReader| {
+    let score_by_price = move |segment_reader: &dyn SegmentReader| {
         let price = price_dynamic_column
             .price_for_segment(segment_reader)
             .unwrap();

src/aggregation/accessor_helpers.rs

@@ -57,7 +57,7 @@ pub(crate) fn get_numeric_or_date_column_types() -> &'static [ColumnType] {
 
 /// Get fast field reader or empty as default.
 pub(crate) fn get_ff_reader(
-    reader: &SegmentReader,
+    reader: &dyn SegmentReader,
     field_name: &str,
     allowed_column_types: Option<&[ColumnType]>,
 ) -> crate::Result<(columnar::Column<u64>, ColumnType)> {
@@ -74,7 +74,7 @@ pub(crate) fn get_ff_reader(
 }
 
 pub(crate) fn get_dynamic_columns(
-    reader: &SegmentReader,
+    reader: &dyn SegmentReader,
     field_name: &str,
 ) -> crate::Result<Vec<columnar::DynamicColumn>> {
     let ff_fields = reader.fast_fields().dynamic_column_handles(field_name)?;
@@ -90,7 +90,7 @@ pub(crate) fn get_dynamic_columns(
 ///
 /// Is guaranteed to return at least one column.
 pub(crate) fn get_all_ff_reader_or_empty(
-    reader: &SegmentReader,
+    reader: &dyn SegmentReader,
     field_name: &str,
     allowed_column_types: Option<&[ColumnType]>,
     fallback_type: ColumnType,

src/aggregation/agg_data.rs

@@ -469,7 +469,7 @@ impl AggKind {
 /// Build AggregationsData by walking the request tree.
 pub(crate) fn build_aggregations_data_from_req(
     aggs: &Aggregations,
-    reader: &SegmentReader,
+    reader: &dyn SegmentReader,
     segment_ordinal: SegmentOrdinal,
     context: AggContextParams,
 ) -> crate::Result<AggregationsSegmentCtx> {
@@ -489,7 +489,7 @@ pub(crate) fn build_aggregations_data_from_req(
 fn build_nodes(
     agg_name: &str,
     req: &Aggregation,
-    reader: &SegmentReader,
+    reader: &dyn SegmentReader,
     segment_ordinal: SegmentOrdinal,
     data: &mut AggregationsSegmentCtx,
     is_top_level: bool,
@@ -728,7 +728,7 @@ fn build_nodes(
             let idx_in_req_data = data.push_filter_req_data(FilterAggReqData {
                 name: agg_name.to_string(),
                 req: filter_req.clone(),
-                segment_reader: reader.clone(),
+                segment_reader: reader.clone_arc(),
                 evaluator,
                 matching_docs_buffer,
                 is_top_level,
@@ -745,7 +745,7 @@ fn build_nodes(
 
 fn build_children(
     aggs: &Aggregations,
-    reader: &SegmentReader,
+    reader: &dyn SegmentReader,
     segment_ordinal: SegmentOrdinal,
     data: &mut AggregationsSegmentCtx,
 ) -> crate::Result<Vec<AggRefNode>> {
@@ -764,7 +764,7 @@ fn build_children(
 }
 
 fn get_term_agg_accessors(
-    reader: &SegmentReader,
+    reader: &dyn SegmentReader,
     field_name: &str,
     missing: &Option<Key>,
 ) -> crate::Result<Vec<(Column<u64>, ColumnType)>> {
@@ -817,7 +817,7 @@ fn build_terms_or_cardinality_nodes(
     agg_name: &str,
     field_name: &str,
     missing: &Option<Key>,
-    reader: &SegmentReader,
+    reader: &dyn SegmentReader,
     segment_ordinal: SegmentOrdinal,
     data: &mut AggregationsSegmentCtx,
     sub_aggs: &Aggregations,

src/aggregation/bucket/filter.rs

@@ -1,4 +1,5 @@
 use std::fmt::Debug;
+use std::sync::Arc;
 
 use common::BitSet;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -402,7 +403,7 @@ pub struct FilterAggReqData {
     /// The filter aggregation
     pub req: FilterAggregation,
     /// The segment reader
-    pub segment_reader: SegmentReader,
+    pub segment_reader: Arc<dyn SegmentReader>,
     /// Document evaluator for the filter query (precomputed BitSet)
     /// This is built once when the request data is created
     pub evaluator: DocumentQueryEvaluator,
@@ -416,7 +417,7 @@ impl FilterAggReqData {
     pub(crate) fn get_memory_consumption(&self) -> usize {
         // Estimate: name + segment reader reference + bitset + buffer capacity
         self.name.len()
-        + std::mem::size_of::<SegmentReader>()
+        + std::mem::size_of::<Arc<dyn SegmentReader>>()
         + self.evaluator.bitset.len() / 8 // BitSet memory (bits to bytes)
         + self.matching_docs_buffer.capacity() * std::mem::size_of::<DocId>()
         + std::mem::size_of::<bool>()
@@ -438,7 +439,7 @@ impl DocumentQueryEvaluator {
     pub(crate) fn new(
         query: Box<dyn Query>,
         schema: Schema,
-        segment_reader: &SegmentReader,
+        segment_reader: &dyn SegmentReader,
     ) -> crate::Result<Self> {
         let max_doc = segment_reader.max_doc();
 

src/aggregation/collector.rs

@@ -66,7 +66,7 @@ impl Collector for DistributedAggregationCollector {
     fn for_segment(
         &self,
         segment_local_id: crate::SegmentOrdinal,
-        reader: &crate::SegmentReader,
+        reader: &dyn SegmentReader,
     ) -> crate::Result<Self::Child> {
         AggregationSegmentCollector::from_agg_req_and_reader(
             &self.agg,
@@ -96,7 +96,7 @@ impl Collector for AggregationCollector {
     fn for_segment(
         &self,
         segment_local_id: crate::SegmentOrdinal,
-        reader: &crate::SegmentReader,
+        reader: &dyn SegmentReader,
     ) -> crate::Result<Self::Child> {
         AggregationSegmentCollector::from_agg_req_and_reader(
             &self.agg,
@@ -145,7 +145,7 @@ impl AggregationSegmentCollector {
     /// reader. Also includes validation, e.g. checking field types and existence.
     pub fn from_agg_req_and_reader(
         agg: &Aggregations,
-        reader: &SegmentReader,
+        reader: &dyn SegmentReader,
         segment_ordinal: SegmentOrdinal,
         context: &AggContextParams,
     ) -> crate::Result<Self> {