rustdoc-search: use stringdex partitioned search tree

Author: notriddle

Cargo.lock

@@ -4725,6 +4725,7 @@ dependencies = [
  "serde_json",
  "sha2",
  "smallvec",
+ "stringdex",
  "tempfile",
  "threadpool",
  "tracing",
@@ -5132,6 +5133,15 @@ dependencies = [
  "quote",
 ]
 
+[[package]]
+name = "stringdex"
+version = "0.0.1-alpha1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a9ed1635ee8462c251e9c85aa1dd345419bd004bb75ac9a96e4c442a7974683"
+dependencies = [
+ "stacker",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"

src/ci/docker/host-x86_64/tidy/eslint.version

@@ -1 +1 @@
-8.6.0
+8.6.0

src/etc/htmldocck.py

@@ -15,6 +15,7 @@
 import re
 import shlex
 from collections import namedtuple
+from pathlib import Path
 
 try:
     from html.parser import HTMLParser
@@ -242,6 +243,11 @@ def resolve_path(self, path):
             return self.last_path
 
     def get_absolute_path(self, path):
+        if "*" in path:
+            paths = list(Path(self.root).glob(path))
+            if len(paths) != 1:
+                raise FailedCheck("glob path does not resolve to one file")
+            path = str(paths[0])
         return os.path.join(self.root, path)
 
     def get_file(self, path):

src/librustdoc/Cargo.toml

@@ -21,6 +21,7 @@ rustdoc-json-types = { path = "../rustdoc-json-types" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 smallvec = "1.8.1"
+stringdex = "0.0.1-alpha1"
 tempfile = "3"
 threadpool = "1.8.1"
 tracing = "0.1"

src/librustdoc/build.rs

@@ -10,6 +10,7 @@ fn main() {
         "static/css/normalize.css",
         "static/js/main.js",
         "static/js/search.js",
+        "static/js/stringdex.js",
         "static/js/settings.js",
         "static/js/src-script.js",
         "static/js/storage.js",

src/librustdoc/formats/cache.rs

@@ -1,6 +1,5 @@
 use std::mem;
 
-use rustc_ast::join_path_syms;
 use rustc_attr_data_structures::StabilityLevel;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet};
@@ -558,7 +557,6 @@ fn add_item_to_search_index(tcx: TyCtxt<'_>, cache: &mut Cache, item: &clean::It
         clean::ItemKind::ImportItem(import) => import.source.did.unwrap_or(item_def_id),
         _ => item_def_id,
     };
-    let path = join_path_syms(parent_path);
     let impl_id = if let Some(ParentStackItem::Impl { item_id, .. }) = cache.parent_stack.last() {
         item_id.as_def_id()
     } else {
@@ -577,11 +575,11 @@ fn add_item_to_search_index(tcx: TyCtxt<'_>, cache: &mut Cache, item: &clean::It
         ty: item.type_(),
         defid: Some(defid),
         name,
-        path,
+        module_path: parent_path.to_vec(),
         desc,
         parent: parent_did,
         parent_idx: None,
-        exact_path: None,
+        exact_module_path: None,
         impl_id,
         search_type,
         aliases,

src/librustdoc/formats/item_type.rs

@@ -4,7 +4,7 @@ use std::fmt;
 
 use rustc_hir::def::{CtorOf, DefKind};
 use rustc_span::hygiene::MacroKind;
-use serde::{Serialize, Serializer};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
 
 use crate::clean;
 
@@ -68,6 +68,52 @@ impl Serialize for ItemType {
     }
 }
 
+impl<'de> Deserialize<'de> for ItemType {
+    fn deserialize<D>(deserializer: D) -> Result<ItemType, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        struct ItemTypeVisitor;
+        impl<'de> de::Visitor<'de> for ItemTypeVisitor {
+            type Value = ItemType;
+            fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                write!(formatter, "an integer between 0 and 25")
+            }
+            fn visit_u64<E: de::Error>(self, v: u64) -> Result<ItemType, E> {
+                Ok(match v {
+                    0 => ItemType::Keyword,
+                    1 => ItemType::Primitive,
+                    2 => ItemType::Module,
+                    3 => ItemType::ExternCrate,
+                    4 => ItemType::Import,
+                    5 => ItemType::Struct,
+                    6 => ItemType::Enum,
+                    7 => ItemType::Function,
+                    8 => ItemType::TypeAlias,
+                    9 => ItemType::Static,
+                    10 => ItemType::Trait,
+                    11 => ItemType::Impl,
+                    12 => ItemType::TyMethod,
+                    13 => ItemType::Method,
+                    14 => ItemType::StructField,
+                    15 => ItemType::Variant,
+                    16 => ItemType::Macro,
+                    17 => ItemType::AssocType,
+                    18 => ItemType::Constant,
+                    19 => ItemType::AssocConst,
+                    20 => ItemType::Union,
+                    21 => ItemType::ForeignType,
+                    23 => ItemType::ProcAttribute,
+                    24 => ItemType::ProcDerive,
+                    25 => ItemType::TraitAlias,
+                    _ => return Err(E::missing_field("unknown number")),
+                })
+            }
+        }
+        deserializer.deserialize_any(ItemTypeVisitor)
+    }
+}
+
 impl<'a> From<&'a clean::Item> for ItemType {
     fn from(item: &'a clean::Item) -> ItemType {
         let kind = match &item.kind {

src/librustdoc/html/render/mod.rs

@@ -132,11 +132,11 @@ pub(crate) struct IndexItem {
     pub(crate) ty: ItemType,
     pub(crate) defid: Option<DefId>,
     pub(crate) name: Symbol,
-    pub(crate) path: String,
+    pub(crate) module_path: Vec<Symbol>,
     pub(crate) desc: String,
     pub(crate) parent: Option<DefId>,
-    pub(crate) parent_idx: Option<isize>,
-    pub(crate) exact_path: Option<String>,
+    pub(crate) parent_idx: Option<usize>,
+    pub(crate) exact_module_path: Option<Vec<Symbol>>,
     pub(crate) impl_id: Option<DefId>,
     pub(crate) search_type: Option<IndexItemFunctionType>,
     pub(crate) aliases: Box<[Symbol]>,
@@ -193,6 +193,62 @@ impl RenderType {
             write_optional_id(self.id, string);
         }
     }
+    fn read_from_bytes(string: &[u8]) -> (RenderType, usize) {
+        let mut i = 0;
+        if string[i] == b'{' {
+            i += 1;
+            let (id, offset) = RenderTypeId::read_from_bytes(&string[i..]);
+            i += offset;
+            let generics = if string[i] == b'{' {
+                i += 1;
+                let mut generics = Vec::new();
+                while string[i] != b'}' {
+                    let (ty, offset) = RenderType::read_from_bytes(&string[i..]);
+                    i += offset;
+                    generics.push(ty);
+                }
+                assert!(string[i] == b'}');
+                i += 1;
+                Some(generics)
+            } else {
+                None
+            };
+            let bindings = if string[i] == b'{' {
+                i += 1;
+                let mut bindings = Vec::new();
+                while string[i] == b'{' {
+                    i += 1;
+                    let (binding, boffset) = RenderTypeId::read_from_bytes(&string[i..]);
+                    i += boffset;
+                    let mut bconstraints = Vec::new();
+                    assert!(string[i] == b'{');
+                    i += 1;
+                    while string[i] != b'}' {
+                        let (constraint, coffset) = RenderType::read_from_bytes(&string[i..]);
+                        i += coffset;
+                        bconstraints.push(constraint);
+                    }
+                    assert!(string[i] == b'}');
+                    i += 1;
+                    bindings.push((binding.unwrap(), bconstraints));
+                    assert!(string[i] == b'}');
+                    i += 1;
+                }
+                assert!(string[i] == b'}');
+                i += 1;
+                Some(bindings)
+            } else {
+                None
+            };
+            assert!(string[i] == b'}');
+            i += 1;
+            (RenderType { id, generics, bindings }, i)
+        } else {
+            let (id, offset) = RenderTypeId::read_from_bytes(string);
+            i += offset;
+            (RenderType { id, generics: None, bindings: None }, i)
+        }
+    }
 }
 
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
@@ -214,7 +270,20 @@ impl RenderTypeId {
             RenderTypeId::Index(idx) => (*idx).try_into().unwrap(),
             _ => panic!("must convert render types to indexes before serializing"),
         };
-        search_index::encode::write_vlqhex_to_string(id, string);
+        search_index::encode::write_signed_vlqhex_to_string(id, string);
+    }
+    fn read_from_bytes(string: &[u8]) -> (Option<RenderTypeId>, usize) {
+        let Some((value, offset)) = search_index::encode::read_signed_vlqhex_from_string(string)
+        else {
+            return (None, 0);
+        };
+        let value = isize::try_from(value).unwrap();
+        let ty = match value {
+            ..0 => Some(RenderTypeId::Index(value)),
+            0 => None,
+            1.. => Some(RenderTypeId::Index(value - 1)),
+        };
+        (ty, offset)
     }
 }
 
@@ -228,12 +297,55 @@ pub(crate) struct IndexItemFunctionType {
 }
 
 impl IndexItemFunctionType {
-    fn write_to_string<'a>(
-        &'a self,
-        string: &mut String,
-        backref_queue: &mut VecDeque<&'a IndexItemFunctionType>,
-    ) {
-        assert!(backref_queue.len() <= 16);
+    fn read_from_string_without_param_names(string: &[u8]) -> (IndexItemFunctionType, usize) {
+        let mut i = 0;
+        if string[i] == b'`' {
+            return (
+                IndexItemFunctionType {
+                    inputs: Vec::new(),
+                    output: Vec::new(),
+                    where_clause: Vec::new(),
+                    param_names: Vec::new(),
+                },
+                1,
+            );
+        }
+        assert_eq!(b'{', string[i]);
+        i += 1;
+        fn read_args_from_string(string: &[u8]) -> (Vec<RenderType>, usize) {
+            let mut i = 0;
+            let mut params = Vec::new();
+            if string[i] == b'{' {
+                // multiple params
+                i += 1;
+                while string[i] != b'}' {
+                    let (ty, offset) = RenderType::read_from_bytes(&string[i..]);
+                    i += offset;
+                    params.push(ty);
+                }
+                i += 1;
+            } else if string[i] != b'}' {
+                let (tyid, offset) = RenderTypeId::read_from_bytes(&string[i..]);
+                params.push(RenderType { id: tyid, generics: None, bindings: None });
+                i += offset;
+            }
+            (params, i)
+        }
+        let (inputs, offset) = read_args_from_string(&string[i..]);
+        i += offset;
+        let (output, offset) = read_args_from_string(&string[i..]);
+        i += offset;
+        let mut where_clause = Vec::new();
+        while string[i] != b'}' {
+            let (constraint, offset) = read_args_from_string(&string[i..]);
+            i += offset;
+            where_clause.push(constraint);
+        }
+        assert_eq!(b'}', string[i], "{} {}", String::from_utf8_lossy(&string), i);
+        i += 1;
+        (IndexItemFunctionType { inputs, output, where_clause, param_names: Vec::new() }, i)
+    }
+    fn write_to_string_without_param_names<'a>(&'a self, string: &mut String) {
         // If we couldn't figure out a type, just write 0,
         // which is encoded as `` ` `` (see RenderTypeId::write_to_string).
         let has_missing = self
@@ -243,18 +355,7 @@ impl IndexItemFunctionType {
             .any(|i| i.id.is_none() && i.generics.is_none());
         if has_missing {
             string.push('`');
-        } else if let Some(idx) = backref_queue.iter().position(|other| *other == self) {
-            // The backref queue has 16 items, so backrefs use
-            // a single hexit, disjoint from the ones used for numbers.
-            string.push(
-                char::try_from('0' as u32 + u32::try_from(idx).unwrap())
-                    .expect("last possible value is '?'"),
-            );
         } else {
-            backref_queue.push_front(self);
-            if backref_queue.len() > 16 {
-                backref_queue.pop_back();
-            }
             string.push('{');
             match &self.inputs[..] {
                 [one] if one.generics.is_none() && one.bindings.is_none() => {