feat: implement ngram tokenizer with token_chars and custom_token_chars [2.6] (#45046)

pr: #45040
issue: #45039

Signed-off-by: SpadeA <[email protected]>

Author: SpadeA-Tang

internal/core/thirdparty/tantivy/tantivy-binding/Cargo.lock

@@ -48,6 +48,7 @@ url = "2.5.4"
 prost = "0.13.5"
 once_cell = "1.20.3"
 unicode-general-category = "1.0.0"
+unicode_categories = "0.1.1"
 
 # lindera dependencies for fetch and prepare dictionary online.
 tokio = { version = "1.45.0", features = [

internal/core/thirdparty/tantivy/tantivy-binding/Cargo.toml

@@ -4,6 +4,7 @@ mod icu_tokneizer;
 mod jieba_tokenizer;
 mod lang_ident_tokenizer;
 mod lindera_tokenizer;
+mod ngram_tokenizer_with_chars;
 mod tokenizer;
 
 pub use self::char_group_tokenizer::CharGroupTokenizer;
@@ -12,5 +13,6 @@ pub use self::icu_tokneizer::IcuTokenizer;
 pub use self::jieba_tokenizer::JiebaTokenizer;
 pub use self::lang_ident_tokenizer::LangIdentTokenizer;
 pub use self::lindera_tokenizer::LinderaTokenizer;
+pub use self::ngram_tokenizer_with_chars::NgramTokenizerWithChars;
 
 pub(crate) use self::tokenizer::*;