feat: handle multiple derivations for words in the metadata

Author: hippietrail

harper-cli/src/main.rs

@@ -76,6 +76,8 @@ enum Args {
         /// The document to mine words from.
         file: PathBuf,
     },
+    /// Get the word associated with a particular word id.
+    WordFromId { hash: u64 },
 }
 
 fn main() -> anyhow::Result<()> {
@@ -227,6 +229,21 @@ fn main() -> anyhow::Result<()> {
 
             println!("{json}");
 
+            // iterate through any and all derived_from and resolve the word from each wordid
+            if let Some(metadata) = dictionary.get_word_metadata_str(&word) {
+                if let Some(derived_from) = &metadata.derived_from {
+                    let derived_words: Vec<String> = derived_from
+                        .iter()
+                        .filter_map(|wordid| dictionary.get_word_from_id(wordid))
+                        .map(|word| word.iter().collect())
+                        .collect();
+
+                    if !derived_words.is_empty() {
+                        println!("derived_from: {:?}", derived_words);
+                    }
+                }
+            }
+
             Ok(())
         }
         Args::SummarizeLintRecord { file } => {
@@ -361,6 +378,11 @@ fn main() -> anyhow::Result<()> {
 
             Ok(())
         }
+        Args::WordFromId { hash } => {
+            let id = WordId::from_hash(hash);
+            println!("{:?}", dictionary.get_word_from_id(&id));
+            Ok(())
+        }
     }
 }
 
@@ -402,9 +424,14 @@ fn print_word_derivations(word: &str, annot: &str, dictionary: &impl Dictionary)
 
     let id = WordId::from_word_str(word);
 
-    let children = dictionary
-        .words_iter()
-        .filter(|e| dictionary.get_word_metadata(e).unwrap().derived_from == Some(id));
+    let children = dictionary.words_iter().filter(|e| {
+        dictionary
+            .get_word_metadata(e)
+            .unwrap()
+            .derived_from
+            .as_ref()
+            .is_some_and(|derived| derived.contains(&id))
+    });
 
     println!(" - {}", word);
 

harper-core/src/fat_token.rs

@@ -4,7 +4,7 @@ use crate::{CharStringExt, TokenKind};
 
 /// A [`Token`](crate::Token) that holds its content as a fat [`Vec<char>`] rather than as a
 /// [`Span`](crate::Span).
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
 pub struct FatToken {
     pub content: Vec<char>,
     pub kind: TokenKind,
@@ -20,7 +20,7 @@ impl From<FatStringToken> for FatToken {
 }
 
 /// Similar to a [`FatToken`], but uses a [`String`] as the underlying store.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct FatStringToken {
     pub content: String,
     pub kind: TokenKind,

harper-core/src/ignored_lints/lint_context.rs

@@ -1,4 +1,5 @@
 use serde::{Deserialize, Serialize};
+use std::hash::{Hash, Hasher};
 
 use crate::{
     Document, FatToken,
@@ -7,7 +8,7 @@ use crate::{
 
 /// A location-agnostic structure that attempts to captures the context and content that a [`Lint`]
 /// occurred.
-#[derive(Debug, Hash, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct LintContext {
     pub lint_kind: LintKind,
     pub suggestions: Vec<Suggestion>,
@@ -16,6 +17,16 @@ pub struct LintContext {
     pub tokens: Vec<FatToken>,
 }
 
+impl Hash for LintContext {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.lint_kind.hash(state);
+        self.suggestions.hash(state);
+        self.message.hash(state);
+        self.priority.hash(state);
+        self.tokens.hash(state);
+    }
+}
+
 impl LintContext {
     pub fn from_lint(lint: &Lint, document: &Document) -> Self {
         let Lint {

harper-core/src/spell/fst_dictionary.rs

@@ -307,52 +307,55 @@ mod tests {
     #[test]
     fn plural_llamas_derived_from_llama() {
         let dict = FstDictionary::curated();
-
-        assert_eq!(
+        assert!(
             dict.get_word_metadata_str("llamas")
                 .unwrap()
                 .derived_from
-                .unwrap(),
-            WordId::from_word_str("llama")
-        )
+                .as_ref()
+                .unwrap()
+                .contains(&WordId::from_word_str("llama"))
+        );
     }
 
     #[test]
     fn plural_cats_derived_from_cat() {
         let dict = FstDictionary::curated();
 
-        assert_eq!(
+        assert!(
             dict.get_word_metadata_str("cats")
                 .unwrap()
                 .derived_from
-                .unwrap(),
-            WordId::from_word_str("cat")
+                .as_ref()
+                .unwrap()
+                .contains(&WordId::from_word_str("cat"))
         );
     }
 
     #[test]
     fn unhappy_derived_from_happy() {
         let dict = FstDictionary::curated();
 
-        assert_eq!(
+        assert!(
             dict.get_word_metadata_str("unhappy")
                 .unwrap()
                 .derived_from
-                .unwrap(),
-            WordId::from_word_str("happy")
+                .as_ref()
+                .unwrap()
+                .contains(&WordId::from_word_str("happy"))
         );
     }
 
     #[test]
     fn quickly_derived_from_quick() {
         let dict = FstDictionary::curated();
 
-        assert_eq!(
+        assert!(
             dict.get_word_metadata_str("quickly")
                 .unwrap()
                 .derived_from
-                .unwrap(),
-            WordId::from_word_str("quick")
+                .as_ref()
+                .unwrap()
+                .contains(&WordId::from_word_str("quick"))
         );
     }
 }

harper-core/src/spell/mod.rs

@@ -14,7 +14,7 @@ mod rune;
 mod word_id;
 mod word_map;
 
-#[derive(PartialEq, Debug, Hash, Eq)]
+#[derive(PartialEq, Debug, Eq)]
 pub struct FuzzyMatchResult<'a> {
     pub word: &'a [char],
     pub edit_distance: u8,

harper-core/src/spell/rune/attribute_list.rs

@@ -1,4 +1,4 @@
-use hashbrown::HashMap;
+use hashbrown::{HashMap, HashSet};
 use serde::{Deserialize, Serialize};
 use smallvec::ToSmallVec;
 
@@ -89,11 +89,17 @@ impl AttributeList {
                     );
                     let t_metadata = dest.get_metadata_mut_chars(&new_word).unwrap();
                     t_metadata.append(&metadata);
-                    t_metadata.derived_from = Some(WordId::from_word_chars(&word.letters))
+                    t_metadata
+                        .derived_from
+                        .get_or_insert_with(HashSet::new)
+                        .insert(WordId::from_word_chars(&word.letters));
                 }
             } else {
                 for (key, mut value) in new_words.into_iter() {
-                    value.derived_from = Some(WordId::from_word_chars(&word.letters));
+                    value
+                        .derived_from
+                        .get_or_insert_with(HashSet::new)
+                        .insert(WordId::from_word_chars(&word.letters));
 
                     if let Some(val) = dest.get_metadata_mut_chars(&key) {
                         val.append(&value);

harper-core/src/spell/word_id.rs

@@ -31,4 +31,8 @@ impl WordId {
         let chars: CharString = text.as_ref().chars().collect();
         Self::from_word_chars(chars)
     }
+
+    pub fn from_hash(hash: u64) -> Self {
+        Self { hash }
+    }
 }

harper-core/src/token_kind.rs

@@ -2,8 +2,9 @@ use is_macro::Is;
 use serde::{Deserialize, Serialize};
 
 use crate::{ConjunctionData, NounData, Number, PronounData, Punctuation, Quote, WordMetadata};
+use std::hash::{Hash, Hasher};
 
-#[derive(Debug, Is, Clone, Serialize, Deserialize, Default, PartialOrd, Hash, Eq, PartialEq)]
+#[derive(Debug, Is, Clone, Serialize, Deserialize, Default, Eq, PartialEq)]
 #[serde(tag = "kind", content = "value")]
 pub enum TokenKind {
     /// `None` if the word does not exist in the dictionary.
@@ -26,6 +27,49 @@ pub enum TokenKind {
     Regexish,
 }
 
+impl Hash for TokenKind {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        match self {
+            TokenKind::Word(metadata) => {
+                metadata.hash(state);
+            }
+            TokenKind::Punctuation(punct) => {
+                punct.hash(state);
+            }
+            TokenKind::Decade => {
+                0.hash(state);
+            }
+            TokenKind::Number(number) => {
+                number.hash(state);
+            }
+            TokenKind::Space(space) => {
+                space.hash(state);
+            }
+            TokenKind::Newline(newline) => {
+                newline.hash(state);
+            }
+            TokenKind::EmailAddress => {
+                0.hash(state);
+            }
+            TokenKind::Url => {
+                0.hash(state);
+            }
+            TokenKind::Hostname => {
+                0.hash(state);
+            }
+            TokenKind::Unlintable => {
+                0.hash(state);
+            }
+            TokenKind::ParagraphBreak => {
+                0.hash(state);
+            }
+            TokenKind::Regexish => {
+                0.hash(state);
+            }
+        }
+    }
+}
+
 impl TokenKind {
     pub fn is_open_square(&self) -> bool {
         matches!(self, TokenKind::Punctuation(Punctuation::OpenSquare))

harper-core/src/word_metadata.rs

@@ -1,11 +1,14 @@
+use hashbrown::HashSet;
+use std::hash::{Hash, Hasher};
+
 use is_macro::Is;
 use paste::paste;
 use serde::{Deserialize, Serialize};
 use strum_macros::{Display, EnumString};
 
 use crate::WordId;
 
-#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Hash)]
+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct WordMetadata {
     pub noun: Option<NounData>,
     pub pronoun: Option<PronounData>,
@@ -28,7 +31,28 @@ pub struct WordMetadata {
     #[serde(default = "default_false")]
     pub common: bool,
     #[serde(default = "default_none")]
-    pub derived_from: Option<WordId>,
+    pub derived_from: Option<HashSet<WordId>>,
+}
+
+impl Hash for WordMetadata {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.noun.hash(state);
+        self.pronoun.hash(state);
+        self.verb.hash(state);
+        self.adjective.hash(state);
+        self.adverb.hash(state);
+        self.conjunction.hash(state);
+        self.swear.hash(state);
+        self.dialect.hash(state);
+        self.determiner.hash(state);
+        self.preposition.hash(state);
+        self.common.hash(state);
+        if let Some(ref derived_from) = self.derived_from {
+            for id in derived_from.iter() {
+                id.hash(state);
+            }
+        }
+    }
 }
 
 /// Needed for `serde`
@@ -111,7 +135,16 @@ impl WordMetadata {
             determiner: self.determiner || other.determiner,
             preposition: self.preposition || other.preposition,
             common: self.common || other.common,
-            derived_from: self.derived_from.or(other.derived_from),
+            derived_from: match (&self.derived_from, &other.derived_from) {
+                (Some(a), Some(b)) => {
+                    let mut set = a.clone();
+                    set.extend(b);
+                    Some(set)
+                }
+                (Some(a), None) => Some(a.clone()),
+                (None, Some(b)) => Some(b.clone()),
+                (None, None) => None,
+            },
         }
     }