@@ -7,37 +7,38 @@ use std::collections::HashSet;
77use base64:: { engine:: general_purpose, Engine as _} ;
88
99/// LZJD digest builder/configuration
10+ #[ derive( Clone , Copy , Debug , Eq , PartialEq ) ]
1011pub struct Lzjd {
1112 /// number of k smallest hashes to keep (digest size)
12- pub k : usize ,
13+ k : usize ,
14+
1315 /// seed for the `MurmurHash3` hashing
14- pub seed : u32 ,
16+ seed : u32 ,
1517}
1618
1719impl Default for Lzjd {
20+ #[ inline]
1821 fn default ( ) -> Self {
1922 Self { k : 1024 , seed : 0 }
2023 }
2124}
2225
2326impl Lzjd {
24- /// Create new LZJD config with given k and seed
27+ /// Create a new Lzjd hasher with given size and seed
28+ #[ inline]
2529 #[ must_use]
2630 pub fn new ( k : usize , seed : u32 ) -> Self {
2731 Self { k, seed }
2832 }
2933
30- /// Build an LZJD digest from input bytes using `MurmurHash3_x86_32`.
31- ///
32- /// Returns a `Vec<u32>` of exactly `k` entries: the k-smallest hash values (sorted
33- /// ascending as signed i32, matching jLZJD), zero-padded when the input yields fewer
34- /// than `k` unique LZ78 phrases.
34+ /// Build an [`LzDigest`] digest from input bytes. Two digests can be used to find similarity,
35+ /// or a digest can be saved to a string.
3536 #[ must_use]
3637 #[ allow( clippy:: cast_possible_wrap) ]
37- pub fn digest_from_bytes ( & self , data : & [ u8 ] ) -> Vec < u32 > {
38+ pub fn digest_from_bytes ( & self , data : & [ u8 ] ) -> LzDigest {
3839 let n = data. len ( ) ;
39- let mut seen: HashSet < u32 > = HashSet :: new ( ) ;
40- let mut hashes: Vec < u32 > = Vec :: new ( ) ;
40+ let mut seen = HashSet :: new ( ) ;
41+ let mut hashes = Vec :: new ( ) ;
4142 let mut start = 0usize ;
4243
4344 while start < n {
@@ -57,89 +58,134 @@ impl Lzjd {
5758 }
5859
5960 hashes. sort_unstable_by_key ( |& v| v as i32 ) ;
60- if hashes. len ( ) > self . k {
61- hashes. truncate ( self . k ) ;
62- } else {
61+ if hashes. len ( ) != self . k {
6362 hashes. resize ( self . k , 0 ) ;
6463 }
65- hashes
64+
65+ LzDigest { digest : hashes }
66+ }
67+
68+ /// Convenience: compute digest and similarity between two raw byte inputs
69+ #[ inline]
70+ #[ must_use]
71+ pub fn similarity_bytes ( & self , a : & [ u8 ] , b : & [ u8 ] ) -> f32 {
72+ let da = self . digest_from_bytes ( a) ;
73+ let db = self . digest_from_bytes ( b) ;
74+ da. similarity ( & db)
75+ }
76+ }
77+
78+ /// LZJD digest
79+ #[ derive( Clone , Debug , Eq , PartialEq , Hash ) ]
80+ pub struct LzDigest {
81+ digest : Vec < u32 > ,
82+ }
83+
84+ impl LzDigest {
85+ /// Convert an LZJD digest from base64 to an [`LzDigest`]
86+ ///
87+ /// # Errors
88+ ///
89+ /// Returns an error if the input is not valid base64 or if the decoded length is not a multiple of 4.
90+ pub fn from_string ( s : & str ) -> Result < Self , String > {
91+ let bytes = general_purpose:: STANDARD
92+ . decode ( s)
93+ . map_err ( |e| format ! ( "Base64 decode failed: {e}" ) ) ?;
94+
95+ if bytes. len ( ) % 4 != 0 {
96+ return Err ( "Invalid LZJD Base64 length (not multiple of 4)" . into ( ) ) ;
97+ }
98+
99+ let mut digest = Vec :: with_capacity ( bytes. len ( ) / 4 ) ;
100+ for chunk in bytes. chunks_exact ( 4 ) {
101+ let h = u32:: from_be_bytes ( [ chunk[ 0 ] , chunk[ 1 ] , chunk[ 2 ] , chunk[ 3 ] ] ) ;
102+ digest. push ( h) ;
103+ }
104+
105+ Ok ( Self { digest } )
66106 }
67107
68- /// Compute approximate Jaccard similarity between two LZJD digests (both sorted) .
108+ /// Compute approximate Jaccard similarity between two LZJD digests.
69109 ///
70- /// Both inputs should be sorted ascending as signed i32 and represent the k-smallest
71- /// hashes of the original sets. Returns a value in `[0.0, 1.0]`.
110+ /// Returns a value in `[0.0, 1.0]`.
72111 #[ must_use]
73- #[ allow( clippy:: cast_possible_wrap, clippy:: cast_precision_loss, clippy:: comparison_chain) ]
74- pub fn similarity_from_digests ( a : & [ u32 ] , b : & [ u32 ] ) -> f64 {
112+ #[ allow(
113+ clippy:: cast_possible_wrap,
114+ clippy:: cast_precision_loss,
115+ clippy:: comparison_chain
116+ ) ]
117+ pub fn similarity ( & self , other : & Self ) -> f32 {
75118 let mut i = 0usize ;
76119 let mut j = 0usize ;
77120 let mut inter = 0usize ;
78121
79- while i < a . len ( ) && j < b . len ( ) {
80- if a [ i] == b [ j] {
122+ while i < self . digest . len ( ) && j < other . digest . len ( ) {
123+ if self . digest [ i] == other . digest [ j] {
81124 inter += 1 ;
82125 i += 1 ;
83126 j += 1 ;
84- } else if ( a [ i] as i32 ) < ( b [ j] as i32 ) {
127+ } else if ( self . digest [ i] as i32 ) < ( other . digest [ j] as i32 ) {
85128 i += 1 ;
86129 } else {
87130 j += 1 ;
88131 }
89132 }
90133
91- let union = a . len ( ) + b . len ( ) - inter;
134+ let union = self . digest . len ( ) + other . digest . len ( ) - inter;
92135 if union == 0 {
93136 0.0
94137 } else {
95- inter as f64 / union as f64
138+ inter as f32 / union as f32
96139 }
97140 }
141+ }
98142
99- /// Convenience: compute digest and similarity between two raw byte inputs
100- #[ must_use]
101- pub fn similarity_bytes ( & self , a : & [ u8 ] , b : & [ u8 ] ) -> f64 {
102- let da = self . digest_from_bytes ( a) ;
103- let db = self . digest_from_bytes ( b) ;
104- Self :: similarity_from_digests ( & da, & db)
143+ impl From < LzDigest > for String {
144+ fn from ( d : LzDigest ) -> Self {
145+ d. to_string ( )
105146 }
147+ }
106148
107- /// Convert an LZJD digest to a base64-encoded string (jLZJD-compatible format).
108- ///
109- /// Each `u32` entry is written as a big-endian i32 (matching Java's default byte order),
110- /// producing a fixed-size array of `k` 4-byte values including any zero-padding.
111- #[ must_use]
112- pub fn lzjd_digest_to_base64 ( digest : & [ u32 ] ) -> String {
113- let mut bytes = Vec :: with_capacity ( digest. len ( ) * 4 ) ;
114- for & h in digest {
115- bytes. extend_from_slice ( & h. to_be_bytes ( ) ) ;
116- }
117- general_purpose:: STANDARD . encode ( bytes)
149+ impl From < & LzDigest > for String {
150+ fn from ( d : & LzDigest ) -> Self {
151+ d. to_string ( )
118152 }
153+ }
119154
120- /// Create a LZJD digest from a base64-encoded string (jLZJD-compatible format).
121- ///
122- /// Reads big-endian i32 values, preserving zero-padded entries.
123- ///
124- /// # Errors
125- ///
126- /// Returns an error if the base64 string is invalid or not a multiple of 4 bytes.
127- pub fn lzjd_digest_from_base64 ( b64 : & str ) -> Result < Vec < u32 > , String > {
128- let bytes = general_purpose:: STANDARD
129- . decode ( b64)
130- . map_err ( |e| format ! ( "Base64 decode failed: {e}" ) ) ?;
155+ impl From < & [ u8 ] > for LzDigest {
156+ fn from ( data : & [ u8 ] ) -> Self {
157+ Lzjd :: default ( ) . digest_from_bytes ( data)
158+ }
159+ }
131160
132- if bytes. len ( ) % 4 != 0 {
133- return Err ( "Invalid LZJD Base64 length (not multiple of 4)" . into ( ) ) ;
161+ impl std:: fmt:: Display for LzDigest {
162+ fn fmt ( & self , f : & mut std:: fmt:: Formatter < ' _ > ) -> std:: fmt:: Result {
163+ let mut bytes = Vec :: with_capacity ( self . digest . len ( ) * 4 ) ;
164+ for h in & self . digest {
165+ bytes. extend_from_slice ( & h. to_be_bytes ( ) ) ;
134166 }
167+ write ! ( f, "{}" , general_purpose:: STANDARD . encode( bytes) )
168+ }
169+ }
135170
136- let mut digest = Vec :: with_capacity ( bytes. len ( ) / 4 ) ;
137- for chunk in bytes. chunks_exact ( 4 ) {
138- let h = u32:: from_be_bytes ( [ chunk[ 0 ] , chunk[ 1 ] , chunk[ 2 ] , chunk[ 3 ] ] ) ;
139- digest. push ( h) ;
140- }
171+ #[ cfg( feature = "serde" ) ]
172+ impl serde:: Serialize for LzDigest {
173+ fn serialize < S > ( & self , serializer : S ) -> Result < S :: Ok , S :: Error >
174+ where
175+ S : serde:: Serializer ,
176+ {
177+ serializer. serialize_str ( & self . to_string ( ) )
178+ }
179+ }
141180
142- Ok ( digest)
181+ #[ cfg( feature = "serde" ) ]
182+ impl < ' de > serde:: Deserialize < ' de > for LzDigest {
183+ fn deserialize < D > ( deserializer : D ) -> Result < Self , D :: Error >
184+ where
185+ D : serde:: Deserializer < ' de > ,
186+ {
187+ let s = String :: deserialize ( deserializer) ?;
188+ LzDigest :: from_string ( & s) . map_err ( serde:: de:: Error :: custom)
143189 }
144190}
145191
@@ -153,7 +199,7 @@ mod tests {
153199 let lz = Lzjd :: new ( 16 , 0 ) ;
154200 let data = b"abracadabra abracadabra" ;
155201 let digest = lz. digest_from_bytes ( data) ;
156- assert ! ( digest. len( ) <= 16 ) ;
202+ assert ! ( digest. digest . len( ) <= 16 ) ;
157203 // subsequent call same input yields same digest
158204 let digest2 = lz. digest_from_bytes ( data) ;
159205 assert_eq ! ( digest, digest2) ;
@@ -188,31 +234,32 @@ mod tests {
188234 let digest = lz. digest_from_bytes ( data) ;
189235
190236 // Save to Base64
191- let b64 = Lzjd :: lzjd_digest_to_base64 ( & digest) ;
237+ let b64 = digest. to_string ( ) ;
192238 println ! ( "Base64 LZJD: {b64}" ) ;
193239
194240 // Load from Base64
195- let decoded = Lzjd :: lzjd_digest_from_base64 ( & b64) . unwrap ( ) ;
241+ let decoded = LzDigest :: from_string ( & b64) . unwrap ( ) ;
196242
197243 // Verify exact match
198244 assert_eq ! ( digest, decoded) ;
199245 }
200246
201247 #[ rstest]
202- #[ case( include_str!( "../testdata/lorem_ipsum_5.txt" ) , include_str!( "../testdata/lorem_ipsum_5.lzjd.txt" ) ) ]
203- #[ case( include_str!( "../testdata/lorem_ipsum_6.txt" ) , include_str!( "../testdata/lorem_ipsum_6.lzjd.txt" ) ) ]
204- #[ case( include_str!( "../testdata/lorem_ipsum_10.txt" ) , include_str!( "../testdata/lorem_ipsum_10.lzjd.txt" ) ) ]
248+ #[ case( include_bytes!( "../testdata/lorem_ipsum_5.txt" ) , include_str!( "../testdata/lorem_ipsum_5.lzjd.txt" ) ) ]
249+ #[ case( include_bytes!( "../testdata/lorem_ipsum_6.txt" ) , include_str!( "../testdata/lorem_ipsum_6.lzjd.txt" ) ) ]
250+ #[ case( include_bytes!( "../testdata/lorem_ipsum_10.txt" ) , include_str!( "../testdata/lorem_ipsum_10.lzjd.txt" ) ) ]
251+ #[ case( include_bytes!( "../testdata/random.bin" ) , include_str!( "../testdata/random.bin.lzjd.txt" ) ) ]
205252 #[ test]
206- fn lorem_ipsum ( #[ case] data : & str , #[ case] expected_hash : & str ) {
253+ fn lorem_ipsum ( #[ case] data : & [ u8 ] , #[ case] expected_hash : & str ) {
207254 let lz = Lzjd :: new ( 1024 , 0 ) ;
208- let digest = lz. digest_from_bytes ( data. as_bytes ( ) ) ;
209- let lzjd_hash = Lzjd :: lzjd_digest_to_base64 ( & digest) ;
255+ let digest = lz. digest_from_bytes ( data) ;
256+ let lzjd_hash = digest. to_string ( ) ;
210257 assert_eq ! ( lzjd_hash, expected_hash) ;
211258
212- let decoded = Lzjd :: lzjd_digest_from_base64 ( expected_hash) . unwrap ( ) ;
259+ let decoded = LzDigest :: from_string ( expected_hash) . unwrap ( ) ;
213260 assert_eq ! ( digest, decoded) ;
214261
215- let similarity = Lzjd :: similarity_from_digests ( & digest, & decoded) ;
262+ let similarity = digest. similarity ( & decoded) ;
216263 assert_eq ! ( format!( "{similarity:.2}" ) , "1.00" ) ;
217264 }
218265}
0 commit comments