feat(embedding,notes): implement ADR-0056 Blake3 stable hashing and ADR-0057 priority-aware retention
This commit is contained in:
1 parent
f546579596
commit
3d77e60a02
3 files changed
+38
-10
No files matched your search
+15
-7
@@ -152,7 +152,6 @@ fn fallback_embed(text: &str) -> Vec<f32> {
|
||||
return vec;
|
||||
}
|
||||
|
||||
use std::hash::{Hash, Hasher};
|
||||
for word in words {
|
||||
let clean: String = word
|
||||
.chars()
|
||||
@@ -163,18 +162,17 @@ fn fallback_embed(text: &str) -> Vec<f32> {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
clean.hash(&mut hasher);
|
||||
let h = hasher.finish();
|
||||
let hash_bytes = blake3::hash(clean.as_bytes());
|
||||
let h = u64::from_le_bytes(hash_bytes.as_bytes()[0..8].try_into().unwrap());
|
||||
let idx = (h as usize) % DIM;
|
||||
let sign = if (h >> 32) & 1 == 0 { 1.0f32 } else { -1.0f32 };
|
||||
vec[idx] += sign;
|
||||
|
||||
let chars: Vec<char> = clean.chars().collect();
|
||||
for window in chars.windows(3) {
|
||||
let mut h2 = std::collections::hash_map::DefaultHasher::new();
|
||||
window.hash(&mut h2);
|
||||
let hv = h2.finish();
|
||||
let window_str: String = window.iter().collect();
|
||||
let h2_bytes = blake3::hash(window_str.as_bytes());
|
||||
let hv = u64::from_le_bytes(h2_bytes.as_bytes()[0..8].try_into().unwrap());
|
||||
let idx2 = (hv as usize) % DIM;
|
||||
let s2 = if (hv >> 32) & 1 == 0 { 0.5f32 } else { -0.5f32 };
|
||||
vec[idx2] += s2;
|
||||
@@ -378,4 +376,14 @@ mod tests {
|
||||
assert_eq!(embedding.len(), 384);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fallback_embed_deterministic_stability() {
|
||||
let text = "The quick brown fox jumps over the lazy dog";
|
||||
let emb1 = fallback_embed(text);
|
||||
let emb2 = fallback_embed(text);
|
||||
assert_eq!(emb1.len(), 384);
|
||||
assert_eq!(emb1, emb2);
|
||||
assert!((cosine_similarity(&emb1, &emb2) - 1.0).abs() < 1e-5);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user