feat(embedding,notes): implement ADR-0056 Blake3 stable hashing and ADR-0057 priority-aware retention

This commit is contained in:
Riz Ashraf committed 2026-10-07 17:04:52 +01:00
1 parent f546579596
commit 3d77e60a02
3 files changed
+38 -10

No files matched your search

+15 -7
View File
@@ -152,7 +152,6 @@ fn fallback_embed(text: &str) -> Vec<f32> {
return vec;
}
use std::hash::{Hash, Hasher};
for word in words {
let clean: String = word
.chars()
@@ -163,18 +162,17 @@ fn fallback_embed(text: &str) -> Vec<f32> {
continue;
}
let mut hasher = std::collections::hash_map::DefaultHasher::new();
clean.hash(&mut hasher);
let h = hasher.finish();
let hash_bytes = blake3::hash(clean.as_bytes());
let h = u64::from_le_bytes(hash_bytes.as_bytes()[0..8].try_into().unwrap());
let idx = (h as usize) % DIM;
let sign = if (h >> 32) & 1 == 0 { 1.0f32 } else { -1.0f32 };
vec[idx] += sign;
let chars: Vec<char> = clean.chars().collect();
for window in chars.windows(3) {
let mut h2 = std::collections::hash_map::DefaultHasher::new();
window.hash(&mut h2);
let hv = h2.finish();
let window_str: String = window.iter().collect();
let h2_bytes = blake3::hash(window_str.as_bytes());
let hv = u64::from_le_bytes(h2_bytes.as_bytes()[0..8].try_into().unwrap());
let idx2 = (hv as usize) % DIM;
let s2 = if (hv >> 32) & 1 == 0 { 0.5f32 } else { -0.5f32 };
vec[idx2] += s2;
@@ -378,4 +376,14 @@ mod tests {
assert_eq!(embedding.len(), 384);
}
}
#[test]
fn test_fallback_embed_deterministic_stability() {
let text = "The quick brown fox jumps over the lazy dog";
let emb1 = fallback_embed(text);
let emb2 = fallback_embed(text);
assert_eq!(emb1.len(), 384);
assert_eq!(emb1, emb2);
assert!((cosine_similarity(&emb1, &emb2) - 1.0).abs() < 1e-5);
}
}